Compare commits

..
44 changed files with 6318 additions and 5474 deletions
+12 -1251
View File
File diff suppressed because it is too large Load Diff
+3 -3
View File
@@ -10,12 +10,12 @@ from backend.features.alerts import AlertService
from backend.features.review import TradeJournalService
from backend.features.screener import StrategyTrackingService
from backend.jobs import InProcessJobRunner, JobRegistry, SQLiteJobRunRepository
from chart_data_provider import MarketChartClient
from database import ReviewDatabase
from ifind_client import IfindHttpClient
from mentor_agent import MentorSkillRegistry
from realtime_aggregator import WebRealtimeAggregator
from screener import ScreenerEngine
from backend.data.providers.ifind_client import IfindHttpClient
from backend.data.realtime import WebRealtimeAggregator
from backend.features.market.charts import MarketChartClient
@dataclass(frozen=True)
+8 -1
View File
@@ -1,4 +1,3 @@
from .gateway import DataGateway, build_data_gateway
from .policy import DataPolicyError, DataSourcePolicy
from .quality import DataQualityError, DataQualityGate, QualityEvidence, QualityReport
@@ -12,3 +11,11 @@ __all__ = [
"QualityReport",
"build_data_gateway",
]
def __getattr__(name: str):
if name in {"DataGateway", "build_data_gateway"}:
from .gateway import DataGateway, build_data_gateway
return {"DataGateway": DataGateway, "build_data_gateway": build_data_gateway}[name]
raise AttributeError(name)
+4 -4
View File
@@ -8,10 +8,10 @@ from backend.data.contracts import DataUsage
from backend.data.policy import DataSourcePolicy
from backend.data.providers import IfindProvider, TushareProvider
from backend.data.quality import DataQualityGate, QualityEvidence, QualityReport
from chart_data_provider import EastmoneyChartClient, MarketChartClient
from ifind_client import IfindHttpClient
from realtime_aggregator import WebRealtimeAggregator
from tushare_client import TushareClient
from backend.data.providers.ifind_client import IfindHttpClient
from backend.data.providers.tushare_client import TushareClient
from backend.data.realtime import WebRealtimeAggregator
from backend.features.market.charts import EastmoneyChartClient, MarketChartClient
@dataclass(frozen=True)
+1 -1
View File
@@ -1,6 +1,6 @@
from __future__ import annotations
from ifind_client import IfindHttpClient
from backend.data.providers.ifind_client import IfindHttpClient
class IfindProvider:
+385
View File
@@ -0,0 +1,385 @@
from __future__ import annotations
import copy
import json
import threading
import time
import urllib.error
import urllib.request
from datetime import datetime, timedelta
from typing import Any
class IfindError(RuntimeError):
pass
class IfindHttpClient:
BASE_URL = "https://quantapi.51ifind.com/api/v1"
AUTH_ENDPOINT = "get_access_token"
AUTH_ERROR_CODES = {-1302, -1303, -1304, -4302, -4303}
def __init__(
self,
refresh_token: str = "",
access_token: str = "",
timeout: int = 15,
) -> None:
self.timeout = max(3, int(timeout))
self._refresh_token = str(refresh_token or "").strip()
self._access_token = str(access_token or "").strip()
self._access_expires_at: datetime | None = None
self._token_lock = threading.Lock()
self._cache_lock = threading.Lock()
self._cache: dict[str, dict[str, Any]] = {}
@property
def configured(self) -> bool:
return bool(self._refresh_token or self._access_token)
def set_credentials(self, refresh_token: str, access_token: str = "") -> None:
refresh_token = str(refresh_token or "").strip()
access_token = str(access_token or "").strip()
with self._token_lock:
refresh_changed = refresh_token != self._refresh_token
self._refresh_token = refresh_token
if access_token or refresh_changed:
self._access_token = access_token
self._access_expires_at = None
if refresh_changed:
with self._cache_lock:
self._cache.clear()
def status(self) -> dict[str, Any]:
return {
"configured": self.configured,
"access_ready": bool(self._access_token),
"access_expires_at": (
self._access_expires_at.isoformat(timespec="seconds")
if self._access_expires_at
else ""
),
}
def test_connection(self) -> dict[str, Any]:
payload = self.real_time(
"000001.SH",
["open", "high", "low", "latest", "preClose"],
cache_ttl=0,
)
return {
"ok": bool(payload),
"sample_time": str(payload[0].get("time") or "") if payload else "",
}
def real_time(
self,
codes: str | list[str],
indicators: list[str],
cache_ttl: int = 10,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"real_time_quotation",
{"codes": code_text, "indicators": ",".join(indicators)},
cache_key=f"rq:{code_text}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def history(
self,
codes: str | list[str],
indicators: list[str],
start_date: str,
end_date: str,
cache_ttl: int = 300,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"cmd_history_quotation",
{
"codes": code_text,
"indicators": ",".join(indicators),
"startdate": self._display_date(start_date),
"enddate": self._display_date(end_date),
"functionpara": {"CPS": "forward1", "Fill": "Omit"},
},
cache_key=f"hq:{code_text}:{start_date}:{end_date}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def intraday(
self,
code: str,
start_time: str,
end_time: str,
cache_ttl: int = 20,
) -> list[dict[str, Any]]:
indicators = ["open", "high", "low", "close", "volume", "amount", "avgPrice"]
payload = self._request(
"high_frequency",
{
"codes": self._codes(code),
"indicators": ",".join(indicators),
"starttime": start_time,
"endtime": end_time,
"functionpara": {
"CPS": "forward1",
"Fill": "Previous",
"Timeformat": "LocalTime",
"Interval": "1",
"Limitstart": "09:30:00",
"Limitend": "15:00:00",
},
},
cache_key=f"hf:{code}:{start_time}:{end_time}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def snapshots(
self,
codes: str | list[str],
indicators: list[str],
start_time: str,
end_time: str,
cache_ttl: int = 8,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"snap_shot",
{
"codes": code_text,
"indicators": ",".join(indicators),
"starttime": start_time,
"endtime": end_time,
},
cache_key=f"ss:{code_text}:{start_time}:{end_time}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def wencai(self, query: str, search_type: str = "stock", cache_ttl: int = 300) -> list[dict[str, Any]]:
normalized = " ".join(str(query or "").split())
if not normalized:
raise IfindError("问财查询不能为空。")
payload = self._request(
"smart_stock_picking",
{"searchstring": normalized, "searchtype": search_type},
cache_key=f"wc:{search_type}:{normalized}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def report_query(
self,
codes: str | list[str],
begin_date: str,
end_date: str,
cache_ttl: int = 300,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"report_query",
{
"codes": code_text,
"beginrDate": self._display_date(begin_date),
"endrDate": self._display_date(end_date),
"outputpara": (
"reportDate:Y,thscode:Y,secName:Y,ctime:Y,"
"reportTitle:Y,pdfURL:Y,seq:Y"
),
},
cache_key=f"report:{code_text}:{begin_date}:{end_date}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def _request(
self,
endpoint: str,
body: dict[str, Any],
cache_key: str = "",
cache_ttl: int = 0,
) -> dict[str, Any]:
if not self.configured:
raise IfindError("iFinD 尚未配置。")
if cache_key and cache_ttl > 0:
cached = self._cached(cache_key, cache_ttl)
if cached is not None:
return cached
payload = self._post(endpoint, body, self._ensure_access_token())
if self._is_auth_error(payload) and self._refresh_token:
self._invalidate_access_token()
payload = self._post(endpoint, body, self._ensure_access_token(force=True))
self._validate_payload(payload)
if cache_key and cache_ttl > 0:
with self._cache_lock:
self._cache[cache_key] = {
"created_at": time.time(),
"payload": copy.deepcopy(payload),
}
return payload
def _ensure_access_token(self, force: bool = False) -> str:
with self._token_lock:
now = datetime.now().astimezone().replace(tzinfo=None)
token_valid = bool(self._access_token) and (
self._access_expires_at is None
or self._access_expires_at > now + timedelta(minutes=2)
)
if token_valid and not force:
return self._access_token
if not self._refresh_token:
if self._access_token:
return self._access_token
raise IfindError("iFinD Refresh Token 尚未配置。")
payload = self._post(self.AUTH_ENDPOINT, {}, "", self._refresh_token)
self._validate_payload(payload)
data = payload.get("data") or {}
token = str(data.get("access_token") or "").strip()
if not token:
raise IfindError("iFinD 未返回 Access Token。")
expires_at = self._parse_datetime(data.get("expired_time"))
self._access_token = token
self._access_expires_at = expires_at
return token
def _post(
self,
endpoint: str,
body: dict[str, Any],
access_token: str,
refresh_token: str = "",
) -> dict[str, Any]:
headers = {
"Accept": "application/json",
"Content-Type": "application/json",
"User-Agent": "XiaobaiReviewWeb/1.0",
"ifindlang": "cn",
}
if access_token:
headers["access_token"] = access_token
if refresh_token:
headers["refresh_token"] = refresh_token
request = urllib.request.Request(
f"{self.BASE_URL}/{endpoint}",
data=json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
headers=headers,
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
payload = json.loads(response.read().decode("utf-8"))
except urllib.error.HTTPError as exc:
detail = ""
try:
detail_payload = json.loads(exc.read().decode("utf-8", errors="replace"))
detail = str(detail_payload.get("errmsg") or detail_payload.get("message") or "")
except (json.JSONDecodeError, OSError):
pass
raise IfindError(f"iFinD HTTP {exc.code}{f'{detail[:160]}' if detail else ''}") from exc
except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
raise IfindError("iFinD 数据请求失败。") from exc
if not isinstance(payload, dict):
raise IfindError("iFinD 返回格式不正确。")
return payload
def _cached(self, key: str, ttl: int) -> dict[str, Any] | None:
with self._cache_lock:
cached = self._cache.get(key)
if not cached:
return None
if time.time() - float(cached.get("created_at") or 0) > ttl:
self._cache.pop(key, None)
return None
return copy.deepcopy(cached["payload"])
def _invalidate_access_token(self) -> None:
with self._token_lock:
self._access_token = ""
self._access_expires_at = None
@classmethod
def _validate_payload(cls, payload: dict[str, Any]) -> None:
try:
error_code = int(payload.get("errorcode") or 0)
except (TypeError, ValueError):
error_code = -1
if error_code != 0:
message = str(payload.get("errmsg") or "未知错误")
raise IfindError(f"iFinD 返回错误:{message[:200]}")
@classmethod
def _is_auth_error(cls, payload: dict[str, Any]) -> bool:
try:
error_code = int(payload.get("errorcode") or 0)
except (TypeError, ValueError):
error_code = 0
message = str(payload.get("errmsg") or "").casefold()
return error_code in cls.AUTH_ERROR_CODES or "token" in message or "鉴权" in message
@staticmethod
def _table_rows(payload: dict[str, Any]) -> list[dict[str, Any]]:
tables = payload.get("tables") or []
if isinstance(tables, dict):
tables = [tables]
rows: list[dict[str, Any]] = []
for block in tables if isinstance(tables, list) else []:
if not isinstance(block, dict):
continue
table = block.get("table") or {}
if not isinstance(table, dict):
continue
times = block.get("time") or []
codes = block.get("thscode") or block.get("thscodes") or []
if isinstance(codes, str):
codes = [codes]
lengths = [len(value) for value in table.values() if isinstance(value, list)]
row_count = max(lengths or [len(times) if isinstance(times, list) else 0, 1 if table else 0])
for index in range(row_count):
row: dict[str, Any] = {}
if isinstance(times, list) and index < len(times):
row["time"] = times[index]
if codes:
row["thscode"] = codes[index] if index < len(codes) else codes[0]
for field, values in table.items():
if isinstance(values, list):
row[field] = values[index] if index < len(values) else None
elif index == 0:
row[field] = values
rows.append(row)
return rows
@staticmethod
def _codes(codes: str | list[str]) -> str:
if isinstance(codes, list):
values = [str(code or "").strip().upper() for code in codes]
else:
values = [part.strip().upper() for part in str(codes or "").split(",")]
values = [value for value in values if value]
if not values:
raise IfindError("iFinD 证券代码不能为空。")
if len(values) > 100:
raise IfindError("iFinD 单次证券代码过多。")
return ",".join(values)
@staticmethod
def _display_date(value: str) -> str:
compact = str(value or "").replace("-", "")
if len(compact) != 8 or not compact.isdigit():
raise IfindError("iFinD 日期格式不正确。")
return f"{compact[:4]}-{compact[4:6]}-{compact[6:]}"
@staticmethod
def _parse_datetime(value: Any) -> datetime | None:
text = str(value or "").strip()
if not text:
return None
try:
return datetime.fromisoformat(text)
except ValueError:
return None
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
from collections.abc import Callable
from tushare_client import TushareClient
from backend.data.providers.tushare_client import TushareClient
class TushareProvider:
File diff suppressed because it is too large Load Diff
+426
View File
@@ -0,0 +1,426 @@
from __future__ import annotations
import copy
import http.client
import json
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from datetime import datetime
from threading import Lock
from typing import Any, ClassVar
class RealtimeAggregateError(RuntimeError):
pass
EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get"
EASTMONEY_SECTOR_URL = "https://push2.eastmoney.com/api/qt/clist/get"
TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006"
THS_LIMIT_URL = "https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool"
XGB_POOL_URL = "https://flash-api.xuangubao.cn/api/pool/detail"
BROWSER_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/138.0.0.0 Safari/537.36"
)
@dataclass
class WebRealtimeAggregator:
timeout: int = 8
retry_attempts: int = 3
retry_delay_seconds: float = 0.2
response_cache_ttl_seconds: int = 90
_sector_cache: ClassVar[dict[str, Any]] = {}
_sector_cache_lock: ClassVar[Lock] = Lock()
_response_cache: ClassVar[dict[str, dict[str, Any]]] = {}
_response_cache_lock: ClassVar[Lock] = Lock()
def health_snapshot(self, sector: str = "") -> dict[str, Any]:
started = time.perf_counter()
sources: dict[str, dict[str, Any]] = {}
indices: list[dict[str, Any]] = []
sector_payload: dict[str, Any] | None = None
indices, sources["eastmoney_indices"] = self._capture(self.eastmoney_indices)
if sector.strip():
sector_payload, sources["eastmoney_sector"] = self._capture(
lambda: self.eastmoney_sector(sector)
)
ths_observation, sources["ths_limit_pool"] = self._capture(self.ths_limit_pool)
xgb_observation, sources["xgb_limit_pool"] = self._capture(self.xgb_limit_pool)
index_times = [int(item.get("quote_time_epoch") or 0) for item in indices or []]
now = datetime.now().astimezone()
max_skew = 120 if now.hour >= 15 else 15
index_consistent = bool(index_times) and max(index_times) - min(index_times) <= max_skew
ready = (
bool(indices)
and len(indices) == 3
and index_consistent
and (not sector.strip() or bool(sector_payload))
)
return {
"ready": ready,
"isolated": True,
"generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"indices": indices or [],
"index_consistent": index_consistent,
"sector": sector_payload,
"sources": sources,
"observations": {
"ths_limit_pool": ths_observation,
"xgb_limit_pool": xgb_observation,
},
"policy": {
"integration": "heaven_realtime_fallback",
"max_index_time_skew_seconds": max_skew,
"notice": "聚合源仅作为盘中观势的实时指数与板块外显,主行情快照仍由Tushare维护。",
},
}
def eastmoney_indices(self) -> list[dict[str, Any]]:
try:
payload = self._get_json(
EASTMONEY_INDEX_URL,
{
"secids": "1.000001,0.399001,0.399006",
"fltt": "2",
"invt": "2",
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124",
},
referer="https://quote.eastmoney.com/",
)
except RealtimeAggregateError:
return self.tencent_indices()
cache_meta = payload.get("_aggregate_cache") or {}
rows = list((payload.get("data") or {}).get("diff") or [])
result = []
for row in rows:
code = str(row.get("f12") or "")
if code not in {"000001", "399001", "399006"}:
continue
epoch = int(_number(row.get("f124")))
result.append(
{
"code": code,
"name": row.get("f14") or code,
"price": _number(row.get("f2")),
"change": _number(row.get("f3")),
"change_amount": _number(row.get("f4")),
"open": _number(row.get("f17")),
"high": _number(row.get("f15")),
"low": _number(row.get("f16")),
"previous_close": _number(row.get("f18")),
"amount_billion": round(_number(row.get("f6")) / 100000000, 2),
"quote_time_epoch": epoch,
"quote_time": (
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
if epoch else ""
),
"source": (
"eastmoney_push2_cache" if cache_meta else "eastmoney_push2"
),
"cache_age_seconds": cache_meta.get("age_seconds", 0),
}
)
if len(result) != 3:
raise RealtimeAggregateError(f"Eastmoney returned {len(result)}/3 indices")
return result
def tencent_indices(self) -> list[dict[str, Any]]:
raw, cache_age = self._get_text(
TENCENT_INDEX_URL,
referer="https://gu.qq.com/",
encoding="gb18030",
)
result = []
for line in raw.splitlines():
if '="' not in line:
continue
fields = line.split('="', 1)[1].rsplit('";', 1)[0].split("~")
if len(fields) < 38:
continue
code = fields[2]
if code not in {"000001", "399001", "399006"}:
continue
try:
quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S").astimezone()
except ValueError as exc:
raise RealtimeAggregateError(
f"Tencent returned invalid quote time for {code}"
) from exc
result.append(
{
"code": code,
"name": fields[1] or code,
"price": _number(fields[3]),
"change": _number(fields[32]),
"change_amount": _number(fields[31]),
"open": _number(fields[5]),
"high": _number(fields[33]),
"low": _number(fields[34]),
"previous_close": _number(fields[4]),
"amount_billion": round(_number(fields[37]) / 10000, 2),
"quote_time_epoch": int(quote_time.timestamp()),
"quote_time": quote_time.isoformat(timespec="seconds"),
"source": "tencent_qt_cache" if cache_age else "tencent_qt",
"cache_age_seconds": cache_age,
}
)
if len(result) != 3:
raise RealtimeAggregateError(f"Tencent returned {len(result)}/3 indices")
return result
def eastmoney_sector(self, query: str) -> dict[str, Any]:
target = _normalize_sector(query)
candidates = self._eastmoney_sector_catalog()
matched = _match_sector(candidates, target)
if not matched:
raise RealtimeAggregateError(f"Eastmoney sector not found: {query}")
epoch = int(_number(matched.get("f124")))
return {
"code": matched.get("f12") or "",
"name": matched.get("f14") or query,
"price": _number(matched.get("f2")),
"change": _number(matched.get("f3")),
"change_amount": _number(matched.get("f4")),
"turnover_rate": _number(matched.get("f8")),
"up_count": int(_number(matched.get("f104"))),
"down_count": int(_number(matched.get("f105"))),
"leader": matched.get("f128") or "--",
"leader_code": matched.get("f140") or "",
"leading_pct": _number(matched.get("f136")),
"quote_time_epoch": epoch,
"quote_time": (
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
if epoch else ""
),
"source": "eastmoney_push2",
"match_query": query,
}
def _eastmoney_sector_catalog(self) -> list[dict[str, Any]]:
now = time.time()
with self._sector_cache_lock:
cached = self._sector_cache.get("eastmoney")
if cached and now - float(cached.get("created_at") or 0) < 600:
return list(cached.get("rows") or [])
def load_page(page: int) -> list[dict[str, Any]]:
payload = self._get_json(
EASTMONEY_SECTOR_URL,
{
"pn": str(page),
"pz": "100",
"po": "1",
"np": "1",
"fltt": "2",
"invt": "2",
"fid": "f3",
"fs": "m:90+t:2",
"fields": "f12,f14,f2,f3,f4,f8,f104,f105,f128,f136,f140,f124",
},
referer="https://quote.eastmoney.com/center/boardlist.html",
)
return list((payload.get("data") or {}).get("diff") or [])
with ThreadPoolExecutor(max_workers=5) as executor:
pages = list(executor.map(load_page, range(1, 6)))
rows = [row for page in pages for row in page]
if not rows:
raise RealtimeAggregateError("Eastmoney sector catalog is empty")
with self._sector_cache_lock:
self._sector_cache["eastmoney"] = {"created_at": now, "rows": rows}
return rows
def ths_limit_pool(self) -> dict[str, Any]:
payload = self._get_json(
THS_LIMIT_URL,
{"page": "1", "limit": "3", "field": "199112"},
referer="https://data.10jqka.com.cn/limit_up/",
)
data = payload.get("data") or payload
return {
"available": True,
"keys": sorted(str(key) for key in data.keys()) if isinstance(data, dict) else [],
"source": "ths_web_dataapi",
}
def xgb_limit_pool(self) -> dict[str, Any]:
payload = self._get_json(
XGB_POOL_URL,
{"pool_name": "limit_up"},
referer="https://xuangubao.cn/",
)
data = payload.get("data") or {}
rows = data if isinstance(data, list) else data.get("pool") or data.get("list") or []
return {
"available": True,
"count": len(rows) if isinstance(rows, list) else 0,
"source": "xuangubao_web_api",
}
def _capture(self, operation):
started = time.perf_counter()
try:
value = operation()
return value, {
"ok": True,
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"error": "",
}
except Exception as exc:
return None, {
"ok": False,
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"error": str(exc)[:500],
}
def _get_json(
self,
url: str,
params: dict[str, str],
referer: str,
) -> dict[str, Any]:
request_url = f"{url}?{urllib.parse.urlencode(params)}"
last_error: Exception | None = None
attempts = max(1, int(self.retry_attempts))
for attempt in range(attempts):
request = urllib.request.Request(
request_url,
headers={
"Accept": "application/json,text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
content_type = response.headers.get("Content-Type", "")
raw = response.read().decode("utf-8", errors="replace")
if "json" not in content_type.lower() and not raw.lstrip().startswith(("{", "[")):
raise RealtimeAggregateError(
f"non-JSON response: {raw[:120].strip()}"
)
payload = json.loads(raw)
if not isinstance(payload, dict):
raise RealtimeAggregateError("unexpected response shape")
if payload.get("rc") not in (None, 0):
raise RealtimeAggregateError(f"provider rc={payload.get('rc')}")
with self._response_cache_lock:
self._response_cache[request_url] = {
"created_at": time.time(),
"payload": copy.deepcopy(payload),
}
return payload
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
json.JSONDecodeError,
RealtimeAggregateError,
) as exc:
last_error = exc
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
time.sleep(self.retry_delay_seconds * (attempt + 1))
now = time.time()
with self._response_cache_lock:
cached = self._response_cache.get(request_url)
cache_age = now - float((cached or {}).get("created_at") or 0)
if cached and cache_age <= self.response_cache_ttl_seconds:
payload = copy.deepcopy(cached.get("payload") or {})
payload["_aggregate_cache"] = {"age_seconds": round(cache_age, 1)}
return payload
raise RealtimeAggregateError(f"request failed after {attempts} attempts: {last_error}") from last_error
def _get_text(
self,
request_url: str,
referer: str,
encoding: str = "utf-8",
) -> tuple[str, float]:
cache_key = f"text:{request_url}"
last_error: Exception | None = None
attempts = max(1, int(self.retry_attempts))
for attempt in range(attempts):
request = urllib.request.Request(
request_url,
headers={
"Accept": "text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
raw = response.read().decode(encoding, errors="replace")
if not raw.strip():
raise RealtimeAggregateError("empty text response")
with self._response_cache_lock:
self._response_cache[cache_key] = {
"created_at": time.time(),
"payload": raw,
}
return raw, 0
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
RealtimeAggregateError,
) as exc:
last_error = exc
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
time.sleep(self.retry_delay_seconds * (attempt + 1))
now = time.time()
with self._response_cache_lock:
cached = self._response_cache.get(cache_key)
cache_age = now - float((cached or {}).get("created_at") or 0)
if cached and cache_age <= self.response_cache_ttl_seconds:
return str(cached.get("payload") or ""), round(cache_age, 1)
raise RealtimeAggregateError(
f"text request failed after {attempts} attempts: {last_error}"
) from last_error
def _normalize_sector(value: Any) -> str:
text = str(value or "").strip().replace(" ", "")
for suffix in ("板块", "概念", "行业", "", "", "(A股)", "A股)"):
text = text.replace(suffix, "")
aliases = {"元器件": "元件", "电子元器件": "元件"}
return aliases.get(text, text)
def _match_sector(rows: list[dict[str, Any]], target: str) -> dict[str, Any] | None:
exact = [row for row in rows if _normalize_sector(row.get("f14")) == target]
if exact:
return min(exact, key=lambda row: len(str(row.get("f14") or "")))
fuzzy = [
row for row in rows
if target and (
target in _normalize_sector(row.get("f14"))
or _normalize_sector(row.get("f14")) in target
)
]
return min(fuzzy, key=lambda row: len(_normalize_sector(row.get("f14")))) if fuzzy else None
def _number(value: Any, default: float = 0.0) -> float:
try:
return float(value)
except (TypeError, ValueError):
return default
+13
View File
@@ -0,0 +1,13 @@
"""Public market data, search, detail and chart feature."""
from .charts import ChartDataError, EastmoneyChartClient, MarketChartClient
from .repository import MarketRepositoryMixin
from .service import MarketServiceMixin
__all__ = [
"ChartDataError",
"EastmoneyChartClient",
"MarketChartClient",
"MarketRepositoryMixin",
"MarketServiceMixin",
]
+497
View File
@@ -0,0 +1,497 @@
from __future__ import annotations
import http.client
import json
import re
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from datetime import datetime, time as dt_time, timedelta
from threading import Lock
from typing import Any, ClassVar
from backend.data.providers.ifind_client import IfindError, IfindHttpClient
class ChartDataError(RuntimeError):
pass
TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get"
BOARD_LIST_URL = "https://push2delay.eastmoney.com/api/qt/clist/get"
BROWSER_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/138.0.0.0 Safari/537.36"
)
INDEX_SECIDS = {
"000001.SH": "1.000001",
"399001.SZ": "0.399001",
"399006.SZ": "0.399006",
}
class MarketChartClient:
"""Prefer iFinD for display charts and retain Eastmoney as a last resort."""
def __init__(self, ifind: IfindHttpClient, fallback: "EastmoneyChartClient") -> None:
self.ifind = ifind
self.fallback = fallback
def stock_intraday(self, code: str) -> dict[str, Any]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
ifind_code = _stock_market_code(normalized)
try:
return self._ifind_intraday(ifind_code, "stock", normalized)
except (IfindError, ChartDataError):
return self.fallback.stock_intraday(normalized)
def stock_daily(self, code: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
return self._ifind_daily(_stock_market_code(normalized), end_date, limit)
def index_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(identifier or "").strip().upper()
if normalized not in INDEX_SECIDS:
raise ChartDataError("Unsupported index")
return self._ifind_daily(normalized, end_date, limit)
def board_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(identifier or "").strip().upper()
if not normalized:
raise ChartDataError("Invalid board code")
return self._ifind_daily(normalized, end_date, limit)
def index_intraday(self, identifier: str) -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
if normalized not in INDEX_SECIDS:
raise ChartDataError("Unsupported index")
try:
return self._ifind_intraday(normalized, "index", normalized)
except (IfindError, ChartDataError):
return self.fallback.index_intraday(normalized)
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
try:
return self._ifind_intraday(normalized, "board", normalized, name)
except (IfindError, ChartDataError):
return self.fallback.board_intraday(normalized, name)
def _ifind_intraday(
self,
ifind_code: str,
entity_type: str,
identifier: str,
name: str = "",
) -> dict[str, Any]:
if not self.ifind.configured:
raise ChartDataError("iFinD is not configured")
now = datetime.now().astimezone()
rows: list[dict[str, Any]] = []
for offset in range(0, 8):
candidate = now.date() - timedelta(days=offset)
if candidate.weekday() >= 5:
continue
display_date = candidate.isoformat()
rows = self.ifind.intraday(
ifind_code,
f"{display_date} 09:30:00",
f"{display_date} 15:00:00",
cache_ttl=20 if offset == 0 else 6 * 60 * 60,
)
if rows:
break
points = [point for row in rows if (point := _ifind_point(row))]
if not points:
raise ChartDataError("No iFinD intraday chart data returned")
latest_date = points[-1]["date"]
points = [point for point in points if point["date"] == latest_date]
previous_close = self._previous_close(ifind_code, latest_date, points[0]["open"])
return {
"entity_type": entity_type,
"identifier": identifier,
"name": name,
"code": identifier,
"trade_date": latest_date,
"previous_close": previous_close,
"points": points,
"source": "ifind",
}
def _ifind_daily(
self, ifind_code: str, end_date: str, limit: int
) -> list[dict[str, Any]]:
if not self.ifind.configured:
raise ChartDataError("iFinD is not configured")
compact_end = str(end_date or "").replace("-", "")
if not re.fullmatch(r"\d{8}", compact_end):
raise ChartDataError("Invalid chart end date")
end = datetime.strptime(compact_end, "%Y%m%d")
start = (end - timedelta(days=max(190, limit * 3))).strftime("%Y%m%d")
try:
rows = self.ifind.history(
ifind_code,
["open", "high", "low", "close", "volume", "amount"],
start,
compact_end,
cache_ttl=300,
)
except IfindError as exc:
raise ChartDataError("No iFinD daily chart data returned") from exc
normalized = []
for row in rows:
stamp = str(row.get("time") or "").strip()
trade_date = stamp[:10]
close = _number(row.get("close"))
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", trade_date) or close <= 0:
continue
normalized.append(
{
"trade_date": trade_date,
"open": _number(row.get("open")),
"high": _number(row.get("high")),
"low": _number(row.get("low")),
"close": close,
"volume": _number(row.get("volume")),
"amount_billion": _number(row.get("amount")) / 100_000_000,
}
)
normalized.sort(key=lambda row: row["trade_date"])
for index, row in enumerate(normalized):
previous = normalized[index - 1]["close"] if index > 0 else 0
row["change"] = round((row["close"] / previous - 1) * 100, 4) if previous else 0.0
market_now = datetime.now().astimezone()
today = market_now.strftime("%Y%m%d")
market_open = (
market_now.weekday() < 5
and market_now.time().replace(tzinfo=None) >= dt_time(9, 30)
)
today_display = market_now.date().isoformat()
if normalized and normalized[-1]["trade_date"] == today_display:
current_bar = normalized[-1]
current_bar_is_valid = (
current_bar["open"] > 0
and current_bar["high"] >= max(current_bar["open"], current_bar["close"])
and 0 < current_bar["low"] <= min(current_bar["open"], current_bar["close"])
and (current_bar["volume"] > 0 or current_bar["amount_billion"] > 0)
)
if not market_open or not current_bar_is_valid:
normalized.pop()
if compact_end == today and market_open:
try:
quote_rows = self.ifind.real_time(
ifind_code,
["open", "high", "low", "latest", "preClose", "volume", "amount"],
cache_ttl=10,
)
quote = quote_rows[0] if quote_rows else {}
latest = _number(quote.get("latest"))
previous = _number(quote.get("preClose"))
open_price = _number(quote.get("open"))
high = _number(quote.get("high"))
low = _number(quote.get("low"))
volume = _number(quote.get("volume"))
amount = _number(quote.get("amount"))
quote_date = str(quote.get("time") or "")[:10].replace("-", "")
quote_is_current = not quote_date or quote_date == today
has_market_activity = volume > 0 or amount > 0
if (
latest > 0
and open_price > 0
and high >= max(open_price, latest)
and 0 < low <= min(open_price, latest)
and has_market_activity
and quote_is_current
):
realtime = {
"trade_date": end.strftime("%Y-%m-%d"),
"open": open_price,
"high": high,
"low": low,
"close": latest,
"change": round((latest / previous - 1) * 100, 4) if previous else 0.0,
"volume": volume,
"amount_billion": amount / 100_000_000,
"realtime": True,
}
if normalized and normalized[-1]["trade_date"] == realtime["trade_date"]:
normalized[-1] = realtime
else:
normalized.append(realtime)
except IfindError:
pass
if not normalized:
raise ChartDataError("No iFinD daily chart data returned")
return normalized[-max(20, min(180, int(limit))):]
def _previous_close(self, code: str, trade_date: str, fallback: float) -> float:
today = datetime.now().astimezone().date().isoformat()
if trade_date == today:
try:
quote = self.ifind.real_time(code, ["preClose"], cache_ttl=20)
value = _number((quote[0] if quote else {}).get("preClose"))
if value > 0:
return value
except IfindError:
pass
end = datetime.strptime(trade_date, "%Y-%m-%d")
try:
rows = self.ifind.history(
code,
["close"],
(end - timedelta(days=12)).strftime("%Y%m%d"),
end.strftime("%Y%m%d"),
cache_ttl=6 * 60 * 60,
)
closes = [_number(row.get("close")) for row in rows if _number(row.get("close")) > 0]
if len(closes) >= 2:
return closes[-2]
except IfindError:
pass
return fallback
@dataclass
class EastmoneyChartClient:
"""Isolated display-only minute chart source.
The returned data must not be used by market snapshots, scoring, screening,
or divination. Its only consumer is a chart-rendering endpoint.
"""
timeout: int = 6
cache_ttl_seconds: int = 20
retry_attempts: int = 2
_cache: ClassVar[dict[str, dict[str, Any]]] = {}
_cache_lock: ClassVar[Lock] = Lock()
_board_catalog: ClassVar[dict[str, dict[str, str]]] = {}
_board_catalog_at: ClassVar[float] = 0.0
_board_catalog_lock: ClassVar[Lock] = Lock()
def stock_intraday(self, code: str) -> dict[str, Any]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
market = "1" if normalized.startswith(("5", "6", "9")) else "0"
return self._intraday(f"{market}.{normalized}", "stock", normalized)
def index_intraday(self, identifier: str) -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
secid = INDEX_SECIDS.get(normalized)
if not secid:
raise ChartDataError("Unsupported index")
return self._intraday(secid, "index", normalized)
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
if re.fullmatch(r"BK\d{4}", normalized):
board_code = normalized
else:
board_code = self._resolve_board_code(name or identifier)
return self._intraday(f"90.{board_code}", "board", board_code)
def _intraday(self, secid: str, entity_type: str, identifier: str) -> dict[str, Any]:
cache_key = f"{entity_type}:{identifier}"
cached = self._get_cached(cache_key)
if cached is not None:
return cached
payload = self._request_json(
TRENDS_URL,
{
"secid": secid,
"fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13",
"fields2": "f51,f52,f53,f54,f55,f56,f57,f58",
"iscr": "0",
"ndays": "1",
},
"https://quote.eastmoney.com/",
)
data = payload.get("data") or {}
points = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))]
if not points:
raise ChartDataError("No intraday chart data returned")
result = {
"entity_type": entity_type,
"identifier": identifier,
"name": str(data.get("name") or ""),
"code": str(data.get("code") or identifier),
"trade_date": points[-1]["date"],
"previous_close": _number(data.get("preClose")),
"points": points,
}
with self._cache_lock:
self._cache[cache_key] = {"created_at": time.time(), "payload": result}
return result
def _get_cached(self, cache_key: str) -> dict[str, Any] | None:
with self._cache_lock:
cached = self._cache.get(cache_key)
if not cached:
return None
if time.time() - float(cached.get("created_at") or 0) > self.cache_ttl_seconds:
with self._cache_lock:
self._cache.pop(cache_key, None)
return None
return dict(cached["payload"])
def _resolve_board_code(self, name: str) -> str:
normalized = _normalize_name(name)
if not normalized:
raise ChartDataError("Board name is required")
catalog = self._load_board_catalog()
item = catalog.get(normalized)
if not item:
raise ChartDataError("No matching chart board")
return item["code"]
def _load_board_catalog(self) -> dict[str, dict[str, str]]:
now = time.time()
with self._board_catalog_lock:
if self._board_catalog and now - self._board_catalog_at < 6 * 60 * 60:
return dict(self._board_catalog)
rows: list[dict[str, Any]] = []
for board_type in ("1", "2", "3"):
for page in range(1, 6):
payload = self._request_json(
BOARD_LIST_URL,
{
"pn": str(page),
"pz": "100",
"po": "1",
"np": "1",
"fltt": "2",
"invt": "2",
"fid": "f3",
"fs": f"m:90+t:{board_type}",
"fields": "f12,f14",
},
"https://quote.eastmoney.com/center/boardlist.html",
)
page_rows = (payload.get("data") or {}).get("diff") or []
rows.extend(page_rows)
if len(page_rows) < 100:
break
catalog: dict[str, dict[str, str]] = {}
for row in rows:
code = str(row.get("f12") or "").strip().upper()
board_name = str(row.get("f14") or "").strip()
if re.fullmatch(r"BK\d{4}", code) and board_name:
catalog.setdefault(_normalize_name(board_name), {"code": code, "name": board_name})
if not catalog:
raise ChartDataError("Board chart directory is unavailable")
with self._board_catalog_lock:
type(self)._board_catalog = catalog
type(self)._board_catalog_at = now
return dict(catalog)
def _request_json(
self, url: str, params: dict[str, str], referer: str
) -> dict[str, Any]:
request_url = f"{url}?{urllib.parse.urlencode(params)}"
last_error: Exception | None = None
for attempt in range(max(1, int(self.retry_attempts))):
request = urllib.request.Request(
request_url,
headers={
"Accept": "application/json,text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
payload = json.loads(response.read().decode("utf-8"))
if not isinstance(payload, dict):
raise ChartDataError("Invalid intraday chart response")
return payload
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
json.JSONDecodeError,
ChartDataError,
) as exc:
last_error = exc
if attempt + 1 < self.retry_attempts:
time.sleep(0.12)
raise ChartDataError("Intraday chart request failed") from last_error
def _parse_trend(raw: Any) -> dict[str, Any] | None:
fields = str(raw or "").split(",")
if len(fields) < 8 or " " not in fields[0]:
return None
stamp = fields[0].strip()
trade_date, trade_time = stamp.split(" ", 1)
close = _number(fields[2])
if close <= 0:
return None
return {
"date": trade_date,
"time": trade_time[:5],
"open": _number(fields[1]),
"close": close,
"high": _number(fields[3]),
"low": _number(fields[4]),
"volume": _number(fields[5]),
"amount": _number(fields[6]),
"average": _number(fields[7]),
}
def _ifind_point(row: dict[str, Any]) -> dict[str, Any] | None:
stamp = str(row.get("time") or "").strip()
if " " not in stamp:
return None
trade_date, trade_time = stamp.split(" ", 1)
close = _number(row.get("close"))
if close <= 0:
return None
return {
"date": trade_date,
"time": trade_time[:5],
"open": _number(row.get("open")),
"close": close,
"high": _number(row.get("high")),
"low": _number(row.get("low")),
"volume": _number(row.get("volume")),
"amount": _number(row.get("amount")),
"average": _number(row.get("avgPrice")),
}
def _stock_market_code(code: str) -> str:
if code.startswith(("4", "8", "9")):
suffix = "BJ"
elif code.startswith("6"):
suffix = "SH"
else:
suffix = "SZ"
return f"{code}.{suffix}"
def _number(value: Any) -> float:
try:
return float(value or 0)
except (TypeError, ValueError):
return 0.0
def _normalize_name(value: Any) -> str:
normalized = re.sub(r"[\s·・()()\-_/]", "", str(value or "")).casefold()
return re.sub(r"(?:概念|行业|[ⅠⅡⅢ])$", "", normalized)
+222
View File
@@ -0,0 +1,222 @@
from __future__ import annotations
import json
from datetime import datetime
from typing import Any
class MarketRepositoryMixin:
def get_snapshot(self, trade_date: str) -> dict[str, Any] | None:
with self.connect() as connection:
row = connection.execute(
"SELECT payload FROM dashboard_snapshots WHERE trade_date = ?",
(trade_date,),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def get_latest_real_snapshot(
self, trade_date: str, strictly_before: bool = False
) -> dict[str, Any] | None:
operator = "<" if strictly_before else "<="
with self.connect() as connection:
row = connection.execute(
f"""
SELECT payload FROM dashboard_snapshots
WHERE trade_date {operator} ? AND source != 'demo'
ORDER BY trade_date DESC LIMIT 1
""",
(trade_date,),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def save_snapshot(self, trade_date: str, source: str, payload: dict[str, Any]) -> None:
updated_at = datetime.now().astimezone().isoformat(timespec="seconds")
record_count = sum(
len(payload.get(key) or [])
for key in ("limits", "broken", "down_limits", "yesterday_limits")
)
content = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
with self.connect() as connection:
connection.execute(
"""
INSERT INTO dashboard_snapshots
(trade_date, source, payload, record_count, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(trade_date) DO UPDATE SET
source = excluded.source,
payload = excluded.payload,
record_count = excluded.record_count,
updated_at = excluded.updated_at
""",
(trade_date, source, content, record_count, updated_at),
)
def get_data_snapshot(self, kind: str, cache_key: str) -> dict[str, Any] | None:
with self.connect() as connection:
row = connection.execute(
"SELECT payload FROM data_snapshots WHERE kind = ? AND cache_key = ?",
(kind, cache_key),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def get_latest_data_snapshot(
self,
kind: str,
cache_key_prefix: str,
maximum_cache_key: str,
exclude_source: str = "",
) -> dict[str, Any] | None:
source_clause = " AND source != ?" if exclude_source else ""
parameters: list[Any] = [kind, f"{cache_key_prefix}%", maximum_cache_key]
if exclude_source:
parameters.append(exclude_source)
with self.connect() as connection:
row = connection.execute(
f"""
SELECT payload FROM data_snapshots
WHERE kind = ? AND cache_key LIKE ? AND cache_key <= ?{source_clause}
ORDER BY cache_key DESC LIMIT 1
""",
parameters,
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def save_data_snapshot(
self, kind: str, cache_key: str, source: str, payload: dict[str, Any]
) -> None:
updated_at = datetime.now().astimezone().isoformat(timespec="seconds")
content = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
with self.connect() as connection:
connection.execute(
"""
INSERT INTO data_snapshots (kind, cache_key, source, payload, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(kind, cache_key) DO UPDATE SET
source = excluded.source,
payload = excluded.payload,
updated_at = excluded.updated_at
""",
(kind, cache_key, source, content, updated_at),
)
def search_stock_master(self, query: str, limit: int = 12) -> list[dict[str, Any]]:
text = str(query or "").strip()
if not text:
return []
escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
with self.connect() as connection:
rows = connection.execute(
"""
SELECT ts_code, code, name, industry, market, list_date
FROM stock_master
WHERE code = ? OR name = ? OR name LIKE ? ESCAPE '\\'
ORDER BY
CASE WHEN code = ? THEN 0 WHEN name = ? THEN 1 ELSE 2 END,
list_date DESC,
code
LIMIT ?
""",
(text, text, f"%{escaped}%", text, text, max(1, min(30, int(limit)))),
).fetchall()
return [dict(row) for row in rows]
def list_snapshot_payloads(self, end_date: str, limit: int = 260) -> list[dict[str, Any]]:
with self.connect() as connection:
rows = connection.execute(
"""
SELECT trade_date, payload FROM dashboard_snapshots
WHERE trade_date <= ? ORDER BY trade_date DESC LIMIT ?
""",
(end_date, limit),
).fetchall()
result: list[dict[str, Any]] = []
for row in reversed(rows):
try:
payload = json.loads(row["payload"])
except json.JSONDecodeError:
continue
payload["_snapshot_date"] = row["trade_date"]
result.append(payload)
return result
def start_sync(self, trade_date: str, source: str) -> int:
started_at = datetime.now().astimezone().isoformat(timespec="seconds")
with self.connect() as connection:
cursor = connection.execute(
"""
INSERT INTO sync_runs (trade_date, source, status, started_at)
VALUES (?, ?, 'running', ?)
""",
(trade_date, source, started_at),
)
return int(cursor.lastrowid)
def finish_sync(
self,
sync_id: int,
status: str,
record_count: int = 0,
message: str = "",
source: str | None = None,
) -> None:
finished_at = datetime.now().astimezone().isoformat(timespec="seconds")
with self.connect() as connection:
connection.execute(
"""
UPDATE sync_runs
SET status = ?, finished_at = ?, record_count = ?, message = ?,
source = COALESCE(?, source)
WHERE id = ?
""",
(status, finished_at, record_count, message[:1000], source, sync_id),
)
def status(self) -> dict[str, Any]:
with self.connect() as connection:
last_sync = connection.execute(
"""
SELECT id, trade_date, source, status, started_at, finished_at,
record_count, message
FROM sync_runs ORDER BY id DESC LIMIT 1
"""
).fetchone()
snapshot_stats = connection.execute(
"""
SELECT COUNT(*) AS dates, COALESCE(SUM(record_count), 0) AS records,
MAX(updated_at) AS updated_at
FROM dashboard_snapshots
"""
).fetchone()
watchlist_count = connection.execute("SELECT COUNT(*) FROM watchlist").fetchone()[0]
note_count = connection.execute("SELECT COUNT(*) FROM review_notes").fetchone()[0]
return {
"database": str(self.path.name),
"snapshot_dates": int(snapshot_stats["dates"]),
"snapshot_records": int(snapshot_stats["records"]),
"updated_at": snapshot_stats["updated_at"],
"last_sync": dict(last_sync) if last_sync else None,
"watchlist_count": int(watchlist_count),
"note_count": int(note_count),
}
+949
View File
@@ -0,0 +1,949 @@
from __future__ import annotations
import copy
import re
from datetime import date, datetime, time as dt_time, timedelta
from typing import Any
from backend.bootstrap.config import (
normalize_date,
tushare_code,
validate_stock_code,
validate_text,
)
from backend.data.providers.ifind_client import IfindError
from backend.data.providers.tushare_client import TushareClient, TushareError
from backend.features.market.charts import ChartDataError
from backend.features.sentiment.engine import SENTIMENT_ENGINE_VERSION
SEARCH_INDEXES = (
{"id": "000001.SH", "code": "000001.SH", "name": "上证指数", "type": "index", "subtitle": "沪市综合指数"},
{"id": "399001.SZ", "code": "399001.SZ", "name": "深证成指", "type": "index", "subtitle": "深市成份指数"},
{"id": "399006.SZ", "code": "399006.SZ", "name": "创业板指", "type": "index", "subtitle": "创业板核心指数"},
)
SEARCH_TYPE_LABELS = {
"stock": "股票",
"sector": "板块",
"theme": "题材",
"index": "指数",
}
THS_SEARCH_TYPES = {
"I": ("sector", "行业板块"),
"R": ("sector", "地域板块"),
"N": ("theme", "概念题材"),
}
class MarketServiceMixin:
def _tushare_client(self) -> TushareClient:
gateway = getattr(self, "data_gateway", None)
if gateway is not None:
return gateway.tushare()
# Compatibility for isolated legacy unit-test service stubs.
return TushareClient(self.token)
def get_dashboard(self, trade_date: str, force: bool = False) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
now = datetime.now().astimezone()
if (
normalized_date == now.strftime("%Y%m%d")
and now.time().replace(tzinfo=None) < datetime.strptime("09:15", "%H:%M").time()
):
previous = self.database.get_latest_real_snapshot(normalized_date, strictly_before=True)
if previous:
carried = self._carry_dashboard(previous, normalized_date, "盘前沿用最近交易日收盘行情")
return self._apply_reason_overrides(self._with_storage(carried, cached=True))
if not force:
snapshot = self.database.get_snapshot(normalized_date)
if snapshot and str((snapshot.get("meta") or {}).get("source") or "") != "demo":
snapshot = copy.deepcopy(snapshot)
if normalized_date != now.strftime("%Y%m%d"):
snapshot.setdefault("meta", {}).update(
{"realtime": False, "market_status": "closed"}
)
if not self._dashboard_sentiment_ready(snapshot):
snapshot = self._enrich_dashboard_sentiment(snapshot, normalized_date)
self.database.save_snapshot(
normalized_date,
str((snapshot.get("meta") or {}).get("source") or "tushare"),
snapshot,
)
snapshot.setdefault("meta", {})["requested_date"] = self._display_compact_date(normalized_date)
return self._apply_reason_overrides(self._with_storage(snapshot, cached=True))
resolved = self.database.get_data_snapshot(
"dashboard_request_v1", normalized_date
)
if resolved and str((resolved.get("meta") or {}).get("source") or "") != "demo":
resolved = copy.deepcopy(resolved)
resolved.setdefault("meta", {})["requested_date"] = self._display_compact_date(
normalized_date
)
return self._apply_reason_overrides(
self._with_storage(resolved, cached=True)
)
if datetime.strptime(normalized_date, "%Y%m%d").weekday() >= 5:
previous = self.database.get_latest_real_snapshot(normalized_date)
if previous:
carried = self._carry_dashboard(
previous,
normalized_date,
"非交易日沿用最近交易日收盘行情",
)
self.database.save_data_snapshot(
"dashboard_request_v1", normalized_date, "sqlite", carried
)
return self._apply_reason_overrides(
self._with_storage(carried, cached=True)
)
return self.sync_dashboard(normalized_date)
@staticmethod
def _dashboard_sentiment_ready(dashboard: dict[str, Any]) -> bool:
overview = dashboard.get("overview") or {}
return int(overview.get("sentiment_engine_version") or 0) == SENTIMENT_ENGINE_VERSION and all(
key in overview
for key in (
"sentiment_score",
"sentiment_label",
"sentiment_phase",
"sentiment_direction",
"sentiment_components",
)
)
@staticmethod
def _display_compact_date(compact: str) -> str:
return f"{compact[:4]}-{compact[4:6]}-{compact[6:8]}"
def _carry_dashboard(
self, snapshot: dict[str, Any], requested_date: str, reason: str
) -> dict[str, Any]:
carried = copy.deepcopy(snapshot)
meta = carried.setdefault("meta", {})
meta.update(
{
"requested_date": self._display_compact_date(requested_date),
"carried_forward": True,
"realtime": False,
"market_status": "closed",
"notice": reason,
}
)
return carried
def _realtime_snapshot_due(
self,
normalized_date: str,
snapshot: dict[str, Any],
) -> bool:
if not self.configured or normalized_date != date.today().strftime("%Y%m%d"):
return False
now = datetime.now().astimezone()
local_time = now.time().replace(tzinfo=None)
realtime_start = datetime.strptime("09:15", "%H:%M").time()
morning_end = datetime.strptime("11:35", "%H:%M").time()
afternoon_start = datetime.strptime("12:55", "%H:%M").time()
realtime_end = datetime.strptime("15:05", "%H:%M").time()
in_session = (
realtime_start <= local_time < morning_end
or afternoon_start <= local_time < realtime_end
)
if not in_session:
return False
meta = snapshot.get("meta") or {}
snapshot_trade_date = str(meta.get("trade_date") or "").replace("-", "")
if snapshot_trade_date and snapshot_trade_date != normalized_date:
return False
if not meta.get("realtime"):
return True
try:
updated_at = datetime.fromisoformat(str(meta.get("updated_at") or ""))
if updated_at.tzinfo is None:
updated_at = updated_at.replace(tzinfo=now.tzinfo)
except ValueError:
return True
age_seconds = (now - updated_at.astimezone(now.tzinfo)).total_seconds()
return age_seconds >= 8
def sync_dashboard(self, trade_date: str) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
source = "tushare"
with self.sync_lock:
sync_id = self.database.start_sync(normalized_date, source)
try:
if not self.configured:
raise TushareError("公共行情尚未配置")
dashboard = self._tushare_client().dashboard(normalized_date)
dashboard["meta"]["source"] = source
dashboard["meta"]["requested_date"] = self._display_compact_date(normalized_date)
dashboard = self._enrich_dashboard_sentiment(dashboard, normalized_date)
record_count = self._record_count(dashboard)
actual_date = normalize_date(
str(dashboard.get("meta", {}).get("trade_date") or normalized_date)
)
self.database.save_snapshot(actual_date, source, dashboard)
if actual_date != normalized_date:
dashboard.setdefault("meta", {}).update(
{
"carried_forward": True,
"realtime": False,
"market_status": "closed",
}
)
self.database.save_data_snapshot(
"dashboard_request_v1", normalized_date, source, dashboard
)
self.database.finish_sync(
sync_id,
"success",
record_count,
dashboard.get("meta", {}).get("notice", ""),
source,
)
return self._apply_reason_overrides(self._with_storage(dashboard, cached=False))
except TushareError as exc:
fallback = self.database.get_latest_real_snapshot(normalized_date)
if fallback:
carried = self._carry_dashboard(
fallback, normalized_date, f"最新行情暂不可用,沿用最近收盘快照:{exc}"
)
self.database.finish_sync(
sync_id, "fallback", self._record_count(carried), str(exc), "tushare"
)
return self._apply_reason_overrides(self._with_storage(carried, cached=True))
self.database.finish_sync(sync_id, "failed", message=str(exc))
raise ValueError("暂无可用的真实行情快照,请等待后台完成首次同步。") from exc
except Exception as exc:
self.database.finish_sync(sync_id, "failed", message=str(exc))
raise
def realtime_aggregate_health(self, sector: str = "") -> dict[str, Any]:
sector = validate_text(sector, "板块名称", 50)
return self.realtime_aggregator.health_snapshot(sector)
def _search_market_directory(self) -> list[dict[str, Any]]:
cached = self.database.get_data_snapshot("search_directory", "ths") or {}
cached_items = list(cached.get("items") or [])
if cached_items and int(cached.get("schema_version") or 0) >= 2:
return cached_items
if not self.configured:
return cached_items
try:
rows = self._tushare_client().query(
"ths_index",
{},
"ts_code,name,count,exchange,list_date,type",
)
except TushareError:
return cached_items
items = []
for row in rows:
mapping = THS_SEARCH_TYPES.get(str(row.get("type") or "").upper())
code = str(row.get("ts_code") or "").strip().upper()
name = str(row.get("name") or "").strip()
if not mapping or not code or not name or str(row.get("exchange") or "").upper() != "A":
continue
entity_type, subtitle = mapping
items.append(
{
"id": code,
"code": code,
"name": name,
"type": entity_type,
"subtitle": subtitle,
"member_count": int(float(row.get("count") or 0)),
}
)
if items:
self.database.save_data_snapshot(
"search_directory", "ths", "tushare", {"schema_version": 2, "items": items}
)
return items
@staticmethod
def _search_match_score(item: dict[str, Any], query: str) -> tuple[int, int, str]:
name = str(item.get("name") or "").casefold()
code = str(item.get("code") or item.get("id") or "").casefold()
needle = query.casefold()
if code == needle:
rank = 0
elif name == needle:
rank = 1
elif code.startswith(needle):
rank = 2
elif name.startswith(needle):
rank = 3
else:
rank = 4
return rank, len(name), code
def search_entities(self, query: str, trade_date: str) -> dict[str, Any]:
needle = str(query or "").strip()
normalized_date = normalize_date(trade_date)
groups: dict[str, list[dict[str, Any]]] = {
"stocks": [],
"sectors": [],
"themes": [],
"indices": [],
}
if not needle:
return {"query": "", "trade_date": normalized_date, "groups": groups}
stocks = []
for row in self.database.search_stock_master(needle, 12):
stocks.append(
{
"id": str(row.get("code") or ""),
"code": str(row.get("code") or ""),
"name": str(row.get("name") or "--"),
"type": "stock",
"type_label": SEARCH_TYPE_LABELS["stock"],
"industry": str(row.get("industry") or "其他"),
"market": str(row.get("market") or ""),
"subtitle": " · ".join(
part for part in (str(row.get("industry") or ""), str(row.get("market") or "")) if part
) or "A股",
}
)
groups["stocks"] = stocks[:8]
market_items = list(self._search_market_directory()) + [dict(item) for item in SEARCH_INDEXES]
matched = [
item for item in market_items
if needle.casefold() in str(item.get("name") or "").casefold()
or needle.casefold() in str(item.get("code") or "").casefold()
]
matched.sort(key=lambda item: self._search_match_score(item, needle))
group_keys = {"sector": "sectors", "theme": "themes", "index": "indices"}
for item in matched:
group_key = group_keys.get(str(item.get("type") or ""))
if not group_key or len(groups[group_key]) >= 8:
continue
groups[group_key].append(
{
**item,
"type_label": SEARCH_TYPE_LABELS[str(item["type"])],
}
)
return {"query": needle, "trade_date": normalized_date, "groups": groups}
def get_search_detail(
self, entity_type: str, identifier: str, trade_date: str
) -> dict[str, Any]:
entity_type = str(entity_type or "").strip().lower()
identifier = str(identifier or "").strip().upper()
normalized_date = normalize_date(trade_date)
if entity_type not in {"sector", "theme", "index"}:
raise ValueError("搜索详情类型不支持。")
if not re.fullmatch(r"[A-Z0-9.]{3,24}", identifier):
raise ValueError("搜索详情标识无效。")
if not self.configured:
raise ValueError("行情数据源尚未配置。")
if entity_type == "index":
index_basic = next((item for item in SEARCH_INDEXES if item["id"] == identifier), None)
if not index_basic:
raise ValueError("暂不支持该指数详情。")
return self._index_search_detail(index_basic, normalized_date)
directory = self._search_market_directory()
basic = next(
(
item for item in directory
if item.get("id") == identifier and item.get("type") == entity_type
),
None,
)
if not basic:
raise ValueError("未找到对应的板块或题材。")
return self._ths_search_detail(basic, normalized_date)
def get_intraday_chart(
self, entity_type: str, identifier: str
) -> dict[str, Any]:
entity_type = str(entity_type or "").strip().lower()
identifier = str(identifier or "").strip().upper()
if entity_type == "stock":
code = validate_stock_code(identifier)
chart = self.chart_data.stock_intraday(code)
type_label = SEARCH_TYPE_LABELS["stock"]
elif entity_type == "index":
basic = next((item for item in SEARCH_INDEXES if item["id"] == identifier), None)
if not basic:
raise ValueError("暂不支持该指数分时行情。")
chart = self.chart_data.index_intraday(identifier)
type_label = SEARCH_TYPE_LABELS["index"]
elif entity_type in {"sector", "theme"}:
basic = next(
(
item for item in self._search_market_directory()
if item.get("id") == identifier and item.get("type") == entity_type
),
None,
)
if not basic:
raise ValueError("未找到对应的板块或题材。")
chart = self.chart_data.board_intraday(identifier, str(basic.get("name") or ""))
type_label = SEARCH_TYPE_LABELS[entity_type]
else:
raise ValueError("分时行情类型不支持。")
return {
"meta": {
"trade_date": str(chart.get("trade_date") or ""),
"previous_close": float(chart.get("previous_close") or 0),
},
"entity": {
"id": identifier,
"code": str(chart.get("code") or identifier),
"name": str(chart.get("name") or ""),
"type": entity_type,
"type_label": type_label,
},
"points": list(chart.get("points") or []),
}
def _ths_search_detail(
self, basic: dict[str, Any], trade_date: str
) -> dict[str, Any]:
client = self._tushare_client()
resolved_date, _ = client.resolve_trade_context(trade_date)
end = datetime.strptime(resolved_date, "%Y%m%d")
start_date = (end - timedelta(days=190)).strftime("%Y%m%d")
identifier = str(basic["id"])
snapshot = client.sector_snapshot(identifier, resolved_date)
rows = client.query(
"ths_daily",
{"ts_code": identifier, "start_date": start_date, "end_date": resolved_date},
"ts_code,trade_date,open,high,low,close,pct_change,vol,turnover_rate,total_mv,float_mv",
)
rows.sort(key=lambda item: str(item.get("trade_date") or ""))
series = [
{
"trade_date": self._display_compact_date(str(row.get("trade_date") or "")),
"open": float(row.get("open") or 0),
"high": float(row.get("high") or 0),
"low": float(row.get("low") or 0),
"close": float(row.get("close") or 0),
"change": float(row.get("pct_change") or 0),
"volume": float(row.get("vol") or 0),
"turnover_rate": float(row.get("turnover_rate") or 0),
}
for row in rows[-90:]
]
try:
chart_series = self.chart_data.board_daily(identifier, resolved_date, 90)
if chart_series:
series = chart_series
except (AttributeError, ChartDataError):
pass
latest = series[-1] if series else {}
snapshot_is_current = str(snapshot.get("trade_date") or "").replace("-", "") == resolved_date
change = float(
snapshot.get("change")
if snapshot_is_current and snapshot.get("change") is not None
else latest.get("change") or 0
)
if latest.get("realtime"):
change = float(latest.get("change") or 0)
turnover_rate = float(
snapshot.get("turnover_rate")
if snapshot_is_current and snapshot.get("turnover_rate") is not None
else latest.get("turnover_rate") or 0
)
metrics = [
{"label": "涨跌幅", "value": round(change, 2), "unit": "%", "tone": "change"},
{"label": "换手率", "value": round(turnover_rate, 2), "unit": "%"},
{"label": "成份数量", "value": int(float(basic.get("member_count") or 0)), "unit": ""},
]
up_count = int(float(snapshot.get("up_count") or 0))
down_count = int(float(snapshot.get("down_count") or 0))
if up_count or down_count:
metrics.extend(
[
{"label": "上涨家数", "value": up_count, "unit": ""},
{"label": "下跌家数", "value": down_count, "unit": ""},
]
)
leader = str(snapshot.get("leader") or "").strip()
if leader and leader != "--":
metrics.extend(
[
{"label": "领涨标的", "value": leader, "unit": ""},
{"label": "领涨幅", "value": round(float(snapshot.get("leading_pct") or 0), 2), "unit": "%", "tone": "change"},
]
)
return {
"meta": {
"trade_date": self._display_compact_date(resolved_date),
"realtime": bool(snapshot.get("realtime")),
},
"entity": {
"id": identifier,
"code": identifier,
"name": str(snapshot.get("name") or basic.get("name") or "--"),
"type": str(basic.get("type") or "sector"),
"type_label": SEARCH_TYPE_LABELS[str(basic.get("type") or "sector")],
"subtitle": str(basic.get("subtitle") or ""),
"value": float(latest.get("close") or 0),
"change": change,
},
"series": series,
"metrics": metrics,
}
def _index_search_detail(
self, basic: dict[str, Any], trade_date: str
) -> dict[str, Any]:
client = self._tushare_client()
resolved_date, _ = client.resolve_trade_context(trade_date)
payload = (
client.realtime_market_indices(resolved_date)
if client.should_use_realtime(trade_date, resolved_date)
else client.market_indices(resolved_date, 90)
)
current = next(
(item for item in payload.get("indices") or [] if item.get("ts_code") == basic["id"]),
None,
)
if not current:
raise ValueError("该指数暂无可用行情。")
end = datetime.strptime(resolved_date, "%Y%m%d")
rows = client.query(
"index_daily",
{
"ts_code": basic["id"],
"start_date": (end - timedelta(days=190)).strftime("%Y%m%d"),
"end_date": resolved_date,
},
"ts_code,trade_date,open,high,low,close,pct_chg,vol,amount",
)
rows.sort(key=lambda item: str(item.get("trade_date") or ""))
series = [
{
"trade_date": self._display_compact_date(str(row.get("trade_date") or "")),
"open": float(row.get("open") or 0),
"high": float(row.get("high") or 0),
"low": float(row.get("low") or 0),
"close": float(row.get("close") or 0),
"change": float(row.get("pct_chg") or 0),
"volume": float(row.get("vol") or 0),
}
for row in rows[-90:]
]
try:
chart_series = self.chart_data.index_daily(str(basic["id"]), resolved_date, 90)
if chart_series:
series = chart_series
except (AttributeError, ChartDataError):
pass
latest = series[-1] if series else {}
latest_close = float(latest.get("close") or current.get("close") or 0)
latest_change = float(latest.get("change") or current.get("pct_chg") or 0)
def series_return(days: int) -> float:
if len(series) <= days:
return 0.0
previous = float(series[-days - 1].get("close") or 0)
return (latest_close / previous - 1) * 100 if previous > 0 else 0.0
return {
"meta": {
"trade_date": self._display_compact_date(str(current.get("trade_date") or resolved_date)),
"realtime": bool(payload.get("realtime")),
},
"entity": {
**basic,
"type_label": SEARCH_TYPE_LABELS["index"],
"value": latest_close,
"change": latest_change,
},
"series": series,
"metrics": [
{"label": "涨跌幅", "value": round(latest_change, 2), "unit": "%", "tone": "change"},
{"label": "近5日", "value": round(series_return(5), 2), "unit": "%", "tone": "change"},
{"label": "近20日", "value": round(series_return(20), 2), "unit": "%", "tone": "change"},
{"label": "成交额", "value": round(float(current.get("amount_billion") or 0), 2), "unit": "亿"},
],
}
def get_stock_detail(
self, code: str, trade_date: str, force: bool = False
) -> dict[str, Any]:
code = validate_stock_code(code)
normalized_date = normalize_date(trade_date)
cache_key = f"{code}:{normalized_date}"
if not force:
cached = self.database.get_data_snapshot("stock_detail", cache_key)
if cached and str((cached.get("meta") or {}).get("source") or "") != "demo":
if not self._stock_detail_cache_needs_refresh(cached, normalized_date):
cached["meta"] = {**cached.get("meta", {}), "cached": True}
return self._prepare_stock_detail(cached, code, normalized_date)
name, sector = self._stock_identity(code, normalized_date)
source = "tushare"
if self.configured:
try:
payload = self._tushare_client().stock_detail(
tushare_code(code), normalized_date
)
if not payload.get("prices"):
raise TushareError("No price history returned")
except TushareError as exc:
payload = self.database.get_latest_data_snapshot(
"stock_detail", f"{code}:", cache_key, exclude_source="demo"
)
if not payload:
raise ValueError(f"暂无 {code} 的真实行情数据:{exc}") from exc
payload = copy.deepcopy(payload)
payload["meta"] = {
**payload.get("meta", {}),
"cached": True,
"notice": "最新行情暂不可用,已沿用最近真实收盘数据。",
}
return self._prepare_stock_detail(payload, code, normalized_date)
else:
payload = self.database.get_latest_data_snapshot(
"stock_detail", f"{code}:", cache_key, exclude_source="demo"
)
if not payload:
raise ValueError(f"暂无 {code} 的真实行情数据,请等待后台完成首次同步。")
payload = copy.deepcopy(payload)
payload["meta"] = {
**payload.get("meta", {}),
"cached": True,
"notice": "公共行情尚未配置,已沿用最近真实收盘数据。",
}
return self._prepare_stock_detail(payload, code, normalized_date)
payload["meta"]["source"] = source
payload["meta"]["cached"] = False
self.database.save_data_snapshot("stock_detail", cache_key, source, payload)
return self._prepare_stock_detail(payload, code, normalized_date)
@staticmethod
def _stock_detail_bar_date(payload: dict[str, Any]) -> str:
prices = list(payload.get("prices") or [])
return str((prices[-1] if prices else {}).get("trade_date") or "").replace("-", "")
def _stock_detail_cache_needs_refresh(
self, payload: dict[str, Any], requested_date: str
) -> bool:
now = datetime.now().astimezone()
return (
requested_date == now.strftime("%Y%m%d")
and now.time().replace(tzinfo=None) >= dt_time(15, 0)
and self._stock_detail_bar_date(payload) < requested_date
)
def _prepare_stock_detail(
self, payload: dict[str, Any], code: str, requested_date: str
) -> dict[str, Any]:
result = copy.deepcopy(payload)
now = datetime.now().astimezone()
try:
result["prices"] = self.chart_data.stock_daily(code, requested_date, 90)
result["meta"] = {**(result.get("meta") or {}), "chart_source": "market_chart"}
except (AttributeError, ChartDataError):
pass
result = self._sanitize_stock_detail_prices(result, now)
actual_date = self._stock_detail_bar_date(result)
if actual_date:
result["meta"] = {
**(result.get("meta") or {}),
"trade_date": f"{actual_date[:4]}-{actual_date[4:6]}-{actual_date[6:]}",
}
today = now.strftime("%Y%m%d")
should_merge = (
requested_date == today
and actual_date <= today
and now.weekday() < 5
and now.time().replace(tzinfo=None) >= dt_time(9, 30)
)
if should_merge:
quote = self._ifind_realtime_stock_quote(code)
if quote and self._valid_realtime_stock_quote(quote, today):
self._merge_realtime_stock_detail(result, quote, requested_date)
elif self.configured and actual_date < today:
client = self._tushare_client()
try:
resolved_date, _ = client.resolve_trade_context(requested_date)
if resolved_date == today:
quote = client.realtime_stock_quote(tushare_code(code), requested_date)
if self._valid_realtime_stock_quote(quote, today):
self._merge_realtime_stock_detail(result, quote, requested_date)
except TushareError:
pass
return self._enrich_stock_detail(result)
@staticmethod
def _sanitize_stock_detail_prices(
payload: dict[str, Any], market_now: datetime
) -> dict[str, Any]:
result = copy.deepcopy(payload)
raw_prices = list(result.get("prices") or [])
raw_latest_date = str(
(raw_prices[-1] if raw_prices else {}).get("trade_date") or ""
).replace("-", "")
prices = []
for bar in raw_prices:
open_price = float(bar.get("open") or 0)
high = float(bar.get("high") or 0)
low = float(bar.get("low") or 0)
close = float(bar.get("close") or 0)
if (
open_price > 0
and high >= max(open_price, close)
and 0 < low <= min(open_price, close)
and close > 0
):
prices.append(bar)
today = market_now.strftime("%Y%m%d")
market_open = (
market_now.weekday() < 5
and market_now.time().replace(tzinfo=None) >= dt_time(9, 30)
)
if prices and str(prices[-1].get("trade_date") or "").replace("-", "") == today:
current = prices[-1]
has_market_activity = (
float(current.get("volume") or 0) > 0
or float(current.get("amount_billion") or 0) > 0
)
if not market_open or not has_market_activity:
prices.pop()
if raw_latest_date == today and (
not prices
or str(prices[-1].get("trade_date") or "").replace("-", "") != today
):
result["meta"] = {**(result.get("meta") or {}), "realtime": False}
result["prices"] = prices
if prices:
latest = prices[-1]
stock = dict(result.get("stock") or {})
stock.update(
{
"price": float(latest.get("close") or 0),
"change": float(latest.get("change") or 0),
"amount_billion": float(latest.get("amount_billion") or 0),
}
)
result["stock"] = stock
return result
@staticmethod
def _valid_realtime_stock_quote(quote: dict[str, Any], trade_date: str) -> bool:
price = float(quote.get("price") or 0)
open_price = float(quote.get("open") or 0)
high = float(quote.get("high") or 0)
low = float(quote.get("low") or 0)
volume = float(quote.get("volume") or 0)
amount = float(quote.get("amount_billion") or 0)
quote_date = str(quote.get("quote_time") or "")[:10].replace("-", "")
return (
price > 0
and open_price > 0
and high >= max(open_price, price)
and 0 < low <= min(open_price, price)
and (volume > 0 or amount > 0)
and (not quote_date or quote_date == trade_date)
)
def _ifind_realtime_stock_quote(self, code: str) -> dict[str, Any] | None:
ifind = getattr(self, "ifind", None)
if not ifind or not ifind.configured:
return None
try:
rows = ifind.real_time(
tushare_code(code),
[
"open", "high", "low", "latest", "preClose",
"volume", "amount", "turnoverRatio",
],
cache_ttl=10,
)
except IfindError:
return None
row = rows[0] if rows else {}
price = float(row.get("latest") or 0)
previous_close = float(row.get("preClose") or 0)
if price <= 0:
return None
change = (price / previous_close - 1) * 100 if previous_close > 0 else 0.0
stock = self._stock_identity(code, date.today().strftime("%Y%m%d"))
return {
"name": stock[0],
"sector": stock[1],
"price": price,
"open": float(row.get("open") or price),
"high": float(row.get("high") or price),
"low": float(row.get("low") or price),
"change": round(change, 4),
"volume": float(row.get("volume") or 0),
"volume_unit": "lots",
"amount_billion": float(row.get("amount") or 0) / 100_000_000,
"turnover_rate": float(row.get("turnoverRatio") or 0),
"quote_time": str(row.get("time") or ""),
}
@staticmethod
def _merge_realtime_stock_detail(
payload: dict[str, Any], quote: dict[str, Any], trade_date: str
) -> None:
display_date = f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:]}"
realtime_bar = {
"trade_date": display_date,
"open": quote["open"],
"high": quote["high"],
"low": quote["low"],
"close": quote["price"],
"change": quote["change"],
"volume": quote["volume"] if quote.get("volume_unit") == "lots" else quote["volume"] / 100,
"amount_billion": quote["amount_billion"],
"realtime": True,
}
prices = list(payload.get("prices") or [])
if prices and str(prices[-1].get("trade_date") or "").replace("-", "") == trade_date:
prices[-1] = realtime_bar
else:
prices.append(realtime_bar)
payload["prices"] = prices[-90:]
stock = dict(payload.get("stock") or {})
stock.update(
{
"name": quote["name"],
"industry": quote["sector"],
"price": quote["price"],
"change": quote["change"],
"amount_billion": quote["amount_billion"],
"turnover_rate": quote["turnover_rate"],
}
)
payload["stock"] = stock
payload["meta"] = {
**(payload.get("meta") or {}),
"trade_date": display_date,
"realtime": True,
"updated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
}
def get_stock_preview(
self, code: str, trade_date: str, force: bool = False
) -> dict[str, Any]:
code = validate_stock_code(code)
# Hover previews deliberately follow the latest market day, independent
# from the review date selected by the page.
detail = self.get_stock_detail(code, date.today().strftime("%Y%m%d"), force)
detail_meta = detail.get("meta") or {}
resolved_date = str(detail_meta.get("trade_date") or trade_date)
intraday_points: list[dict[str, Any]] = []
intraday_status = "unavailable"
intraday_notice = "分时行情暂不可用。"
intraday_trade_date = ""
intraday_previous_close = 0.0
try:
intraday = self.chart_data.stock_intraday(code)
intraday_points = list(intraday.get("points") or [])
intraday_trade_date = str(intraday.get("trade_date") or "")
intraday_previous_close = float(intraday.get("previous_close") or 0)
if intraday_points:
intraday_status = "available"
intraday_notice = ""
else:
intraday_status = "empty"
intraday_notice = "最近交易日暂无分时数据。"
except ChartDataError:
intraday_status = "unavailable"
intraday_notice = "分时行情暂不可用,请稍后重试。"
prices = list(detail.get("prices") or [])[-60:]
stock = dict(detail.get("stock") or {"code": code})
realtime = bool(detail_meta.get("realtime"))
return {
"meta": {
"trade_date": resolved_date,
"source": detail_meta.get("source") or "unavailable",
"notice": detail_meta.get("notice") or "",
"intraday_status": intraday_status,
"intraday_notice": intraday_notice,
"intraday_trade_date": intraday_trade_date,
"intraday_previous_close": intraday_previous_close,
"realtime": realtime,
"refresh_interval_seconds": 10 if realtime else 0,
},
"stock": stock,
"prices": prices,
"intraday": intraday_points,
}
def backfill(self, start_date: str, end_date: str) -> list[dict[str, Any]]:
start = datetime.strptime(normalize_date(start_date), "%Y%m%d").date()
end = datetime.strptime(normalize_date(end_date), "%Y%m%d").date()
if start > end:
raise ValueError("开始日期不能晚于结束日期。")
weekdays = []
current = start
while current <= end:
if current.weekday() < 5:
weekdays.append(current)
current += timedelta(days=1)
if len(weekdays) > 15:
raise ValueError("单次最多回补 15 个工作日。")
results = []
for day in weekdays:
dashboard = self.sync_dashboard(day.strftime("%Y%m%d"))
results.append(
{
"requested_date": day.isoformat(),
"trade_date": dashboard["meta"]["trade_date"],
"source": dashboard["meta"]["source"],
"records": self._record_count(dashboard),
}
)
return results
def _stock_identity(self, code: str, trade_date: str) -> tuple[str, str]:
snapshot = self.database.get_snapshot(trade_date) or {}
for key in ("limits", "broken", "down_limits"):
for row in snapshot.get(key) or []:
if str(row.get("code")) == code:
return row.get("name") or "--", row.get("sector") or "其他"
for item in self.database.list_watchlist(self.current_user_id):
if item["code"] == code:
return item["name"], item["sector"] or "其他"
return "--", "其他"
def _enrich_stock_detail(self, payload: dict[str, Any]) -> dict[str, Any]:
result = dict(payload)
stock = dict(payload.get("stock") or {})
code = str(stock.get("code") or "")
watched = {
item["code"]: item
for item in self.database.list_watchlist(self.current_user_id)
}
stock["watchlist"] = watched.get(code)
result["stock"] = stock
result["notes"] = self.database.list_notes(self.current_user_id, code=code)
return result
def _with_storage(self, dashboard: dict[str, Any], cached: bool) -> dict[str, Any]:
result = dict(dashboard)
result["meta"] = {
**dashboard.get("meta", {}),
"storage": "sqlite",
"cached": cached,
}
return result
@staticmethod
def _record_count(dashboard: dict[str, Any]) -> int:
return sum(
len(dashboard.get(key) or [])
for key in ("limits", "broken", "down_limits", "yesterday_limits")
)
+6
View File
@@ -0,0 +1,6 @@
"""Limit-up, broken-board, limit-down and prior-limit pool feature."""
from .repository import PoolRepositoryMixin
from .service import PoolServiceMixin
__all__ = ["PoolRepositoryMixin", "PoolServiceMixin"]
+27
View File
@@ -0,0 +1,27 @@
from __future__ import annotations
from datetime import datetime
class PoolRepositoryMixin:
def save_reason_override(self, trade_date: str, code: str, reason: str) -> None:
now = datetime.now().astimezone().isoformat(timespec="seconds")
with self.connect() as connection:
connection.execute(
"""
INSERT INTO reason_overrides (trade_date, code, reason, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(trade_date, code) DO UPDATE SET
reason = excluded.reason,
updated_at = excluded.updated_at
""",
(trade_date, code, reason, now),
)
def reason_overrides(self, trade_date: str) -> dict[str, str]:
with self.connect() as connection:
rows = connection.execute(
"SELECT code, reason FROM reason_overrides WHERE trade_date = ?",
(trade_date,),
).fetchall()
return {row["code"]: row["reason"] for row in rows}
+150
View File
@@ -0,0 +1,150 @@
from __future__ import annotations
import re
from datetime import datetime, time as dt_time
from typing import Any
from backend.bootstrap.config import normalize_date, validate_stock_code
from backend.data.providers.ifind_client import IfindError
class PoolServiceMixin:
def save_reason(self, trade_date: str, code: str, reason: str) -> None:
normalized_date = normalize_date(trade_date)
code = validate_stock_code(code)
reason = reason.strip()
if not reason or len(reason) > 200:
raise ValueError("涨停原因应为 1 至 200 个字符。")
self.database.save_reason_override(normalized_date, code, reason)
def _apply_reason_overrides(self, dashboard: dict[str, Any]) -> dict[str, Any]:
trade_date = str(dashboard.get("meta", {}).get("trade_date", "")).replace("-", "")
enrichment = self.database.get_data_snapshot("ifind_event_enrichment_v1", trade_date)
if enrichment:
self._merge_ifind_event_enrichment(dashboard, enrichment)
else:
self._schedule_ifind_event_enrichment(trade_date)
overrides = self.database.reason_overrides(trade_date)
if not overrides:
return dashboard
for key in ("limits", "broken", "down_limits"):
for row in dashboard.get(key) or []:
if row.get("code") in overrides:
row["reason"] = overrides[row["code"]]
row["reason_source"] = "manual"
return dashboard
def _schedule_ifind_event_enrichment(self, trade_date: str) -> None:
ifind = getattr(self, "ifind", None)
if not ifind or not ifind.configured or not re.fullmatch(r"\d{8}", trade_date):
return
now = datetime.now().astimezone()
if trade_date == now.strftime("%Y%m%d") and now.time().replace(tzinfo=None) < dt_time(15, 0):
return
self.jobs.submit(
"market.ifind-event-enrichment",
f"{trade_date}:v1",
lambda: self._refresh_ifind_event_enrichment(trade_date),
{"trade_date": trade_date, "trigger": "dashboard-enrichment"},
)
def _refresh_ifind_event_enrichment(self, trade_date: str) -> None:
if not self._ifind_event_lock.acquire(blocking=False):
return
try:
if self.database.get_data_snapshot("ifind_event_enrichment_v1", trade_date):
return
ifind = getattr(self, "ifind", None)
if not ifind or not ifind.configured:
return
current = datetime.strptime(trade_date, "%Y%m%d")
display_date = f"{current.year}{current.month}{current.day}"
requests = {
"limits": (
f"{display_date}涨停股票,股票代码、股票简称、涨停原因、"
"首次涨停时间、最终涨停时间、开板次数"
),
"broken": (
f"{display_date}曾涨停但收盘未涨停的股票,股票代码、股票简称、"
"涨停原因、首次涨停时间、开板次数"
),
"down_limits": (
f"{display_date}跌停股票,股票代码、股票简称、跌停原因"
),
}
result: dict[str, Any] = {
"trade_date": trade_date,
"generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"limits": {}, "broken": {}, "down_limits": {}, "partial": False,
}
for kind, query in requests.items():
try:
rows = ifind.wencai(query, "stock", cache_ttl=900)
except IfindError:
result["partial"] = True
continue
for raw in rows:
code = self._ifind_row_code(raw)
if not code:
continue
reason_tokens = (
("跌停原因", "风险线索", "原因")
if kind == "down_limits"
else ("涨停原因类别", "涨停原因", "触板逻辑", "原因")
)
reason = str(self._ifind_field(raw, reason_tokens) or "").strip()
first_time = self._normalize_ifind_event_time(
self._ifind_field(raw, ("首次涨停时间", "首次触板时间", "首次封板时间"))
)
last_time = self._normalize_ifind_event_time(
self._ifind_field(raw, ("最终涨停时间", "最后涨停时间", "最后封板时间"))
)
open_times = self._ifind_field(raw, ("开板次数", "打开涨停次数"))
try:
open_count = max(0, int(float(open_times))) if open_times not in (None, "") else None
except (TypeError, ValueError):
open_count = None
result[kind][code] = {
"reason": reason,
"first_time": first_time,
"last_time": last_time,
"open_times": open_count,
}
if any(result[kind] for kind in ("limits", "broken", "down_limits")):
self.database.save_data_snapshot(
"ifind_event_enrichment_v1", trade_date, "ifind", result
)
finally:
self._ifind_event_lock.release()
@staticmethod
def _normalize_ifind_event_time(value: Any) -> str:
text = str(value or "").strip()
match = re.search(r"(?:^|\s)(\d{1,2}:\d{2}(?::\d{2})?)(?:$|\s)", text)
if not match:
match = re.search(r"(?<!\d)(\d{6})(?!\d)", text)
if match:
compact = match.group(1)
return f"{compact[:2]}:{compact[2:4]}:{compact[4:]}"
return ""
parts = match.group(1).split(":")
return ":".join(part.zfill(2) for part in parts)
@staticmethod
def _merge_ifind_event_enrichment(
dashboard: dict[str, Any], enrichment: dict[str, Any]
) -> None:
for kind in ("limits", "broken", "down_limits"):
records = enrichment.get(kind) or {}
for row in dashboard.get(kind) or []:
event = records.get(str(row.get("code") or "")) or {}
reason = str(event.get("reason") or "").strip()
if reason:
row["reason"] = reason
row["reason_source"] = "market_event"
if event.get("first_time"):
row["first_time"] = event["first_time"]
if event.get("last_time"):
row["last_time"] = event["last_time"]
if event.get("open_times") is not None:
row["open_times"] = event["open_times"]
@@ -0,0 +1,5 @@
"""Sector rotation history and constituent detail feature."""
from .service import RotationServiceMixin
__all__ = ["RotationServiceMixin"]
+165
View File
@@ -0,0 +1,165 @@
from __future__ import annotations
from typing import Any
from backend.bootstrap.config import normalize_date, validate_text
from backend.data.providers.tushare_client import TushareError
from backend.features.sentiment.engine import (
build_sentiment_history,
latest_contiguous_history,
)
class RotationServiceMixin:
def rotation_history(self, trade_date: str, limit: int = 9) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
# 板块轮动固定展示最近 9 个交易日,按由近到远排列。
limit = 9
snapshots = self.database.list_snapshot_payloads(normalized_date, 240)
by_trade_date: dict[str, dict[str, Any]] = {}
for snapshot in snapshots:
meta = snapshot.get("meta") or {}
actual_date = str(meta.get("trade_date") or snapshot.get("_snapshot_date") or "")
compact_date = actual_date.replace("-", "")
if len(compact_date) == 8:
by_trade_date[compact_date] = snapshot
sentiment_dates = {
str(row.get("trade_date") or "").replace("-", "")
for row in latest_contiguous_history(build_sentiment_history(snapshots))
}
ordered_dates = sorted(
date_key for date_key in by_trade_date
if not sentiment_dates or date_key in sentiment_dates
)[-limit:][::-1]
rows = []
for date_key in ordered_dates:
snapshot = by_trade_date[date_key]
sector_context = {
str(item.get("name") or ""): item
for item in snapshot.get("sectors") or []
}
sectors = []
for item in (snapshot.get("sector_rotation") or [])[:12]:
name = str(item.get("name") or "").strip()
context = sector_context.get(name, {})
sectors.append(
{
"name": name,
"rank": int(item.get("rank") or len(sectors) + 1),
"trend": item.get("trend") or "持平",
"count": int(item.get("count") or 0),
"strength": float(item.get("strength") or context.get("strength") or 0),
"change": float(context.get("change") or 0),
"leader": item.get("leader") or context.get("leader") or "--",
}
)
rows.append(
{
"trade_date": f"{date_key[:4]}-{date_key[4:6]}-{date_key[6:]}",
"sectors": sectors,
}
)
return {
"trade_date": rows[0]["trade_date"] if rows else normalized_date,
"available_days": len(ordered_dates),
"requested_days": limit,
"rows": rows,
}
def rotation_sector_members(self, trade_date: str, sector_name: str) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
sector_name = validate_text(sector_name, "板块名称", 60, required=True)
dashboard = self.get_dashboard(normalized_date)
actual_date = normalize_date(
str((dashboard.get("meta") or {}).get("trade_date") or normalized_date)
)
cache_key = f"{actual_date}:{sector_name}"
cached = self.database.get_data_snapshot("rotation_sector_members_v1", cache_key)
if cached:
cached["meta"] = {**(cached.get("meta") or {}), "cached": True}
return cached
if not self.configured:
raise ValueError("板块成分数据暂不可用。")
representative = next(
(
item for item in dashboard.get("limits") or []
if str(item.get("sector") or "").strip() == sector_name
),
None,
)
if not representative:
raise ValueError("未找到该板块的代表股票,暂时无法核验成分股。")
raw_code = str(representative.get("ts_code") or representative.get("code") or "")
if "." in raw_code:
ts_code = raw_code
elif raw_code.startswith(("4", "8", "92")):
ts_code = f"{raw_code}.BJ"
elif raw_code.startswith(("6", "68", "90")):
ts_code = f"{raw_code}.SH"
else:
ts_code = f"{raw_code}.SZ"
client = self._tushare_client()
try:
industry = client.sw_stock_industry(ts_code, actual_date)
sector_code = str(industry.get("l2_code") or "")
members = client.sw_sector_members(sector_code, actual_date)
except TushareError as exc:
raise ValueError(f"该板块成分股暂不可用:{exc}") from exc
daily_rows = self.database.daily_bars_for_date(actual_date)
if len(daily_rows) < 1000:
try:
daily_rows = client.query(
"daily",
{"trade_date": actual_date},
"ts_code,trade_date,open,high,low,close,pct_chg,vol,amount",
)
if daily_rows:
self.database.upsert_daily_bars(daily_rows)
except TushareError:
daily_rows = self.database.daily_bars_for_date(actual_date)
daily_map = {str(item.get("ts_code") or ""): item for item in daily_rows}
rows = []
for member in members:
member_code = str(member.get("ts_code") or "")
quote = daily_map.get(member_code) or {}
rows.append(
{
"code": member_code.split(".")[0],
"ts_code": member_code,
"name": str(member.get("name") or "--"),
"change": quote.get("pct_chg"),
"open": quote.get("open"),
"close": quote.get("close"),
"amount_billion": (
round(float(quote.get("amount") or 0) / 100000, 2)
if quote else None
),
"quoted": bool(quote),
}
)
rows.sort(
key=lambda item: (
bool(item.get("quoted")),
float(item.get("change") or -999),
float(item.get("amount_billion") or 0),
),
reverse=True,
)
result = {
"meta": {
"trade_date": self._display_compact_date(actual_date),
"sector_name": str(industry.get("l2_name") or sector_name),
"sector_code": sector_code,
"member_count": len(rows),
"quoted_count": sum(bool(item.get("quoted")) for item in rows),
"cached": False,
},
"rows": rows,
}
self.database.save_data_snapshot(
"rotation_sector_members_v1", cache_key, "tushare", result
)
return result
@@ -0,0 +1,19 @@
"""Market sentiment cycle and history feature."""
from .engine import (
COMPONENT_WEIGHTS,
SENTIMENT_ENGINE_VERSION,
apply_sentiment_to_dashboard,
build_sentiment_history,
latest_contiguous_history,
)
from .service import SentimentServiceMixin
__all__ = [
"COMPONENT_WEIGHTS",
"SENTIMENT_ENGINE_VERSION",
"SentimentServiceMixin",
"apply_sentiment_to_dashboard",
"build_sentiment_history",
"latest_contiguous_history",
]
+496
View File
@@ -0,0 +1,496 @@
from __future__ import annotations
from copy import deepcopy
from statistics import mean, median
from typing import Any
COMPONENT_WEIGHTS = {
"breadth": 20,
"limit_ecology": 25,
"profit_effect": 30,
"ladder_structure": 15,
"liquidity": 10,
}
SENTIMENT_ENGINE_VERSION = 2
def _number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if number == number else default
except (TypeError, ValueError):
return default
def _clamp(value: float, lower: float = 0.0, upper: float = 100.0) -> float:
return min(upper, max(lower, value))
def _linear(value: float, low: float, high: float) -> float:
if high <= low:
return 50.0
return _clamp((value - low) / (high - low) * 100)
def _percentile(value: float, history: list[float]) -> float:
if not history:
return 50.0
below = sum(item < value for item in history)
equal = sum(item == value for item in history)
return _clamp((below + equal * 0.5) / len(history) * 100)
def _adaptive_score(value: float, fixed: float, history: list[float]) -> float:
if len(history) < 20:
return fixed
return fixed * 0.25 + _percentile(value, history[-250:]) * 0.75
def _trade_date(payload: dict[str, Any]) -> str:
meta = payload.get("meta") or {}
return str(meta.get("trade_date") or payload.get("_snapshot_date") or "").replace("-", "")
def _deduplicate_snapshots(snapshots: list[dict[str, Any]]) -> list[dict[str, Any]]:
by_trade_date: dict[str, dict[str, Any]] = {}
for payload in snapshots:
trade_date = _trade_date(payload)
if trade_date:
by_trade_date[trade_date] = payload
return [by_trade_date[key] for key in sorted(by_trade_date)]
def _snapshot_stats(payload: dict[str, Any]) -> dict[str, Any]:
overview = payload.get("overview") or {}
meta = payload.get("meta") or {}
limits = list(payload.get("limits") or [])
broken = list(payload.get("broken") or [])
down_limits = list(payload.get("down_limits") or [])
yesterday = list(payload.get("yesterday_limits") or [])
limit_up = len(limits) if limits else int(_number(overview.get("limit_up_count")))
broken_count = len(broken) if broken else int(_number(overview.get("broken_count")))
limit_down = len(down_limits) if down_limits else int(_number(overview.get("limit_down_count")))
streaks = [max(1, int(_number(row.get("streak"), 1))) for row in limits]
first_board = sum(streak == 1 for streak in streaks)
second_board = sum(streak == 2 for streak in streaks)
three_plus = sum(streak >= 3 for streak in streaks)
max_height = max(streaks, default=0)
present_levels = set(streaks)
ladder_completeness = (
sum(level in present_levels for level in range(1, max_height + 1)) / max_height * 100
if max_height else 0.0
)
up_count = int(_number(overview.get("up_count")))
down_count = int(_number(overview.get("down_count")))
flat_count = int(_number(overview.get("flat_count")))
active_count = up_count + down_count
breadth_ratio = up_count / max(active_count, 1) * 100
seal_rate = _number(overview.get("seal_rate"))
if not seal_rate and limit_up + broken_count:
seal_rate = limit_up / (limit_up + broken_count) * 100
previous_limit_count = len(yesterday)
previous_positive_count = sum(_number(row.get("current_change")) > 0 for row in yesterday)
previous_positive_rate = previous_positive_count / max(previous_limit_count, 1) * 100
advanced_count = sum(row.get("outcome") == "晋级" for row in yesterday)
advance_rate = advanced_count / max(previous_limit_count, 1) * 100
average_previous_change = (
mean(_number(row.get("current_change")) for row in yesterday) if yesterday else 0.0
)
median_previous_change = (
median(_number(row.get("current_change")) for row in yesterday) if yesterday else 0.0
)
severe_loss_count = sum(_number(row.get("current_change")) <= -5 for row in yesterday)
severe_loss_rate = severe_loss_count / max(previous_limit_count, 1) * 100
previous_down_count = sum(row.get("outcome") == "跌停" for row in yesterday)
high_previous = [row for row in yesterday if int(_number(row.get("prior_streak"), 1)) >= 2]
high_positive_rate = (
sum(_number(row.get("current_change")) > 0 for row in high_previous)
/ max(len(high_previous), 1)
* 100
)
amount_billion = _number(overview.get("amount_billion"))
limit_amount_billion = sum(_number(row.get("amount_billion")) for row in limits)
return {
"trade_date": _trade_date(payload),
"previous_trade_date": str(meta.get("previous_trade_date") or "").replace("-", ""),
"up_count": up_count,
"down_count": down_count,
"flat_count": flat_count,
"breadth_ratio": round(breadth_ratio, 1),
"limit_up_count": limit_up,
"first_board_count": first_board,
"second_board_count": second_board,
"three_plus_count": three_plus,
"max_height": max_height,
"ladder_completeness": round(ladder_completeness, 1),
"broken_count": broken_count,
"limit_down_count": limit_down,
"seal_rate": round(seal_rate, 1),
"previous_limit_count": previous_limit_count,
"previous_positive_count": previous_positive_count,
"previous_positive_rate": round(previous_positive_rate, 1),
"advance_rate": round(advance_rate, 1),
"average_previous_change": round(average_previous_change, 2),
"median_previous_change": round(median_previous_change, 2),
"severe_loss_count": severe_loss_count,
"severe_loss_rate": round(severe_loss_rate, 1),
"previous_down_count": previous_down_count,
"high_positive_rate": round(high_positive_rate, 1),
"amount_billion": round(amount_billion, 1),
"limit_amount_billion": round(limit_amount_billion, 2),
}
def _sentiment_label(score: float) -> str:
if score >= 80:
return "情绪高涨"
if score >= 60:
return "情绪偏强"
if score >= 40:
return "情绪中性"
if score >= 20:
return "情绪偏弱"
return "情绪冰点"
def _phase_signal(score: float, momentum: float, profit_score: float) -> str:
if score < 25:
return "修复" if momentum > 3 else "冰点"
if score < 45:
return "修复" if momentum > 3 else "退潮"
if score >= 80:
return "高潮" if momentum >= -2 and profit_score >= 60 else "分化"
if score >= 65:
return "分化" if momentum < -3 or profit_score < 50 else "发酵"
if momentum < -5:
return "退潮"
return "发酵" if momentum >= 0 and profit_score >= 45 else "分化"
def _confirmed_phase(
previous: dict[str, Any] | None,
score: float,
day_change: float,
systemic_health: float,
profit_score: float,
ecology_score: float,
phase_signal: str,
extreme_ice: bool,
fermentation_signal_count: int,
) -> tuple[str, str]:
if previous is None:
return phase_signal, "首个连续交易日,采用原始阶段信号"
previous_phase = str(previous.get("phase") or phase_signal)
if extreme_ice:
return "冰点", "市场宽度与跌停数量触发极端冰点"
recovery = day_change >= 6 and score >= 25 and systemic_health >= 24
fermentation_confirmed = fermentation_signal_count >= 2
climax_ready = (
score >= 80
and profit_score >= 60
and systemic_health >= 60
and ecology_score >= 70
)
if previous_phase == "冰点":
return ("修复", "冰点后首次有效回升") if recovery else ("冰点", "冰点尚未形成有效修复")
if previous_phase == "退潮":
if score < 25:
return "冰点", "退潮继续下探至冰点区间"
return ("修复", "退潮后出现有效回升") if recovery else ("退潮", "退潮尚未形成有效修复")
if previous_phase == "修复":
if score < 25:
return "冰点", "修复失败并重新跌入冰点区间"
if day_change <= -6 and score < 45:
return "退潮", "修复失败且温度显著回落"
if fermentation_confirmed:
return "发酵", "发酵条件连续两个交易日成立"
return "修复", "修复延续,等待发酵确认"
if previous_phase == "发酵":
if score < 25:
return "冰点", "发酵阶段出现极端情绪坍塌"
if score < 45 and (day_change < 0 or systemic_health < 35):
return "退潮", "发酵阶段温度与系统健康度同步转弱"
if climax_ready:
return "高潮", "温度、赚钱效应与涨停生态共同达到高潮条件"
if phase_signal in {"分化", "退潮"} or day_change <= -6:
return "分化", "发酵阶段出现降温或赚钱效应弱化"
return "发酵", "发酵状态延续"
if previous_phase == "高潮":
if score < 25:
return "冰点", "高潮后出现极端情绪坍塌"
if climax_ready:
return "高潮", "高潮条件继续成立"
if score < 45 or systemic_health < 30:
return "退潮", "高潮后风险快速释放"
return "分化", "高潮条件消退,进入分化"
if previous_phase == "分化":
if score < 25:
return "冰点", "分化继续恶化至冰点区间"
if score < 45 or systemic_health < 30:
return "退潮", "分化后温度或系统健康度继续下降"
if fermentation_confirmed:
return "发酵", "分化转强条件连续两个交易日成立"
return "分化", "分化延续,等待方向确认"
return phase_signal, "采用原始阶段信号"
def build_sentiment_history(snapshots: list[dict[str, Any]]) -> list[dict[str, Any]]:
payloads = _deduplicate_snapshots(snapshots)
raw_rows = [_snapshot_stats(payload) for payload in payloads]
results: list[dict[str, Any]] = []
for index, stats in enumerate(raw_rows):
previous = raw_rows[:index]
limit_history = [float(row["limit_up_count"]) for row in previous]
down_limit_history = [float(row["limit_down_count"]) for row in previous]
height_history = [float(row["max_height"]) for row in previous]
three_plus_history = [float(row["three_plus_count"]) for row in previous]
amount_history = [float(row["amount_billion"]) for row in previous[-20:] if row["amount_billion"]]
breadth_score = _clamp(float(stats["breadth_ratio"]))
limit_strength = _adaptive_score(
float(stats["limit_up_count"]),
_linear(float(stats["limit_up_count"]), 10, 100),
limit_history,
)
down_relief = 100 - _adaptive_score(
float(stats["limit_down_count"]),
_linear(float(stats["limit_down_count"]), 0, 50),
down_limit_history,
)
seal_quality = _linear(float(stats["seal_rate"]), 35, 90)
systemic_health = breadth_score * 0.60 + down_relief * 0.40
systemic_gate = 1.0 if systemic_health >= 35 else 0.35 + systemic_health / 35 * 0.65
ecology_base_score = limit_strength * 0.35 + seal_quality * 0.35 + down_relief * 0.30
# Systemic risk is applied once to the final temperature. Reapplying it here
# would count market breadth and limit-down pressure twice.
limit_ecology_score = ecology_base_score
if stats["previous_limit_count"]:
positive_score = float(stats["previous_positive_rate"])
average_change_score = _clamp(50 + float(stats["average_previous_change"]) * 6)
median_change_score = _clamp(50 + float(stats["median_previous_change"]) * 7)
advance_score = _clamp(float(stats["advance_rate"]) * 2.5)
severe_loss_safety = _clamp(100 - float(stats["severe_loss_rate"]) * 3)
down_safety = _clamp(100 - float(stats["previous_down_count"]) / stats["previous_limit_count"] * 700)
tail_safety_score = severe_loss_safety * 0.70 + down_safety * 0.30
profit_effect_score = (
positive_score * 0.30
+ median_change_score * 0.25
+ average_change_score * 0.10
+ advance_score * 0.20
+ tail_safety_score * 0.15
)
else:
profit_effect_score = 50.0
max_height_score = _adaptive_score(
float(stats["max_height"]),
_linear(float(stats["max_height"]), 1, 7),
height_history,
)
continuation_rate = (
(float(stats["second_board_count"]) + float(stats["three_plus_count"]))
/ max(float(stats["limit_up_count"]), 1)
* 100
)
three_plus_density = float(stats["three_plus_count"]) / max(float(stats["limit_up_count"]), 1) * 100
three_plus_score = _adaptive_score(
float(stats["three_plus_count"]),
_clamp(three_plus_density * 5),
three_plus_history,
)
ladder_structure_score = (
max_height_score * 0.30
+ _clamp(continuation_rate * 3) * 0.25
+ three_plus_score * 0.25
+ float(stats["ladder_completeness"]) * 0.20
)
amount_baseline = mean(amount_history) if amount_history else float(stats["amount_billion"] or 1)
amount_ratio = float(stats["amount_billion"]) / max(amount_baseline, 1)
amount_score = _clamp(50 + (amount_ratio - 1) * 100)
limit_amount_share = float(stats["limit_amount_billion"]) / max(float(stats["amount_billion"]), 1) * 100
liquidity_score = amount_score * 0.70 + _clamp(limit_amount_share * 20) * 0.30
component_scores = {
"breadth": breadth_score,
"limit_ecology": limit_ecology_score,
"profit_effect": profit_effect_score,
"ladder_structure": ladder_structure_score,
"liquidity": liquidity_score,
}
raw_score = sum(component_scores[key] * weight / 100 for key, weight in COMPONENT_WEIGHTS.items())
score = round(
raw_score * systemic_gate
)
extreme_ice = float(stats["breadth_ratio"]) <= 15 and float(stats["limit_down_count"]) >= 100
if extreme_ice:
score = min(score, 15)
elif float(stats["breadth_ratio"]) <= 25 and float(stats["limit_down_count"]) >= 50:
score = min(score, 24)
previous_scores: list[float] = []
expected_date = str(stats.get("previous_trade_date") or "")
for prior_result in reversed(results):
if not expected_date or str(prior_result.get("trade_date") or "") != expected_date:
break
previous_scores.append(float(prior_result["score"]))
expected_date = str(prior_result.get("previous_trade_date") or "")
if len(previous_scores) == 3:
break
momentum = score - mean(previous_scores) if previous_scores else 0.0
direction = "升温" if momentum > 3 else "降温" if momentum < -3 else "持平"
normalization = "历史百分位" if len(previous) >= 20 else "固定锚点"
previous_result = (
results[-1]
if results and str(stats.get("previous_trade_date") or "") == str(results[-1].get("trade_date") or "")
else None
)
day_change = score - float(previous_result["score"]) if previous_result else 0.0
ema_score = round(
score if not previous_result
else score * 0.5 + float(previous_result.get("ema_score", previous_result["score"])) * 0.5,
1,
)
phase_signal = _phase_signal(score, momentum, profit_effect_score)
fermentation_ready = (
phase_signal == "发酵"
and score >= 45
and profit_effect_score >= 45
and systemic_health >= 35
and not extreme_ice
)
previous_fermentation_count = int(previous_result.get("fermentation_signal_count") or 0) if previous_result else 0
fermentation_signal_count = previous_fermentation_count + 1 if fermentation_ready else 0
phase, transition_reason = _confirmed_phase(
previous_result,
score,
day_change,
systemic_health,
profit_effect_score,
limit_ecology_score,
phase_signal,
extreme_ice,
fermentation_signal_count,
)
previous_phase = str(previous_result.get("phase") or "") if previous_result else ""
if phase not in {"修复", "分化"}:
fermentation_signal_count = 0
elif phase == "分化" and previous_phase != "分化":
fermentation_signal_count = 0
components = {
"breadth": {
"label": "市场宽度",
"score": round(breadth_score, 1),
"weight": COMPONENT_WEIGHTS["breadth"],
"summary": f"上涨占比 {stats['breadth_ratio']:.1f}%",
},
"limit_ecology": {
"label": "涨停生态",
"score": round(limit_ecology_score, 1),
"weight": COMPONENT_WEIGHTS["limit_ecology"],
"summary": (
f"涨停 {stats['limit_up_count']} · 跌停 {stats['limit_down_count']} · "
f"封板 {stats['seal_rate']:.1f}%"
),
},
"profit_effect": {
"label": "赚钱效应",
"score": round(profit_effect_score, 1),
"weight": COMPONENT_WEIGHTS["profit_effect"],
"summary": (
f"昨涨停红盘 {stats['previous_positive_rate']:.1f}% · "
f"中位 {stats['median_previous_change']:+.2f}% · "
f"重亏 {stats['severe_loss_rate']:.1f}%"
if stats["previous_limit_count"] else "缺少前一交易日样本"
),
},
"ladder_structure": {
"label": "连板结构",
"score": round(ladder_structure_score, 1),
"weight": COMPONENT_WEIGHTS["ladder_structure"],
"summary": f"最高 {stats['max_height']} 板 · 三板以上 {stats['three_plus_count']}",
},
"liquidity": {
"label": "成交活跃度",
"score": round(liquidity_score, 1),
"weight": COMPONENT_WEIGHTS["liquidity"],
"summary": f"成交 {stats['amount_billion']:.1f} 亿 · 均值比 {amount_ratio:.2f}",
},
}
results.append(
{
**stats,
"score": score,
"ema_score": ema_score,
"label": _sentiment_label(score),
"phase": phase,
"phase_signal": phase_signal,
"transition_reason": transition_reason,
"fermentation_signal_count": fermentation_signal_count,
"day_change": round(day_change, 1),
"direction": direction,
"momentum": round(momentum, 1),
"normalization": "250日历史百分位" if len(previous) >= 20 else normalization,
"history_days": len(previous) + 1,
"systemic_health": round(systemic_health, 1),
"risk_multiplier": round(systemic_gate, 3),
"components": components,
}
)
return results
def latest_contiguous_history(series: list[dict[str, Any]]) -> list[dict[str, Any]]:
if not series:
return []
contiguous = [series[-1]]
for row in reversed(series[:-1]):
expected_previous = str(contiguous[0].get("previous_trade_date") or "")
if not expected_previous or expected_previous != str(row.get("trade_date") or ""):
break
contiguous.insert(0, row)
return contiguous
def apply_sentiment_to_dashboard(
dashboard: dict[str, Any],
historical_snapshots: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
result = deepcopy(dashboard)
history = list(historical_snapshots or [])
history.append(result)
series = build_sentiment_history(history)
target_date = _trade_date(result)
sentiment = next((row for row in reversed(series) if row["trade_date"] == target_date), None)
if not sentiment:
return result
overview = dict(result.get("overview") or {})
overview.update(
{
"sentiment_score": sentiment["score"],
"sentiment_trend_score": sentiment["ema_score"],
"sentiment_label": sentiment["label"],
"sentiment_phase": sentiment["phase"],
"sentiment_direction": sentiment["direction"],
"sentiment_components": sentiment["components"],
"sentiment_engine_version": SENTIMENT_ENGINE_VERSION,
}
)
result["overview"] = overview
return result
+39
View File
@@ -0,0 +1,39 @@
from __future__ import annotations
from typing import Any
from backend.bootstrap.config import normalize_date
from backend.features.sentiment.engine import (
COMPONENT_WEIGHTS,
apply_sentiment_to_dashboard,
build_sentiment_history,
latest_contiguous_history,
)
class SentimentServiceMixin:
def _enrich_dashboard_sentiment(
self,
dashboard: dict[str, Any],
end_date: str,
) -> dict[str, Any]:
history = self.database.list_snapshot_payloads(end_date, 260)
return apply_sentiment_to_dashboard(dashboard, history)
def sentiment_history(self, trade_date: str, limit: int = 20) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
limit = max(10, min(120, int(limit)))
full_series = build_sentiment_history(
self.database.list_snapshot_payloads(normalized_date, 240)
)
series = latest_contiguous_history(full_series)
rows = series[-limit:]
return {
"trade_date": rows[-1]["trade_date"] if rows else normalized_date,
"available_days": len(series),
"stored_days": len(full_series),
"requested_days": limit,
"rows": rows,
"weights": COMPONENT_WEIGHTS,
"normalization": rows[-1]["normalization"] if rows else "固定锚点",
}
+4 -494
View File
@@ -1,497 +1,7 @@
from __future__ import annotations
"""Compatibility alias for the canonical market chart clients."""
import http.client
import json
import re
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from datetime import datetime, time as dt_time, timedelta
from threading import Lock
from typing import Any, ClassVar
import sys
from ifind_client import IfindError, IfindHttpClient
from backend.features.market import charts as _implementation
class ChartDataError(RuntimeError):
pass
TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get"
BOARD_LIST_URL = "https://push2delay.eastmoney.com/api/qt/clist/get"
BROWSER_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/138.0.0.0 Safari/537.36"
)
INDEX_SECIDS = {
"000001.SH": "1.000001",
"399001.SZ": "0.399001",
"399006.SZ": "0.399006",
}
class MarketChartClient:
"""Prefer iFinD for display charts and retain Eastmoney as a last resort."""
def __init__(self, ifind: IfindHttpClient, fallback: "EastmoneyChartClient") -> None:
self.ifind = ifind
self.fallback = fallback
def stock_intraday(self, code: str) -> dict[str, Any]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
ifind_code = _stock_market_code(normalized)
try:
return self._ifind_intraday(ifind_code, "stock", normalized)
except (IfindError, ChartDataError):
return self.fallback.stock_intraday(normalized)
def stock_daily(self, code: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
return self._ifind_daily(_stock_market_code(normalized), end_date, limit)
def index_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(identifier or "").strip().upper()
if normalized not in INDEX_SECIDS:
raise ChartDataError("Unsupported index")
return self._ifind_daily(normalized, end_date, limit)
def board_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(identifier or "").strip().upper()
if not normalized:
raise ChartDataError("Invalid board code")
return self._ifind_daily(normalized, end_date, limit)
def index_intraday(self, identifier: str) -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
if normalized not in INDEX_SECIDS:
raise ChartDataError("Unsupported index")
try:
return self._ifind_intraday(normalized, "index", normalized)
except (IfindError, ChartDataError):
return self.fallback.index_intraday(normalized)
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
try:
return self._ifind_intraday(normalized, "board", normalized, name)
except (IfindError, ChartDataError):
return self.fallback.board_intraday(normalized, name)
def _ifind_intraday(
self,
ifind_code: str,
entity_type: str,
identifier: str,
name: str = "",
) -> dict[str, Any]:
if not self.ifind.configured:
raise ChartDataError("iFinD is not configured")
now = datetime.now().astimezone()
rows: list[dict[str, Any]] = []
for offset in range(0, 8):
candidate = now.date() - timedelta(days=offset)
if candidate.weekday() >= 5:
continue
display_date = candidate.isoformat()
rows = self.ifind.intraday(
ifind_code,
f"{display_date} 09:30:00",
f"{display_date} 15:00:00",
cache_ttl=20 if offset == 0 else 6 * 60 * 60,
)
if rows:
break
points = [point for row in rows if (point := _ifind_point(row))]
if not points:
raise ChartDataError("No iFinD intraday chart data returned")
latest_date = points[-1]["date"]
points = [point for point in points if point["date"] == latest_date]
previous_close = self._previous_close(ifind_code, latest_date, points[0]["open"])
return {
"entity_type": entity_type,
"identifier": identifier,
"name": name,
"code": identifier,
"trade_date": latest_date,
"previous_close": previous_close,
"points": points,
"source": "ifind",
}
def _ifind_daily(
self, ifind_code: str, end_date: str, limit: int
) -> list[dict[str, Any]]:
if not self.ifind.configured:
raise ChartDataError("iFinD is not configured")
compact_end = str(end_date or "").replace("-", "")
if not re.fullmatch(r"\d{8}", compact_end):
raise ChartDataError("Invalid chart end date")
end = datetime.strptime(compact_end, "%Y%m%d")
start = (end - timedelta(days=max(190, limit * 3))).strftime("%Y%m%d")
try:
rows = self.ifind.history(
ifind_code,
["open", "high", "low", "close", "volume", "amount"],
start,
compact_end,
cache_ttl=300,
)
except IfindError as exc:
raise ChartDataError("No iFinD daily chart data returned") from exc
normalized = []
for row in rows:
stamp = str(row.get("time") or "").strip()
trade_date = stamp[:10]
close = _number(row.get("close"))
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", trade_date) or close <= 0:
continue
normalized.append(
{
"trade_date": trade_date,
"open": _number(row.get("open")),
"high": _number(row.get("high")),
"low": _number(row.get("low")),
"close": close,
"volume": _number(row.get("volume")),
"amount_billion": _number(row.get("amount")) / 100_000_000,
}
)
normalized.sort(key=lambda row: row["trade_date"])
for index, row in enumerate(normalized):
previous = normalized[index - 1]["close"] if index > 0 else 0
row["change"] = round((row["close"] / previous - 1) * 100, 4) if previous else 0.0
market_now = datetime.now().astimezone()
today = market_now.strftime("%Y%m%d")
market_open = (
market_now.weekday() < 5
and market_now.time().replace(tzinfo=None) >= dt_time(9, 30)
)
today_display = market_now.date().isoformat()
if normalized and normalized[-1]["trade_date"] == today_display:
current_bar = normalized[-1]
current_bar_is_valid = (
current_bar["open"] > 0
and current_bar["high"] >= max(current_bar["open"], current_bar["close"])
and 0 < current_bar["low"] <= min(current_bar["open"], current_bar["close"])
and (current_bar["volume"] > 0 or current_bar["amount_billion"] > 0)
)
if not market_open or not current_bar_is_valid:
normalized.pop()
if compact_end == today and market_open:
try:
quote_rows = self.ifind.real_time(
ifind_code,
["open", "high", "low", "latest", "preClose", "volume", "amount"],
cache_ttl=10,
)
quote = quote_rows[0] if quote_rows else {}
latest = _number(quote.get("latest"))
previous = _number(quote.get("preClose"))
open_price = _number(quote.get("open"))
high = _number(quote.get("high"))
low = _number(quote.get("low"))
volume = _number(quote.get("volume"))
amount = _number(quote.get("amount"))
quote_date = str(quote.get("time") or "")[:10].replace("-", "")
quote_is_current = not quote_date or quote_date == today
has_market_activity = volume > 0 or amount > 0
if (
latest > 0
and open_price > 0
and high >= max(open_price, latest)
and 0 < low <= min(open_price, latest)
and has_market_activity
and quote_is_current
):
realtime = {
"trade_date": end.strftime("%Y-%m-%d"),
"open": open_price,
"high": high,
"low": low,
"close": latest,
"change": round((latest / previous - 1) * 100, 4) if previous else 0.0,
"volume": volume,
"amount_billion": amount / 100_000_000,
"realtime": True,
}
if normalized and normalized[-1]["trade_date"] == realtime["trade_date"]:
normalized[-1] = realtime
else:
normalized.append(realtime)
except IfindError:
pass
if not normalized:
raise ChartDataError("No iFinD daily chart data returned")
return normalized[-max(20, min(180, int(limit))):]
def _previous_close(self, code: str, trade_date: str, fallback: float) -> float:
today = datetime.now().astimezone().date().isoformat()
if trade_date == today:
try:
quote = self.ifind.real_time(code, ["preClose"], cache_ttl=20)
value = _number((quote[0] if quote else {}).get("preClose"))
if value > 0:
return value
except IfindError:
pass
end = datetime.strptime(trade_date, "%Y-%m-%d")
try:
rows = self.ifind.history(
code,
["close"],
(end - timedelta(days=12)).strftime("%Y%m%d"),
end.strftime("%Y%m%d"),
cache_ttl=6 * 60 * 60,
)
closes = [_number(row.get("close")) for row in rows if _number(row.get("close")) > 0]
if len(closes) >= 2:
return closes[-2]
except IfindError:
pass
return fallback
@dataclass
class EastmoneyChartClient:
"""Isolated display-only minute chart source.
The returned data must not be used by market snapshots, scoring, screening,
or divination. Its only consumer is a chart-rendering endpoint.
"""
timeout: int = 6
cache_ttl_seconds: int = 20
retry_attempts: int = 2
_cache: ClassVar[dict[str, dict[str, Any]]] = {}
_cache_lock: ClassVar[Lock] = Lock()
_board_catalog: ClassVar[dict[str, dict[str, str]]] = {}
_board_catalog_at: ClassVar[float] = 0.0
_board_catalog_lock: ClassVar[Lock] = Lock()
def stock_intraday(self, code: str) -> dict[str, Any]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
market = "1" if normalized.startswith(("5", "6", "9")) else "0"
return self._intraday(f"{market}.{normalized}", "stock", normalized)
def index_intraday(self, identifier: str) -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
secid = INDEX_SECIDS.get(normalized)
if not secid:
raise ChartDataError("Unsupported index")
return self._intraday(secid, "index", normalized)
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
if re.fullmatch(r"BK\d{4}", normalized):
board_code = normalized
else:
board_code = self._resolve_board_code(name or identifier)
return self._intraday(f"90.{board_code}", "board", board_code)
def _intraday(self, secid: str, entity_type: str, identifier: str) -> dict[str, Any]:
cache_key = f"{entity_type}:{identifier}"
cached = self._get_cached(cache_key)
if cached is not None:
return cached
payload = self._request_json(
TRENDS_URL,
{
"secid": secid,
"fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13",
"fields2": "f51,f52,f53,f54,f55,f56,f57,f58",
"iscr": "0",
"ndays": "1",
},
"https://quote.eastmoney.com/",
)
data = payload.get("data") or {}
points = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))]
if not points:
raise ChartDataError("No intraday chart data returned")
result = {
"entity_type": entity_type,
"identifier": identifier,
"name": str(data.get("name") or ""),
"code": str(data.get("code") or identifier),
"trade_date": points[-1]["date"],
"previous_close": _number(data.get("preClose")),
"points": points,
}
with self._cache_lock:
self._cache[cache_key] = {"created_at": time.time(), "payload": result}
return result
def _get_cached(self, cache_key: str) -> dict[str, Any] | None:
with self._cache_lock:
cached = self._cache.get(cache_key)
if not cached:
return None
if time.time() - float(cached.get("created_at") or 0) > self.cache_ttl_seconds:
with self._cache_lock:
self._cache.pop(cache_key, None)
return None
return dict(cached["payload"])
def _resolve_board_code(self, name: str) -> str:
normalized = _normalize_name(name)
if not normalized:
raise ChartDataError("Board name is required")
catalog = self._load_board_catalog()
item = catalog.get(normalized)
if not item:
raise ChartDataError("No matching chart board")
return item["code"]
def _load_board_catalog(self) -> dict[str, dict[str, str]]:
now = time.time()
with self._board_catalog_lock:
if self._board_catalog and now - self._board_catalog_at < 6 * 60 * 60:
return dict(self._board_catalog)
rows: list[dict[str, Any]] = []
for board_type in ("1", "2", "3"):
for page in range(1, 6):
payload = self._request_json(
BOARD_LIST_URL,
{
"pn": str(page),
"pz": "100",
"po": "1",
"np": "1",
"fltt": "2",
"invt": "2",
"fid": "f3",
"fs": f"m:90+t:{board_type}",
"fields": "f12,f14",
},
"https://quote.eastmoney.com/center/boardlist.html",
)
page_rows = (payload.get("data") or {}).get("diff") or []
rows.extend(page_rows)
if len(page_rows) < 100:
break
catalog: dict[str, dict[str, str]] = {}
for row in rows:
code = str(row.get("f12") or "").strip().upper()
board_name = str(row.get("f14") or "").strip()
if re.fullmatch(r"BK\d{4}", code) and board_name:
catalog.setdefault(_normalize_name(board_name), {"code": code, "name": board_name})
if not catalog:
raise ChartDataError("Board chart directory is unavailable")
with self._board_catalog_lock:
type(self)._board_catalog = catalog
type(self)._board_catalog_at = now
return dict(catalog)
def _request_json(
self, url: str, params: dict[str, str], referer: str
) -> dict[str, Any]:
request_url = f"{url}?{urllib.parse.urlencode(params)}"
last_error: Exception | None = None
for attempt in range(max(1, int(self.retry_attempts))):
request = urllib.request.Request(
request_url,
headers={
"Accept": "application/json,text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
payload = json.loads(response.read().decode("utf-8"))
if not isinstance(payload, dict):
raise ChartDataError("Invalid intraday chart response")
return payload
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
json.JSONDecodeError,
ChartDataError,
) as exc:
last_error = exc
if attempt + 1 < self.retry_attempts:
time.sleep(0.12)
raise ChartDataError("Intraday chart request failed") from last_error
def _parse_trend(raw: Any) -> dict[str, Any] | None:
fields = str(raw or "").split(",")
if len(fields) < 8 or " " not in fields[0]:
return None
stamp = fields[0].strip()
trade_date, trade_time = stamp.split(" ", 1)
close = _number(fields[2])
if close <= 0:
return None
return {
"date": trade_date,
"time": trade_time[:5],
"open": _number(fields[1]),
"close": close,
"high": _number(fields[3]),
"low": _number(fields[4]),
"volume": _number(fields[5]),
"amount": _number(fields[6]),
"average": _number(fields[7]),
}
def _ifind_point(row: dict[str, Any]) -> dict[str, Any] | None:
stamp = str(row.get("time") or "").strip()
if " " not in stamp:
return None
trade_date, trade_time = stamp.split(" ", 1)
close = _number(row.get("close"))
if close <= 0:
return None
return {
"date": trade_date,
"time": trade_time[:5],
"open": _number(row.get("open")),
"close": close,
"high": _number(row.get("high")),
"low": _number(row.get("low")),
"volume": _number(row.get("volume")),
"amount": _number(row.get("amount")),
"average": _number(row.get("avgPrice")),
}
def _stock_market_code(code: str) -> str:
if code.startswith(("4", "8", "9")):
suffix = "BJ"
elif code.startswith("6"):
suffix = "SH"
else:
suffix = "SZ"
return f"{code}.{suffix}"
def _number(value: Any) -> float:
try:
return float(value or 0)
except (TypeError, ValueError):
return 0.0
def _normalize_name(value: Any) -> str:
normalized = re.sub(r"[\s·・()()\-_/]", "", str(value or "")).casefold()
return re.sub(r"(?:概念|行业|[ⅠⅡⅢ])$", "", normalized)
sys.modules[__name__] = _implementation
+8 -234
View File
@@ -8,6 +8,8 @@ from typing import Any
from backend.database import MIGRATIONS, MigrationRunner, SQLiteConnectionFactory
from backend.features.accounts.repository import AccountRepositoryMixin
from backend.features.market.repository import MarketRepositoryMixin
from backend.features.pools.repository import PoolRepositoryMixin
from backend.features.system.repository import SystemSettingsRepositoryMixin
@@ -20,7 +22,12 @@ def _optional_float(value: Any) -> float | None:
return None
class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
class ReviewDatabase(
AccountRepositoryMixin,
MarketRepositoryMixin,
PoolRepositoryMixin,
SystemSettingsRepositoryMixin,
):
def __init__(self, path: Path) -> None:
self.path = path
self.path.parent.mkdir(parents=True, exist_ok=True)
@@ -690,118 +697,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
).fetchone()
return int(row["total"] if row else 0)
def get_snapshot(self, trade_date: str) -> dict[str, Any] | None:
with self.connect() as connection:
row = connection.execute(
"SELECT payload FROM dashboard_snapshots WHERE trade_date = ?",
(trade_date,),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def get_latest_real_snapshot(
self, trade_date: str, strictly_before: bool = False
) -> dict[str, Any] | None:
operator = "<" if strictly_before else "<="
with self.connect() as connection:
row = connection.execute(
f"""
SELECT payload FROM dashboard_snapshots
WHERE trade_date {operator} ? AND source != 'demo'
ORDER BY trade_date DESC LIMIT 1
""",
(trade_date,),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def save_snapshot(self, trade_date: str, source: str, payload: dict[str, Any]) -> None:
updated_at = datetime.now().astimezone().isoformat(timespec="seconds")
record_count = sum(
len(payload.get(key) or [])
for key in ("limits", "broken", "down_limits", "yesterday_limits")
)
content = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
with self.connect() as connection:
connection.execute(
"""
INSERT INTO dashboard_snapshots
(trade_date, source, payload, record_count, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(trade_date) DO UPDATE SET
source = excluded.source,
payload = excluded.payload,
record_count = excluded.record_count,
updated_at = excluded.updated_at
""",
(trade_date, source, content, record_count, updated_at),
)
def get_data_snapshot(self, kind: str, cache_key: str) -> dict[str, Any] | None:
with self.connect() as connection:
row = connection.execute(
"SELECT payload FROM data_snapshots WHERE kind = ? AND cache_key = ?",
(kind, cache_key),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def get_latest_data_snapshot(
self,
kind: str,
cache_key_prefix: str,
maximum_cache_key: str,
exclude_source: str = "",
) -> dict[str, Any] | None:
source_clause = " AND source != ?" if exclude_source else ""
parameters: list[Any] = [kind, f"{cache_key_prefix}%", maximum_cache_key]
if exclude_source:
parameters.append(exclude_source)
with self.connect() as connection:
row = connection.execute(
f"""
SELECT payload FROM data_snapshots
WHERE kind = ? AND cache_key LIKE ? AND cache_key <= ?{source_clause}
ORDER BY cache_key DESC LIMIT 1
""",
parameters,
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def save_data_snapshot(
self, kind: str, cache_key: str, source: str, payload: dict[str, Any]
) -> None:
updated_at = datetime.now().astimezone().isoformat(timespec="seconds")
content = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
with self.connect() as connection:
connection.execute(
"""
INSERT INTO data_snapshots (kind, cache_key, source, payload, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(kind, cache_key) DO UPDATE SET
source = excluded.source,
payload = excluded.payload,
updated_at = excluded.updated_at
""",
(kind, cache_key, source, content, updated_at),
)
def list_watchlist(self, user_id: int) -> list[dict[str, Any]]:
with self.connect() as connection:
@@ -869,7 +764,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
(int(user_id), code),
)
return cursor.rowcount > 0
def list_notes(
self,
user_id: int,
@@ -944,27 +838,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
)
return cursor.rowcount > 0
def save_reason_override(self, trade_date: str, code: str, reason: str) -> None:
now = datetime.now().astimezone().isoformat(timespec="seconds")
with self.connect() as connection:
connection.execute(
"""
INSERT INTO reason_overrides (trade_date, code, reason, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(trade_date, code) DO UPDATE SET
reason = excluded.reason,
updated_at = excluded.updated_at
""",
(trade_date, code, reason, now),
)
def reason_overrides(self, trade_date: str) -> dict[str, str]:
with self.connect() as connection:
rows = connection.execute(
"SELECT code, reason FROM reason_overrides WHERE trade_date = ?",
(trade_date,),
).fetchall()
return {row["code"]: row["reason"] for row in rows}
def list_seat_aliases(self) -> dict[str, str]:
with self.connect() as connection:
@@ -1042,26 +915,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
)
return len(values)
def search_stock_master(self, query: str, limit: int = 12) -> list[dict[str, Any]]:
text = str(query or "").strip()
if not text:
return []
escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
with self.connect() as connection:
rows = connection.execute(
"""
SELECT ts_code, code, name, industry, market, list_date
FROM stock_master
WHERE code = ? OR name = ? OR name LIKE ? ESCAPE '\\'
ORDER BY
CASE WHEN code = ? THEN 0 WHEN name = ? THEN 1 ELSE 2 END,
list_date DESC,
code
LIMIT ?
""",
(text, text, f"%{escaped}%", text, text, max(1, min(30, int(limit)))),
).fetchall()
return [dict(row) for row in rows]
def list_stock_master(self) -> list[dict[str, Any]]:
with self.connect() as connection:
@@ -1666,24 +1519,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
for row in series[-limit:]
]
def list_snapshot_payloads(self, end_date: str, limit: int = 260) -> list[dict[str, Any]]:
with self.connect() as connection:
rows = connection.execute(
"""
SELECT trade_date, payload FROM dashboard_snapshots
WHERE trade_date <= ? ORDER BY trade_date DESC LIMIT ?
""",
(end_date, limit),
).fetchall()
result: list[dict[str, Any]] = []
for row in reversed(rows):
try:
payload = json.loads(row["payload"])
except json.JSONDecodeError:
continue
payload["_snapshot_date"] = row["trade_date"]
result.append(payload)
return result
def save_screener_strategy(
self, user_id: int | None, name: str, description: str, regimes: list[str], formula: dict[str, Any],
@@ -2529,64 +2364,3 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
(int(reading_id), int(user_id)),
)
return cursor.rowcount > 0
def start_sync(self, trade_date: str, source: str) -> int:
started_at = datetime.now().astimezone().isoformat(timespec="seconds")
with self.connect() as connection:
cursor = connection.execute(
"""
INSERT INTO sync_runs (trade_date, source, status, started_at)
VALUES (?, ?, 'running', ?)
""",
(trade_date, source, started_at),
)
return int(cursor.lastrowid)
def finish_sync(
self,
sync_id: int,
status: str,
record_count: int = 0,
message: str = "",
source: str | None = None,
) -> None:
finished_at = datetime.now().astimezone().isoformat(timespec="seconds")
with self.connect() as connection:
connection.execute(
"""
UPDATE sync_runs
SET status = ?, finished_at = ?, record_count = ?, message = ?,
source = COALESCE(?, source)
WHERE id = ?
""",
(status, finished_at, record_count, message[:1000], source, sync_id),
)
def status(self) -> dict[str, Any]:
with self.connect() as connection:
last_sync = connection.execute(
"""
SELECT id, trade_date, source, status, started_at, finished_at,
record_count, message
FROM sync_runs ORDER BY id DESC LIMIT 1
"""
).fetchone()
snapshot_stats = connection.execute(
"""
SELECT COUNT(*) AS dates, COALESCE(SUM(record_count), 0) AS records,
MAX(updated_at) AS updated_at
FROM dashboard_snapshots
"""
).fetchone()
watchlist_count = connection.execute("SELECT COUNT(*) FROM watchlist").fetchone()[0]
note_count = connection.execute("SELECT COUNT(*) FROM review_notes").fetchone()[0]
return {
"database": str(self.path.name),
"snapshot_dates": int(snapshot_stats["dates"]),
"snapshot_records": int(snapshot_stats["records"]),
"updated_at": snapshot_stats["updated_at"],
"last_sync": dict(last_sync) if last_sync else None,
"watchlist_count": int(watchlist_count),
"note_count": int(note_count),
}
+1 -1
View File
@@ -5,7 +5,7 @@ import math
from datetime import datetime, timedelta
from typing import Any
from sentiment_engine import apply_sentiment_to_dashboard
from backend.features.sentiment.engine import apply_sentiment_to_dashboard
DEMO_LIMITS = [
+4 -382
View File
@@ -1,385 +1,7 @@
from __future__ import annotations
"""Compatibility alias for the canonical iFinD provider implementation."""
import copy
import json
import threading
import time
import urllib.error
import urllib.request
from datetime import datetime, timedelta
from typing import Any
import sys
from backend.data.providers import ifind_client as _implementation
class IfindError(RuntimeError):
pass
class IfindHttpClient:
BASE_URL = "https://quantapi.51ifind.com/api/v1"
AUTH_ENDPOINT = "get_access_token"
AUTH_ERROR_CODES = {-1302, -1303, -1304, -4302, -4303}
def __init__(
self,
refresh_token: str = "",
access_token: str = "",
timeout: int = 15,
) -> None:
self.timeout = max(3, int(timeout))
self._refresh_token = str(refresh_token or "").strip()
self._access_token = str(access_token or "").strip()
self._access_expires_at: datetime | None = None
self._token_lock = threading.Lock()
self._cache_lock = threading.Lock()
self._cache: dict[str, dict[str, Any]] = {}
@property
def configured(self) -> bool:
return bool(self._refresh_token or self._access_token)
def set_credentials(self, refresh_token: str, access_token: str = "") -> None:
refresh_token = str(refresh_token or "").strip()
access_token = str(access_token or "").strip()
with self._token_lock:
refresh_changed = refresh_token != self._refresh_token
self._refresh_token = refresh_token
if access_token or refresh_changed:
self._access_token = access_token
self._access_expires_at = None
if refresh_changed:
with self._cache_lock:
self._cache.clear()
def status(self) -> dict[str, Any]:
return {
"configured": self.configured,
"access_ready": bool(self._access_token),
"access_expires_at": (
self._access_expires_at.isoformat(timespec="seconds")
if self._access_expires_at
else ""
),
}
def test_connection(self) -> dict[str, Any]:
payload = self.real_time(
"000001.SH",
["open", "high", "low", "latest", "preClose"],
cache_ttl=0,
)
return {
"ok": bool(payload),
"sample_time": str(payload[0].get("time") or "") if payload else "",
}
def real_time(
self,
codes: str | list[str],
indicators: list[str],
cache_ttl: int = 10,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"real_time_quotation",
{"codes": code_text, "indicators": ",".join(indicators)},
cache_key=f"rq:{code_text}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def history(
self,
codes: str | list[str],
indicators: list[str],
start_date: str,
end_date: str,
cache_ttl: int = 300,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"cmd_history_quotation",
{
"codes": code_text,
"indicators": ",".join(indicators),
"startdate": self._display_date(start_date),
"enddate": self._display_date(end_date),
"functionpara": {"CPS": "forward1", "Fill": "Omit"},
},
cache_key=f"hq:{code_text}:{start_date}:{end_date}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def intraday(
self,
code: str,
start_time: str,
end_time: str,
cache_ttl: int = 20,
) -> list[dict[str, Any]]:
indicators = ["open", "high", "low", "close", "volume", "amount", "avgPrice"]
payload = self._request(
"high_frequency",
{
"codes": self._codes(code),
"indicators": ",".join(indicators),
"starttime": start_time,
"endtime": end_time,
"functionpara": {
"CPS": "forward1",
"Fill": "Previous",
"Timeformat": "LocalTime",
"Interval": "1",
"Limitstart": "09:30:00",
"Limitend": "15:00:00",
},
},
cache_key=f"hf:{code}:{start_time}:{end_time}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def snapshots(
self,
codes: str | list[str],
indicators: list[str],
start_time: str,
end_time: str,
cache_ttl: int = 8,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"snap_shot",
{
"codes": code_text,
"indicators": ",".join(indicators),
"starttime": start_time,
"endtime": end_time,
},
cache_key=f"ss:{code_text}:{start_time}:{end_time}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def wencai(self, query: str, search_type: str = "stock", cache_ttl: int = 300) -> list[dict[str, Any]]:
normalized = " ".join(str(query or "").split())
if not normalized:
raise IfindError("问财查询不能为空。")
payload = self._request(
"smart_stock_picking",
{"searchstring": normalized, "searchtype": search_type},
cache_key=f"wc:{search_type}:{normalized}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def report_query(
self,
codes: str | list[str],
begin_date: str,
end_date: str,
cache_ttl: int = 300,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"report_query",
{
"codes": code_text,
"beginrDate": self._display_date(begin_date),
"endrDate": self._display_date(end_date),
"outputpara": (
"reportDate:Y,thscode:Y,secName:Y,ctime:Y,"
"reportTitle:Y,pdfURL:Y,seq:Y"
),
},
cache_key=f"report:{code_text}:{begin_date}:{end_date}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def _request(
self,
endpoint: str,
body: dict[str, Any],
cache_key: str = "",
cache_ttl: int = 0,
) -> dict[str, Any]:
if not self.configured:
raise IfindError("iFinD 尚未配置。")
if cache_key and cache_ttl > 0:
cached = self._cached(cache_key, cache_ttl)
if cached is not None:
return cached
payload = self._post(endpoint, body, self._ensure_access_token())
if self._is_auth_error(payload) and self._refresh_token:
self._invalidate_access_token()
payload = self._post(endpoint, body, self._ensure_access_token(force=True))
self._validate_payload(payload)
if cache_key and cache_ttl > 0:
with self._cache_lock:
self._cache[cache_key] = {
"created_at": time.time(),
"payload": copy.deepcopy(payload),
}
return payload
def _ensure_access_token(self, force: bool = False) -> str:
with self._token_lock:
now = datetime.now().astimezone().replace(tzinfo=None)
token_valid = bool(self._access_token) and (
self._access_expires_at is None
or self._access_expires_at > now + timedelta(minutes=2)
)
if token_valid and not force:
return self._access_token
if not self._refresh_token:
if self._access_token:
return self._access_token
raise IfindError("iFinD Refresh Token 尚未配置。")
payload = self._post(self.AUTH_ENDPOINT, {}, "", self._refresh_token)
self._validate_payload(payload)
data = payload.get("data") or {}
token = str(data.get("access_token") or "").strip()
if not token:
raise IfindError("iFinD 未返回 Access Token。")
expires_at = self._parse_datetime(data.get("expired_time"))
self._access_token = token
self._access_expires_at = expires_at
return token
def _post(
self,
endpoint: str,
body: dict[str, Any],
access_token: str,
refresh_token: str = "",
) -> dict[str, Any]:
headers = {
"Accept": "application/json",
"Content-Type": "application/json",
"User-Agent": "XiaobaiReviewWeb/1.0",
"ifindlang": "cn",
}
if access_token:
headers["access_token"] = access_token
if refresh_token:
headers["refresh_token"] = refresh_token
request = urllib.request.Request(
f"{self.BASE_URL}/{endpoint}",
data=json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
headers=headers,
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
payload = json.loads(response.read().decode("utf-8"))
except urllib.error.HTTPError as exc:
detail = ""
try:
detail_payload = json.loads(exc.read().decode("utf-8", errors="replace"))
detail = str(detail_payload.get("errmsg") or detail_payload.get("message") or "")
except (json.JSONDecodeError, OSError):
pass
raise IfindError(f"iFinD HTTP {exc.code}{f'{detail[:160]}' if detail else ''}") from exc
except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
raise IfindError("iFinD 数据请求失败。") from exc
if not isinstance(payload, dict):
raise IfindError("iFinD 返回格式不正确。")
return payload
def _cached(self, key: str, ttl: int) -> dict[str, Any] | None:
with self._cache_lock:
cached = self._cache.get(key)
if not cached:
return None
if time.time() - float(cached.get("created_at") or 0) > ttl:
self._cache.pop(key, None)
return None
return copy.deepcopy(cached["payload"])
def _invalidate_access_token(self) -> None:
with self._token_lock:
self._access_token = ""
self._access_expires_at = None
@classmethod
def _validate_payload(cls, payload: dict[str, Any]) -> None:
try:
error_code = int(payload.get("errorcode") or 0)
except (TypeError, ValueError):
error_code = -1
if error_code != 0:
message = str(payload.get("errmsg") or "未知错误")
raise IfindError(f"iFinD 返回错误:{message[:200]}")
@classmethod
def _is_auth_error(cls, payload: dict[str, Any]) -> bool:
try:
error_code = int(payload.get("errorcode") or 0)
except (TypeError, ValueError):
error_code = 0
message = str(payload.get("errmsg") or "").casefold()
return error_code in cls.AUTH_ERROR_CODES or "token" in message or "鉴权" in message
@staticmethod
def _table_rows(payload: dict[str, Any]) -> list[dict[str, Any]]:
tables = payload.get("tables") or []
if isinstance(tables, dict):
tables = [tables]
rows: list[dict[str, Any]] = []
for block in tables if isinstance(tables, list) else []:
if not isinstance(block, dict):
continue
table = block.get("table") or {}
if not isinstance(table, dict):
continue
times = block.get("time") or []
codes = block.get("thscode") or block.get("thscodes") or []
if isinstance(codes, str):
codes = [codes]
lengths = [len(value) for value in table.values() if isinstance(value, list)]
row_count = max(lengths or [len(times) if isinstance(times, list) else 0, 1 if table else 0])
for index in range(row_count):
row: dict[str, Any] = {}
if isinstance(times, list) and index < len(times):
row["time"] = times[index]
if codes:
row["thscode"] = codes[index] if index < len(codes) else codes[0]
for field, values in table.items():
if isinstance(values, list):
row[field] = values[index] if index < len(values) else None
elif index == 0:
row[field] = values
rows.append(row)
return rows
@staticmethod
def _codes(codes: str | list[str]) -> str:
if isinstance(codes, list):
values = [str(code or "").strip().upper() for code in codes]
else:
values = [part.strip().upper() for part in str(codes or "").split(",")]
values = [value for value in values if value]
if not values:
raise IfindError("iFinD 证券代码不能为空。")
if len(values) > 100:
raise IfindError("iFinD 单次证券代码过多。")
return ",".join(values)
@staticmethod
def _display_date(value: str) -> str:
compact = str(value or "").replace("-", "")
if len(compact) != 8 or not compact.isdigit():
raise IfindError("iFinD 日期格式不正确。")
return f"{compact[:4]}-{compact[4:6]}-{compact[6:]}"
@staticmethod
def _parse_datetime(value: Any) -> datetime | None:
text = str(value or "").strip()
if not text:
return None
try:
return datetime.fromisoformat(text)
except ValueError:
return None
sys.modules[__name__] = _implementation
+4 -423
View File
@@ -1,426 +1,7 @@
from __future__ import annotations
"""Compatibility alias for the canonical display-only realtime observer."""
import copy
import http.client
import json
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from datetime import datetime
from threading import Lock
from typing import Any, ClassVar
import sys
from backend.data import realtime as _implementation
class RealtimeAggregateError(RuntimeError):
pass
EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get"
EASTMONEY_SECTOR_URL = "https://push2.eastmoney.com/api/qt/clist/get"
TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006"
THS_LIMIT_URL = "https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool"
XGB_POOL_URL = "https://flash-api.xuangubao.cn/api/pool/detail"
BROWSER_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/138.0.0.0 Safari/537.36"
)
@dataclass
class WebRealtimeAggregator:
timeout: int = 8
retry_attempts: int = 3
retry_delay_seconds: float = 0.2
response_cache_ttl_seconds: int = 90
_sector_cache: ClassVar[dict[str, Any]] = {}
_sector_cache_lock: ClassVar[Lock] = Lock()
_response_cache: ClassVar[dict[str, dict[str, Any]]] = {}
_response_cache_lock: ClassVar[Lock] = Lock()
def health_snapshot(self, sector: str = "") -> dict[str, Any]:
started = time.perf_counter()
sources: dict[str, dict[str, Any]] = {}
indices: list[dict[str, Any]] = []
sector_payload: dict[str, Any] | None = None
indices, sources["eastmoney_indices"] = self._capture(self.eastmoney_indices)
if sector.strip():
sector_payload, sources["eastmoney_sector"] = self._capture(
lambda: self.eastmoney_sector(sector)
)
ths_observation, sources["ths_limit_pool"] = self._capture(self.ths_limit_pool)
xgb_observation, sources["xgb_limit_pool"] = self._capture(self.xgb_limit_pool)
index_times = [int(item.get("quote_time_epoch") or 0) for item in indices or []]
now = datetime.now().astimezone()
max_skew = 120 if now.hour >= 15 else 15
index_consistent = bool(index_times) and max(index_times) - min(index_times) <= max_skew
ready = (
bool(indices)
and len(indices) == 3
and index_consistent
and (not sector.strip() or bool(sector_payload))
)
return {
"ready": ready,
"isolated": True,
"generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"indices": indices or [],
"index_consistent": index_consistent,
"sector": sector_payload,
"sources": sources,
"observations": {
"ths_limit_pool": ths_observation,
"xgb_limit_pool": xgb_observation,
},
"policy": {
"integration": "heaven_realtime_fallback",
"max_index_time_skew_seconds": max_skew,
"notice": "聚合源仅作为盘中观势的实时指数与板块外显,主行情快照仍由Tushare维护。",
},
}
def eastmoney_indices(self) -> list[dict[str, Any]]:
try:
payload = self._get_json(
EASTMONEY_INDEX_URL,
{
"secids": "1.000001,0.399001,0.399006",
"fltt": "2",
"invt": "2",
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124",
},
referer="https://quote.eastmoney.com/",
)
except RealtimeAggregateError:
return self.tencent_indices()
cache_meta = payload.get("_aggregate_cache") or {}
rows = list((payload.get("data") or {}).get("diff") or [])
result = []
for row in rows:
code = str(row.get("f12") or "")
if code not in {"000001", "399001", "399006"}:
continue
epoch = int(_number(row.get("f124")))
result.append(
{
"code": code,
"name": row.get("f14") or code,
"price": _number(row.get("f2")),
"change": _number(row.get("f3")),
"change_amount": _number(row.get("f4")),
"open": _number(row.get("f17")),
"high": _number(row.get("f15")),
"low": _number(row.get("f16")),
"previous_close": _number(row.get("f18")),
"amount_billion": round(_number(row.get("f6")) / 100000000, 2),
"quote_time_epoch": epoch,
"quote_time": (
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
if epoch else ""
),
"source": (
"eastmoney_push2_cache" if cache_meta else "eastmoney_push2"
),
"cache_age_seconds": cache_meta.get("age_seconds", 0),
}
)
if len(result) != 3:
raise RealtimeAggregateError(f"Eastmoney returned {len(result)}/3 indices")
return result
def tencent_indices(self) -> list[dict[str, Any]]:
raw, cache_age = self._get_text(
TENCENT_INDEX_URL,
referer="https://gu.qq.com/",
encoding="gb18030",
)
result = []
for line in raw.splitlines():
if '="' not in line:
continue
fields = line.split('="', 1)[1].rsplit('";', 1)[0].split("~")
if len(fields) < 38:
continue
code = fields[2]
if code not in {"000001", "399001", "399006"}:
continue
try:
quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S").astimezone()
except ValueError as exc:
raise RealtimeAggregateError(
f"Tencent returned invalid quote time for {code}"
) from exc
result.append(
{
"code": code,
"name": fields[1] or code,
"price": _number(fields[3]),
"change": _number(fields[32]),
"change_amount": _number(fields[31]),
"open": _number(fields[5]),
"high": _number(fields[33]),
"low": _number(fields[34]),
"previous_close": _number(fields[4]),
"amount_billion": round(_number(fields[37]) / 10000, 2),
"quote_time_epoch": int(quote_time.timestamp()),
"quote_time": quote_time.isoformat(timespec="seconds"),
"source": "tencent_qt_cache" if cache_age else "tencent_qt",
"cache_age_seconds": cache_age,
}
)
if len(result) != 3:
raise RealtimeAggregateError(f"Tencent returned {len(result)}/3 indices")
return result
def eastmoney_sector(self, query: str) -> dict[str, Any]:
target = _normalize_sector(query)
candidates = self._eastmoney_sector_catalog()
matched = _match_sector(candidates, target)
if not matched:
raise RealtimeAggregateError(f"Eastmoney sector not found: {query}")
epoch = int(_number(matched.get("f124")))
return {
"code": matched.get("f12") or "",
"name": matched.get("f14") or query,
"price": _number(matched.get("f2")),
"change": _number(matched.get("f3")),
"change_amount": _number(matched.get("f4")),
"turnover_rate": _number(matched.get("f8")),
"up_count": int(_number(matched.get("f104"))),
"down_count": int(_number(matched.get("f105"))),
"leader": matched.get("f128") or "--",
"leader_code": matched.get("f140") or "",
"leading_pct": _number(matched.get("f136")),
"quote_time_epoch": epoch,
"quote_time": (
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
if epoch else ""
),
"source": "eastmoney_push2",
"match_query": query,
}
def _eastmoney_sector_catalog(self) -> list[dict[str, Any]]:
now = time.time()
with self._sector_cache_lock:
cached = self._sector_cache.get("eastmoney")
if cached and now - float(cached.get("created_at") or 0) < 600:
return list(cached.get("rows") or [])
def load_page(page: int) -> list[dict[str, Any]]:
payload = self._get_json(
EASTMONEY_SECTOR_URL,
{
"pn": str(page),
"pz": "100",
"po": "1",
"np": "1",
"fltt": "2",
"invt": "2",
"fid": "f3",
"fs": "m:90+t:2",
"fields": "f12,f14,f2,f3,f4,f8,f104,f105,f128,f136,f140,f124",
},
referer="https://quote.eastmoney.com/center/boardlist.html",
)
return list((payload.get("data") or {}).get("diff") or [])
with ThreadPoolExecutor(max_workers=5) as executor:
pages = list(executor.map(load_page, range(1, 6)))
rows = [row for page in pages for row in page]
if not rows:
raise RealtimeAggregateError("Eastmoney sector catalog is empty")
with self._sector_cache_lock:
self._sector_cache["eastmoney"] = {"created_at": now, "rows": rows}
return rows
def ths_limit_pool(self) -> dict[str, Any]:
payload = self._get_json(
THS_LIMIT_URL,
{"page": "1", "limit": "3", "field": "199112"},
referer="https://data.10jqka.com.cn/limit_up/",
)
data = payload.get("data") or payload
return {
"available": True,
"keys": sorted(str(key) for key in data.keys()) if isinstance(data, dict) else [],
"source": "ths_web_dataapi",
}
def xgb_limit_pool(self) -> dict[str, Any]:
payload = self._get_json(
XGB_POOL_URL,
{"pool_name": "limit_up"},
referer="https://xuangubao.cn/",
)
data = payload.get("data") or {}
rows = data if isinstance(data, list) else data.get("pool") or data.get("list") or []
return {
"available": True,
"count": len(rows) if isinstance(rows, list) else 0,
"source": "xuangubao_web_api",
}
def _capture(self, operation):
started = time.perf_counter()
try:
value = operation()
return value, {
"ok": True,
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"error": "",
}
except Exception as exc:
return None, {
"ok": False,
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"error": str(exc)[:500],
}
def _get_json(
self,
url: str,
params: dict[str, str],
referer: str,
) -> dict[str, Any]:
request_url = f"{url}?{urllib.parse.urlencode(params)}"
last_error: Exception | None = None
attempts = max(1, int(self.retry_attempts))
for attempt in range(attempts):
request = urllib.request.Request(
request_url,
headers={
"Accept": "application/json,text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
content_type = response.headers.get("Content-Type", "")
raw = response.read().decode("utf-8", errors="replace")
if "json" not in content_type.lower() and not raw.lstrip().startswith(("{", "[")):
raise RealtimeAggregateError(
f"non-JSON response: {raw[:120].strip()}"
)
payload = json.loads(raw)
if not isinstance(payload, dict):
raise RealtimeAggregateError("unexpected response shape")
if payload.get("rc") not in (None, 0):
raise RealtimeAggregateError(f"provider rc={payload.get('rc')}")
with self._response_cache_lock:
self._response_cache[request_url] = {
"created_at": time.time(),
"payload": copy.deepcopy(payload),
}
return payload
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
json.JSONDecodeError,
RealtimeAggregateError,
) as exc:
last_error = exc
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
time.sleep(self.retry_delay_seconds * (attempt + 1))
now = time.time()
with self._response_cache_lock:
cached = self._response_cache.get(request_url)
cache_age = now - float((cached or {}).get("created_at") or 0)
if cached and cache_age <= self.response_cache_ttl_seconds:
payload = copy.deepcopy(cached.get("payload") or {})
payload["_aggregate_cache"] = {"age_seconds": round(cache_age, 1)}
return payload
raise RealtimeAggregateError(f"request failed after {attempts} attempts: {last_error}") from last_error
def _get_text(
self,
request_url: str,
referer: str,
encoding: str = "utf-8",
) -> tuple[str, float]:
cache_key = f"text:{request_url}"
last_error: Exception | None = None
attempts = max(1, int(self.retry_attempts))
for attempt in range(attempts):
request = urllib.request.Request(
request_url,
headers={
"Accept": "text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
raw = response.read().decode(encoding, errors="replace")
if not raw.strip():
raise RealtimeAggregateError("empty text response")
with self._response_cache_lock:
self._response_cache[cache_key] = {
"created_at": time.time(),
"payload": raw,
}
return raw, 0
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
RealtimeAggregateError,
) as exc:
last_error = exc
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
time.sleep(self.retry_delay_seconds * (attempt + 1))
now = time.time()
with self._response_cache_lock:
cached = self._response_cache.get(cache_key)
cache_age = now - float((cached or {}).get("created_at") or 0)
if cached and cache_age <= self.response_cache_ttl_seconds:
return str(cached.get("payload") or ""), round(cache_age, 1)
raise RealtimeAggregateError(
f"text request failed after {attempts} attempts: {last_error}"
) from last_error
def _normalize_sector(value: Any) -> str:
text = str(value or "").strip().replace(" ", "")
for suffix in ("板块", "概念", "行业", "", "", "(A股)", "A股)"):
text = text.replace(suffix, "")
aliases = {"元器件": "元件", "电子元器件": "元件"}
return aliases.get(text, text)
def _match_sector(rows: list[dict[str, Any]], target: str) -> dict[str, Any] | None:
exact = [row for row in rows if _normalize_sector(row.get("f14")) == target]
if exact:
return min(exact, key=lambda row: len(str(row.get("f14") or "")))
fuzzy = [
row for row in rows
if target and (
target in _normalize_sector(row.get("f14"))
or _normalize_sector(row.get("f14")) in target
)
]
return min(fuzzy, key=lambda row: len(_normalize_sector(row.get("f14")))) if fuzzy else None
def _number(value: Any, default: float = 0.0) -> float:
try:
return float(value)
except (TypeError, ValueError):
return default
sys.modules[__name__] = _implementation
+1 -1
View File
@@ -10,7 +10,7 @@ from typing import Any
from advanced_strategies import ADVANCED_CURATED_STRATEGIES
from database import ReviewDatabase
from sentiment_engine import build_sentiment_history, latest_contiguous_history
from backend.features.sentiment.engine import build_sentiment_history, latest_contiguous_history
from tushare_client import TushareClient, TushareError
+4 -493
View File
@@ -1,496 +1,7 @@
from __future__ import annotations
"""Compatibility alias for the canonical sentiment engine implementation."""
from copy import deepcopy
from statistics import mean, median
from typing import Any
import sys
from backend.features.sentiment import engine as _implementation
COMPONENT_WEIGHTS = {
"breadth": 20,
"limit_ecology": 25,
"profit_effect": 30,
"ladder_structure": 15,
"liquidity": 10,
}
SENTIMENT_ENGINE_VERSION = 2
def _number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if number == number else default
except (TypeError, ValueError):
return default
def _clamp(value: float, lower: float = 0.0, upper: float = 100.0) -> float:
return min(upper, max(lower, value))
def _linear(value: float, low: float, high: float) -> float:
if high <= low:
return 50.0
return _clamp((value - low) / (high - low) * 100)
def _percentile(value: float, history: list[float]) -> float:
if not history:
return 50.0
below = sum(item < value for item in history)
equal = sum(item == value for item in history)
return _clamp((below + equal * 0.5) / len(history) * 100)
def _adaptive_score(value: float, fixed: float, history: list[float]) -> float:
if len(history) < 20:
return fixed
return fixed * 0.25 + _percentile(value, history[-250:]) * 0.75
def _trade_date(payload: dict[str, Any]) -> str:
meta = payload.get("meta") or {}
return str(meta.get("trade_date") or payload.get("_snapshot_date") or "").replace("-", "")
def _deduplicate_snapshots(snapshots: list[dict[str, Any]]) -> list[dict[str, Any]]:
by_trade_date: dict[str, dict[str, Any]] = {}
for payload in snapshots:
trade_date = _trade_date(payload)
if trade_date:
by_trade_date[trade_date] = payload
return [by_trade_date[key] for key in sorted(by_trade_date)]
def _snapshot_stats(payload: dict[str, Any]) -> dict[str, Any]:
overview = payload.get("overview") or {}
meta = payload.get("meta") or {}
limits = list(payload.get("limits") or [])
broken = list(payload.get("broken") or [])
down_limits = list(payload.get("down_limits") or [])
yesterday = list(payload.get("yesterday_limits") or [])
limit_up = len(limits) if limits else int(_number(overview.get("limit_up_count")))
broken_count = len(broken) if broken else int(_number(overview.get("broken_count")))
limit_down = len(down_limits) if down_limits else int(_number(overview.get("limit_down_count")))
streaks = [max(1, int(_number(row.get("streak"), 1))) for row in limits]
first_board = sum(streak == 1 for streak in streaks)
second_board = sum(streak == 2 for streak in streaks)
three_plus = sum(streak >= 3 for streak in streaks)
max_height = max(streaks, default=0)
present_levels = set(streaks)
ladder_completeness = (
sum(level in present_levels for level in range(1, max_height + 1)) / max_height * 100
if max_height else 0.0
)
up_count = int(_number(overview.get("up_count")))
down_count = int(_number(overview.get("down_count")))
flat_count = int(_number(overview.get("flat_count")))
active_count = up_count + down_count
breadth_ratio = up_count / max(active_count, 1) * 100
seal_rate = _number(overview.get("seal_rate"))
if not seal_rate and limit_up + broken_count:
seal_rate = limit_up / (limit_up + broken_count) * 100
previous_limit_count = len(yesterday)
previous_positive_count = sum(_number(row.get("current_change")) > 0 for row in yesterday)
previous_positive_rate = previous_positive_count / max(previous_limit_count, 1) * 100
advanced_count = sum(row.get("outcome") == "晋级" for row in yesterday)
advance_rate = advanced_count / max(previous_limit_count, 1) * 100
average_previous_change = (
mean(_number(row.get("current_change")) for row in yesterday) if yesterday else 0.0
)
median_previous_change = (
median(_number(row.get("current_change")) for row in yesterday) if yesterday else 0.0
)
severe_loss_count = sum(_number(row.get("current_change")) <= -5 for row in yesterday)
severe_loss_rate = severe_loss_count / max(previous_limit_count, 1) * 100
previous_down_count = sum(row.get("outcome") == "跌停" for row in yesterday)
high_previous = [row for row in yesterday if int(_number(row.get("prior_streak"), 1)) >= 2]
high_positive_rate = (
sum(_number(row.get("current_change")) > 0 for row in high_previous)
/ max(len(high_previous), 1)
* 100
)
amount_billion = _number(overview.get("amount_billion"))
limit_amount_billion = sum(_number(row.get("amount_billion")) for row in limits)
return {
"trade_date": _trade_date(payload),
"previous_trade_date": str(meta.get("previous_trade_date") or "").replace("-", ""),
"up_count": up_count,
"down_count": down_count,
"flat_count": flat_count,
"breadth_ratio": round(breadth_ratio, 1),
"limit_up_count": limit_up,
"first_board_count": first_board,
"second_board_count": second_board,
"three_plus_count": three_plus,
"max_height": max_height,
"ladder_completeness": round(ladder_completeness, 1),
"broken_count": broken_count,
"limit_down_count": limit_down,
"seal_rate": round(seal_rate, 1),
"previous_limit_count": previous_limit_count,
"previous_positive_count": previous_positive_count,
"previous_positive_rate": round(previous_positive_rate, 1),
"advance_rate": round(advance_rate, 1),
"average_previous_change": round(average_previous_change, 2),
"median_previous_change": round(median_previous_change, 2),
"severe_loss_count": severe_loss_count,
"severe_loss_rate": round(severe_loss_rate, 1),
"previous_down_count": previous_down_count,
"high_positive_rate": round(high_positive_rate, 1),
"amount_billion": round(amount_billion, 1),
"limit_amount_billion": round(limit_amount_billion, 2),
}
def _sentiment_label(score: float) -> str:
if score >= 80:
return "情绪高涨"
if score >= 60:
return "情绪偏强"
if score >= 40:
return "情绪中性"
if score >= 20:
return "情绪偏弱"
return "情绪冰点"
def _phase_signal(score: float, momentum: float, profit_score: float) -> str:
if score < 25:
return "修复" if momentum > 3 else "冰点"
if score < 45:
return "修复" if momentum > 3 else "退潮"
if score >= 80:
return "高潮" if momentum >= -2 and profit_score >= 60 else "分化"
if score >= 65:
return "分化" if momentum < -3 or profit_score < 50 else "发酵"
if momentum < -5:
return "退潮"
return "发酵" if momentum >= 0 and profit_score >= 45 else "分化"
def _confirmed_phase(
previous: dict[str, Any] | None,
score: float,
day_change: float,
systemic_health: float,
profit_score: float,
ecology_score: float,
phase_signal: str,
extreme_ice: bool,
fermentation_signal_count: int,
) -> tuple[str, str]:
if previous is None:
return phase_signal, "首个连续交易日,采用原始阶段信号"
previous_phase = str(previous.get("phase") or phase_signal)
if extreme_ice:
return "冰点", "市场宽度与跌停数量触发极端冰点"
recovery = day_change >= 6 and score >= 25 and systemic_health >= 24
fermentation_confirmed = fermentation_signal_count >= 2
climax_ready = (
score >= 80
and profit_score >= 60
and systemic_health >= 60
and ecology_score >= 70
)
if previous_phase == "冰点":
return ("修复", "冰点后首次有效回升") if recovery else ("冰点", "冰点尚未形成有效修复")
if previous_phase == "退潮":
if score < 25:
return "冰点", "退潮继续下探至冰点区间"
return ("修复", "退潮后出现有效回升") if recovery else ("退潮", "退潮尚未形成有效修复")
if previous_phase == "修复":
if score < 25:
return "冰点", "修复失败并重新跌入冰点区间"
if day_change <= -6 and score < 45:
return "退潮", "修复失败且温度显著回落"
if fermentation_confirmed:
return "发酵", "发酵条件连续两个交易日成立"
return "修复", "修复延续,等待发酵确认"
if previous_phase == "发酵":
if score < 25:
return "冰点", "发酵阶段出现极端情绪坍塌"
if score < 45 and (day_change < 0 or systemic_health < 35):
return "退潮", "发酵阶段温度与系统健康度同步转弱"
if climax_ready:
return "高潮", "温度、赚钱效应与涨停生态共同达到高潮条件"
if phase_signal in {"分化", "退潮"} or day_change <= -6:
return "分化", "发酵阶段出现降温或赚钱效应弱化"
return "发酵", "发酵状态延续"
if previous_phase == "高潮":
if score < 25:
return "冰点", "高潮后出现极端情绪坍塌"
if climax_ready:
return "高潮", "高潮条件继续成立"
if score < 45 or systemic_health < 30:
return "退潮", "高潮后风险快速释放"
return "分化", "高潮条件消退,进入分化"
if previous_phase == "分化":
if score < 25:
return "冰点", "分化继续恶化至冰点区间"
if score < 45 or systemic_health < 30:
return "退潮", "分化后温度或系统健康度继续下降"
if fermentation_confirmed:
return "发酵", "分化转强条件连续两个交易日成立"
return "分化", "分化延续,等待方向确认"
return phase_signal, "采用原始阶段信号"
def build_sentiment_history(snapshots: list[dict[str, Any]]) -> list[dict[str, Any]]:
payloads = _deduplicate_snapshots(snapshots)
raw_rows = [_snapshot_stats(payload) for payload in payloads]
results: list[dict[str, Any]] = []
for index, stats in enumerate(raw_rows):
previous = raw_rows[:index]
limit_history = [float(row["limit_up_count"]) for row in previous]
down_limit_history = [float(row["limit_down_count"]) for row in previous]
height_history = [float(row["max_height"]) for row in previous]
three_plus_history = [float(row["three_plus_count"]) for row in previous]
amount_history = [float(row["amount_billion"]) for row in previous[-20:] if row["amount_billion"]]
breadth_score = _clamp(float(stats["breadth_ratio"]))
limit_strength = _adaptive_score(
float(stats["limit_up_count"]),
_linear(float(stats["limit_up_count"]), 10, 100),
limit_history,
)
down_relief = 100 - _adaptive_score(
float(stats["limit_down_count"]),
_linear(float(stats["limit_down_count"]), 0, 50),
down_limit_history,
)
seal_quality = _linear(float(stats["seal_rate"]), 35, 90)
systemic_health = breadth_score * 0.60 + down_relief * 0.40
systemic_gate = 1.0 if systemic_health >= 35 else 0.35 + systemic_health / 35 * 0.65
ecology_base_score = limit_strength * 0.35 + seal_quality * 0.35 + down_relief * 0.30
# Systemic risk is applied once to the final temperature. Reapplying it here
# would count market breadth and limit-down pressure twice.
limit_ecology_score = ecology_base_score
if stats["previous_limit_count"]:
positive_score = float(stats["previous_positive_rate"])
average_change_score = _clamp(50 + float(stats["average_previous_change"]) * 6)
median_change_score = _clamp(50 + float(stats["median_previous_change"]) * 7)
advance_score = _clamp(float(stats["advance_rate"]) * 2.5)
severe_loss_safety = _clamp(100 - float(stats["severe_loss_rate"]) * 3)
down_safety = _clamp(100 - float(stats["previous_down_count"]) / stats["previous_limit_count"] * 700)
tail_safety_score = severe_loss_safety * 0.70 + down_safety * 0.30
profit_effect_score = (
positive_score * 0.30
+ median_change_score * 0.25
+ average_change_score * 0.10
+ advance_score * 0.20
+ tail_safety_score * 0.15
)
else:
profit_effect_score = 50.0
max_height_score = _adaptive_score(
float(stats["max_height"]),
_linear(float(stats["max_height"]), 1, 7),
height_history,
)
continuation_rate = (
(float(stats["second_board_count"]) + float(stats["three_plus_count"]))
/ max(float(stats["limit_up_count"]), 1)
* 100
)
three_plus_density = float(stats["three_plus_count"]) / max(float(stats["limit_up_count"]), 1) * 100
three_plus_score = _adaptive_score(
float(stats["three_plus_count"]),
_clamp(three_plus_density * 5),
three_plus_history,
)
ladder_structure_score = (
max_height_score * 0.30
+ _clamp(continuation_rate * 3) * 0.25
+ three_plus_score * 0.25
+ float(stats["ladder_completeness"]) * 0.20
)
amount_baseline = mean(amount_history) if amount_history else float(stats["amount_billion"] or 1)
amount_ratio = float(stats["amount_billion"]) / max(amount_baseline, 1)
amount_score = _clamp(50 + (amount_ratio - 1) * 100)
limit_amount_share = float(stats["limit_amount_billion"]) / max(float(stats["amount_billion"]), 1) * 100
liquidity_score = amount_score * 0.70 + _clamp(limit_amount_share * 20) * 0.30
component_scores = {
"breadth": breadth_score,
"limit_ecology": limit_ecology_score,
"profit_effect": profit_effect_score,
"ladder_structure": ladder_structure_score,
"liquidity": liquidity_score,
}
raw_score = sum(component_scores[key] * weight / 100 for key, weight in COMPONENT_WEIGHTS.items())
score = round(
raw_score * systemic_gate
)
extreme_ice = float(stats["breadth_ratio"]) <= 15 and float(stats["limit_down_count"]) >= 100
if extreme_ice:
score = min(score, 15)
elif float(stats["breadth_ratio"]) <= 25 and float(stats["limit_down_count"]) >= 50:
score = min(score, 24)
previous_scores: list[float] = []
expected_date = str(stats.get("previous_trade_date") or "")
for prior_result in reversed(results):
if not expected_date or str(prior_result.get("trade_date") or "") != expected_date:
break
previous_scores.append(float(prior_result["score"]))
expected_date = str(prior_result.get("previous_trade_date") or "")
if len(previous_scores) == 3:
break
momentum = score - mean(previous_scores) if previous_scores else 0.0
direction = "升温" if momentum > 3 else "降温" if momentum < -3 else "持平"
normalization = "历史百分位" if len(previous) >= 20 else "固定锚点"
previous_result = (
results[-1]
if results and str(stats.get("previous_trade_date") or "") == str(results[-1].get("trade_date") or "")
else None
)
day_change = score - float(previous_result["score"]) if previous_result else 0.0
ema_score = round(
score if not previous_result
else score * 0.5 + float(previous_result.get("ema_score", previous_result["score"])) * 0.5,
1,
)
phase_signal = _phase_signal(score, momentum, profit_effect_score)
fermentation_ready = (
phase_signal == "发酵"
and score >= 45
and profit_effect_score >= 45
and systemic_health >= 35
and not extreme_ice
)
previous_fermentation_count = int(previous_result.get("fermentation_signal_count") or 0) if previous_result else 0
fermentation_signal_count = previous_fermentation_count + 1 if fermentation_ready else 0
phase, transition_reason = _confirmed_phase(
previous_result,
score,
day_change,
systemic_health,
profit_effect_score,
limit_ecology_score,
phase_signal,
extreme_ice,
fermentation_signal_count,
)
previous_phase = str(previous_result.get("phase") or "") if previous_result else ""
if phase not in {"修复", "分化"}:
fermentation_signal_count = 0
elif phase == "分化" and previous_phase != "分化":
fermentation_signal_count = 0
components = {
"breadth": {
"label": "市场宽度",
"score": round(breadth_score, 1),
"weight": COMPONENT_WEIGHTS["breadth"],
"summary": f"上涨占比 {stats['breadth_ratio']:.1f}%",
},
"limit_ecology": {
"label": "涨停生态",
"score": round(limit_ecology_score, 1),
"weight": COMPONENT_WEIGHTS["limit_ecology"],
"summary": (
f"涨停 {stats['limit_up_count']} · 跌停 {stats['limit_down_count']} · "
f"封板 {stats['seal_rate']:.1f}%"
),
},
"profit_effect": {
"label": "赚钱效应",
"score": round(profit_effect_score, 1),
"weight": COMPONENT_WEIGHTS["profit_effect"],
"summary": (
f"昨涨停红盘 {stats['previous_positive_rate']:.1f}% · "
f"中位 {stats['median_previous_change']:+.2f}% · "
f"重亏 {stats['severe_loss_rate']:.1f}%"
if stats["previous_limit_count"] else "缺少前一交易日样本"
),
},
"ladder_structure": {
"label": "连板结构",
"score": round(ladder_structure_score, 1),
"weight": COMPONENT_WEIGHTS["ladder_structure"],
"summary": f"最高 {stats['max_height']} 板 · 三板以上 {stats['three_plus_count']}",
},
"liquidity": {
"label": "成交活跃度",
"score": round(liquidity_score, 1),
"weight": COMPONENT_WEIGHTS["liquidity"],
"summary": f"成交 {stats['amount_billion']:.1f} 亿 · 均值比 {amount_ratio:.2f}",
},
}
results.append(
{
**stats,
"score": score,
"ema_score": ema_score,
"label": _sentiment_label(score),
"phase": phase,
"phase_signal": phase_signal,
"transition_reason": transition_reason,
"fermentation_signal_count": fermentation_signal_count,
"day_change": round(day_change, 1),
"direction": direction,
"momentum": round(momentum, 1),
"normalization": "250日历史百分位" if len(previous) >= 20 else normalization,
"history_days": len(previous) + 1,
"systemic_health": round(systemic_health, 1),
"risk_multiplier": round(systemic_gate, 3),
"components": components,
}
)
return results
def latest_contiguous_history(series: list[dict[str, Any]]) -> list[dict[str, Any]]:
if not series:
return []
contiguous = [series[-1]]
for row in reversed(series[:-1]):
expected_previous = str(contiguous[0].get("previous_trade_date") or "")
if not expected_previous or expected_previous != str(row.get("trade_date") or ""):
break
contiguous.insert(0, row)
return contiguous
def apply_sentiment_to_dashboard(
dashboard: dict[str, Any],
historical_snapshots: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
result = deepcopy(dashboard)
history = list(historical_snapshots or [])
history.append(result)
series = build_sentiment_history(history)
target_date = _trade_date(result)
sentiment = next((row for row in reversed(series) if row["trade_date"] == target_date), None)
if not sentiment:
return result
overview = dict(result.get("overview") or {})
overview.update(
{
"sentiment_score": sentiment["score"],
"sentiment_trend_score": sentiment["ema_score"],
"sentiment_label": sentiment["label"],
"sentiment_phase": sentiment["phase"],
"sentiment_direction": sentiment["direction"],
"sentiment_components": sentiment["components"],
"sentiment_engine_version": SENTIMENT_ENGINE_VERSION,
}
)
result["overview"] = overview
return result
sys.modules[__name__] = _implementation
+5 -1
View File
@@ -48,7 +48,11 @@ class DataGatewayTests(unittest.TestCase):
from pathlib import Path
source = (
Path(__file__).resolve().parents[1] / "backend" / "application.py"
Path(__file__).resolve().parents[1]
/ "backend"
/ "features"
/ "market"
/ "service.py"
).read_text(encoding="utf-8")
self.assertEqual(source.count("TushareClient(self.token)"), 1)
self.assertIn("return gateway.tushare()", source)
@@ -0,0 +1,92 @@
from __future__ import annotations
import ast
import hashlib
import unittest
from pathlib import Path
APP_ROOT = Path(__file__).resolve().parents[1]
ORIGINAL_ROOT = APP_ROOT.parent
ROTATION_METHODS = {
"rotation_history",
"rotation_sector_members",
}
LADDER_ROTATION_BUILDERS = {
"_build_ladders",
"_build_sector_rotation",
}
def class_methods(path: Path, class_name: str) -> dict[str, str]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
owner = next(
node
for node in tree.body
if isinstance(node, ast.ClassDef) and node.name == class_name
)
return {
node.name: ast.dump(node, include_attributes=False)
for node in owner.body
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
}
def top_level_functions(path: Path) -> dict[str, str]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
return {
node.name: ast.dump(node, include_attributes=False)
for node in tree.body
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
and node.name in LADDER_ROTATION_BUILDERS
}
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
class LadderRotationSliceSourceEquivalenceTests(unittest.TestCase):
def test_rotation_service_methods_are_exact_original_ast(self) -> None:
original = class_methods(ORIGINAL_ROOT / "server.py", "DashboardService")
migrated = class_methods(
APP_ROOT / "backend" / "features" / "rotation" / "service.py",
"RotationServiceMixin",
)
self.assertEqual(set(migrated), ROTATION_METHODS)
for name in sorted(ROTATION_METHODS):
self.assertEqual(migrated[name], original[name], name)
def test_dashboard_service_no_longer_duplicates_rotation_methods(self) -> None:
remaining = class_methods(
APP_ROOT / "backend" / "application.py", "DashboardService"
)
self.assertTrue(ROTATION_METHODS.isdisjoint(remaining))
def test_ladder_and_rotation_builders_are_exact_original_ast(self) -> None:
self.assertEqual(
top_level_functions(ORIGINAL_ROOT / "tushare_client.py"),
top_level_functions(
APP_ROOT / "backend" / "data" / "providers" / "tushare_client.py"
),
)
def test_api_and_frontend_assets_are_unchanged(self) -> None:
for relative in (
"config/api.config.json",
"static/index.html",
"static/app.js",
"static/styles.css",
"static/pages/ladder/page.js",
"static/pages/rotation/page.js",
):
self.assertEqual(
sha256(APP_ROOT / relative),
sha256(ORIGINAL_ROOT / relative),
relative,
)
if __name__ == "__main__":
unittest.main()
+162
View File
@@ -0,0 +1,162 @@
from __future__ import annotations
import ast
import hashlib
import unittest
from pathlib import Path
import chart_data_provider
import ifind_client
import realtime_aggregator
import tushare_client
from backend.data import realtime
from backend.data.providers import ifind_client as canonical_ifind
from backend.data.providers import tushare_client as canonical_tushare
from backend.features.market import charts
APP_ROOT = Path(__file__).resolve().parents[1]
ORIGINAL_ROOT = APP_ROOT.parent
MARKET_METHODS = {
"_tushare_client",
"get_dashboard",
"_dashboard_sentiment_ready",
"_display_compact_date",
"_carry_dashboard",
"_realtime_snapshot_due",
"sync_dashboard",
"realtime_aggregate_health",
"_search_market_directory",
"_search_match_score",
"search_entities",
"get_search_detail",
"get_intraday_chart",
"_ths_search_detail",
"_index_search_detail",
"get_stock_detail",
"_stock_detail_bar_date",
"_stock_detail_cache_needs_refresh",
"_prepare_stock_detail",
"_sanitize_stock_detail_prices",
"_valid_realtime_stock_quote",
"_ifind_realtime_stock_quote",
"_merge_realtime_stock_detail",
"get_stock_preview",
"backfill",
"_stock_identity",
"_enrich_stock_detail",
"_with_storage",
"_record_count",
}
MARKET_REPOSITORY_METHODS = {
"get_snapshot",
"get_latest_real_snapshot",
"save_snapshot",
"get_data_snapshot",
"get_latest_data_snapshot",
"save_data_snapshot",
"search_stock_master",
"list_snapshot_payloads",
"start_sync",
"finish_sync",
"status",
}
def class_methods(path: Path, class_name: str) -> dict[str, str]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
owner = next(
node
for node in tree.body
if isinstance(node, ast.ClassDef) and node.name == class_name
)
return {
node.name: ast.dump(node, include_attributes=False)
for node in owner.body
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
}
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def top_level_definitions(path: Path) -> dict[str, str]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
return {
node.name: ast.dump(node, include_attributes=False)
for node in tree.body
if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef))
}
class MarketSliceSourceEquivalenceTests(unittest.TestCase):
def test_market_service_methods_are_exact_original_ast(self) -> None:
original = class_methods(ORIGINAL_ROOT / "server.py", "DashboardService")
migrated = class_methods(
APP_ROOT / "backend" / "features" / "market" / "service.py",
"MarketServiceMixin",
)
self.assertEqual(set(migrated), MARKET_METHODS)
for name in sorted(MARKET_METHODS):
self.assertEqual(migrated[name], original[name], name)
def test_market_repository_methods_are_exact_original_ast(self) -> None:
original = class_methods(ORIGINAL_ROOT / "database.py", "ReviewDatabase")
migrated = class_methods(
APP_ROOT / "backend" / "features" / "market" / "repository.py",
"MarketRepositoryMixin",
)
self.assertEqual(set(migrated), MARKET_REPOSITORY_METHODS)
for name in sorted(MARKET_REPOSITORY_METHODS):
self.assertEqual(migrated[name], original[name], name)
def test_original_classes_no_longer_duplicate_moved_methods(self) -> None:
remaining_service = class_methods(APP_ROOT / "backend" / "application.py", "DashboardService")
remaining_database = class_methods(APP_ROOT / "database.py", "ReviewDatabase")
self.assertTrue(MARKET_METHODS.isdisjoint(remaining_service))
self.assertTrue(MARKET_REPOSITORY_METHODS.isdisjoint(remaining_database))
def test_provider_compatibility_modules_are_canonical_aliases(self) -> None:
self.assertIs(tushare_client.TushareClient, canonical_tushare.TushareClient)
self.assertIs(ifind_client.IfindHttpClient, canonical_ifind.IfindHttpClient)
self.assertIs(realtime_aggregator.WebRealtimeAggregator, realtime.WebRealtimeAggregator)
self.assertIs(chart_data_provider.MarketChartClient, charts.MarketChartClient)
def test_provider_logic_is_the_original_implementation(self) -> None:
exact_moves = (
("ifind_client.py", "backend/data/providers/ifind_client.py"),
("realtime_aggregator.py", "backend/data/realtime.py"),
)
for original, migrated in exact_moves:
self.assertEqual(sha256(ORIGINAL_ROOT / original), sha256(APP_ROOT / migrated))
self.assertEqual(
top_level_definitions(ORIGINAL_ROOT / "tushare_client.py"),
top_level_definitions(APP_ROOT / "backend/data/providers/tushare_client.py"),
)
self.assertEqual(
top_level_definitions(ORIGINAL_ROOT / "chart_data_provider.py"),
top_level_definitions(APP_ROOT / "backend/features/market/charts.py"),
)
def test_unchanged_frontend_assets_match_the_original(self) -> None:
for relative in (
"index.html",
"app.js",
"styles.css",
"renovation.css",
"redesign-v2.css",
"theme.css",
"wentian-v2.css",
):
self.assertEqual(
sha256(APP_ROOT / "static" / relative),
sha256(ORIGINAL_ROOT / "static" / relative),
relative,
)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,114 @@
from __future__ import annotations
import ast
import hashlib
import unittest
from pathlib import Path
import sentiment_engine
from backend.features.sentiment import engine as canonical_engine
APP_ROOT = Path(__file__).resolve().parents[1]
ORIGINAL_ROOT = APP_ROOT.parent
SENTIMENT_METHODS = {
"_enrich_dashboard_sentiment",
"sentiment_history",
}
POOL_METHODS = {
"save_reason",
"_apply_reason_overrides",
"_schedule_ifind_event_enrichment",
"_refresh_ifind_event_enrichment",
"_normalize_ifind_event_time",
"_merge_ifind_event_enrichment",
}
POOL_REPOSITORY_METHODS = {
"save_reason_override",
"reason_overrides",
}
def class_methods(path: Path, class_name: str) -> dict[str, str]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
owner = next(
node
for node in tree.body
if isinstance(node, ast.ClassDef) and node.name == class_name
)
return {
node.name: ast.dump(node, include_attributes=False)
for node in owner.body
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
}
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
class SentimentPoolSliceSourceEquivalenceTests(unittest.TestCase):
def test_sentiment_service_methods_are_exact_original_ast(self) -> None:
original = class_methods(ORIGINAL_ROOT / "server.py", "DashboardService")
migrated = class_methods(
APP_ROOT / "backend" / "features" / "sentiment" / "service.py",
"SentimentServiceMixin",
)
self.assertEqual(set(migrated), SENTIMENT_METHODS)
for name in sorted(SENTIMENT_METHODS):
self.assertEqual(migrated[name], original[name], name)
def test_pool_service_methods_are_exact_original_ast(self) -> None:
original = class_methods(ORIGINAL_ROOT / "server.py", "DashboardService")
migrated = class_methods(
APP_ROOT / "backend" / "features" / "pools" / "service.py",
"PoolServiceMixin",
)
self.assertEqual(set(migrated), POOL_METHODS)
for name in sorted(POOL_METHODS):
self.assertEqual(migrated[name], original[name], name)
def test_pool_repository_methods_are_exact_original_ast(self) -> None:
original = class_methods(ORIGINAL_ROOT / "database.py", "ReviewDatabase")
migrated = class_methods(
APP_ROOT / "backend" / "features" / "pools" / "repository.py",
"PoolRepositoryMixin",
)
self.assertEqual(set(migrated), POOL_REPOSITORY_METHODS)
for name in sorted(POOL_REPOSITORY_METHODS):
self.assertEqual(migrated[name], original[name], name)
def test_original_classes_no_longer_duplicate_moved_methods(self) -> None:
remaining_service = class_methods(
APP_ROOT / "backend" / "application.py", "DashboardService"
)
remaining_database = class_methods(APP_ROOT / "database.py", "ReviewDatabase")
self.assertTrue((SENTIMENT_METHODS | POOL_METHODS).isdisjoint(remaining_service))
self.assertTrue(POOL_REPOSITORY_METHODS.isdisjoint(remaining_database))
def test_sentiment_engine_is_exact_original_with_legacy_alias(self) -> None:
self.assertEqual(
sha256(ORIGINAL_ROOT / "sentiment_engine.py"),
sha256(APP_ROOT / "backend" / "features" / "sentiment" / "engine.py"),
)
self.assertIs(sentiment_engine, canonical_engine)
def test_api_and_frontend_assets_are_unchanged(self) -> None:
for relative in (
"config/api.config.json",
"static/index.html",
"static/app.js",
"static/styles.css",
"static/pages/sentiment/page.js",
"static/pages/pools/page.js",
):
self.assertEqual(
sha256(APP_ROOT / relative),
sha256(ORIGINAL_ROOT / relative),
relative,
)
if __name__ == "__main__":
unittest.main()
+6 -6
View File
@@ -90,8 +90,8 @@ class StockDetailRealtimeTests(unittest.TestCase):
"moneyflow": {},
}
with patch("backend.application.datetime", FixedMarketDatetime), patch(
"backend.application.TushareClient", RealtimeClientStub
with patch("backend.features.market.service.datetime", FixedMarketDatetime), patch(
"backend.features.market.service.TushareClient", RealtimeClientStub
):
result = self.service._prepare_stock_detail(cached, "002141", today)
@@ -112,8 +112,8 @@ class StockDetailRealtimeTests(unittest.TestCase):
"stock": {"code": "002141", "price": 10, "change": 1.2},
"prices": [{"trade_date": historical, "close": 10, "change": 1.2}],
}
with patch("backend.application.datetime", FixedMarketDatetime), patch(
"backend.application.TushareClient", RealtimeClientStub
with patch("backend.features.market.service.datetime", FixedMarketDatetime), patch(
"backend.features.market.service.TushareClient", RealtimeClientStub
):
result = self.service._prepare_stock_detail(payload, "002141", historical)
@@ -151,8 +151,8 @@ class StockDetailRealtimeTests(unittest.TestCase):
},
],
}
with patch("backend.application.datetime", FixedPreopenDatetime), patch(
"backend.application.TushareClient", RealtimeClientStub
with patch("backend.features.market.service.datetime", FixedPreopenDatetime), patch(
"backend.features.market.service.TushareClient", RealtimeClientStub
):
result = self.service._prepare_stock_detail(payload, "002141", today)
+76
View File
@@ -0,0 +1,76 @@
from __future__ import annotations
import argparse
import ast
from pathlib import Path
MARKER = " # PRESERVATION_METHODS\n"
def method_span(node: ast.FunctionDef | ast.AsyncFunctionDef) -> tuple[int, int]:
start = min((decorator.lineno for decorator in node.decorator_list), default=node.lineno)
if node.end_lineno is None:
raise ValueError(f"Missing end position for {node.name}")
return start - 1, node.end_lineno
def move_methods(
source_path: Path,
class_name: str,
target_path: Path,
method_names: list[str],
) -> None:
source = source_path.read_text(encoding="utf-8")
tree = ast.parse(source, filename=str(source_path))
owner = next(
(
node
for node in tree.body
if isinstance(node, ast.ClassDef) and node.name == class_name
),
None,
)
if owner is None:
raise ValueError(f"Class not found: {class_name}")
methods = {
node.name: node
for node in owner.body
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
}
missing = [name for name in method_names if name not in methods]
if missing:
raise ValueError(f"Methods not found in {class_name}: {', '.join(missing)}")
lines = source.splitlines(keepends=True)
ordered = sorted((methods[name] for name in method_names), key=lambda node: node.lineno)
blocks = ["".join(lines[start:end]).rstrip() for start, end in map(method_span, ordered)]
for start, end in sorted(map(method_span, ordered), reverse=True):
del lines[start:end]
while start < len(lines) - 1 and lines[start] == "\n" and lines[start + 1] == "\n":
del lines[start]
target = target_path.read_text(encoding="utf-8")
if target.count(MARKER) != 1:
raise ValueError(f"Target must contain exactly one method marker: {target_path}")
target = target.replace(MARKER, "\n\n".join(blocks) + "\n")
source_path.write_text("".join(lines), encoding="utf-8")
target_path.write_text(target, encoding="utf-8")
def main() -> None:
parser = argparse.ArgumentParser(description="Mechanically move class methods between modules")
parser.add_argument("--source", type=Path, required=True)
parser.add_argument("--class-name", required=True)
parser.add_argument("--target", type=Path, required=True)
parser.add_argument("methods", nargs="+")
args = parser.parse_args()
move_methods(args.source, args.class_name, args.target, args.methods)
if __name__ == "__main__":
main()
+4 -2172
View File
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,61 @@
# 切片 02:公共行情、搜索、详情、图表与数据网关
> 基线:`4002f09`(切片 01
> 回档标签:`xiaobai-preservation-slice-02-20260731`
> 结论:源码、API、数据库、真实页面和浏览器回归通过;最终视觉仍等待全站人工验收
## 1. 原实现归位
本切片从原版副本机械移动公共行情纵向链路,没有从 `next/` 取用代码,也没有修改计算逻辑。
| 原位置 | 新的唯一实现位置 | 原位置兼容 |
|---|---|---|
| `app/backend/application.py` 的 30 个总览、搜索、详情、图表方法 | `app/backend/features/market/service.py` | `DashboardService` 继承 `MarketServiceMixin` |
| `app/database.py` 的 11 个行情快照、搜索目录、同步记录方法 | `app/backend/features/market/repository.py` | `ReviewDatabase` 继承 `MarketRepositoryMixin` |
| `app/tushare_client.py` | `app/backend/data/providers/tushare_client.py` | 根模块为同一模块对象的兼容别名 |
| `app/ifind_client.py` | `app/backend/data/providers/ifind_client.py` | 根模块为同一模块对象的兼容别名 |
| `app/realtime_aggregator.py` | `app/backend/data/realtime.py` | 根模块为同一模块对象的兼容别名 |
| `app/chart_data_provider.py` | `app/backend/features/market/charts.py` | 根模块为同一模块对象的兼容别名 |
`app/tools/move_class_methods.py` 使用 Python AST 确定方法及装饰器的源码边界,只移动原文本片段。
该工具会在缺失方法、目标标记不唯一或源码无法解析时停止,供后续切片继续复用。
## 2. 等价证据
- `test_preservation_slice_market.py` 对 30 个业务方法和 11 个 Repository 方法逐项执行无位置信息
AST 比较,全部与根目录原版 `server.py``database.py` 完全相同。
- Tushare、iFinD 和实时观察器文件与原版 SHA-256 完全相同;图表模块全部类和函数 AST 与原版相同,
仅内部导入改为新规范位置。
- 四个根级兼容模块与新模块共享同一类对象,旧导入和旧 monkeypatch 路径继续有效。
- `config/api.config.json`、API路径、鉴权角色、错误结构和数据库 schema 未修改。
- `app/static/` 未修改;七个核心 HTML/JS/CSS 文件哈希继续与原版相同。
- `app-light-1280x720.png` 为真实 `8785` 服务完成载入后的日间模式截图,SHA-256 为
`a7ee1b682f68c418d792727dbc4534d7494dae4f4811cdbba208bd86dcf25d10`
## 3. 真实运行检查
- 迁移副本:`http://127.0.0.1:8785/`,管理员登录成功。
- 总览:返回 2026-07-30 Tushare 已缓存行情,涨停 56 只。
- 搜索:搜索“中国平安”返回 `601318`,点击后打开完整个股详情、日 K、资金流、事件逻辑和复盘笔记。
- 页面:1280px 视口无横向溢出,数据加载状态正常,浏览器控制台 0 个错误。
- 分时:迁移版与原版在当前本机网络环境均返回同一个 `Intraday chart request failed`,因此记录为
既存外部接口状态,不是本切片差异;没有擅自增加降级或改变来源策略。
## 4. 自动验证
| 验证 | 结果 |
|---|---:|
| `python -m unittest discover -s tests -q` | 241 项通过 |
| `python -m unittest tests.test_preservation_slice_market -q` | 6 项通过 |
| 行情、图表、实时与数据库专项集合 | 61 项通过 |
| `npx playwright test --reporter=dot` | 45 项通过 |
| `python -m compileall -q ...` | 通过 |
| `git diff --check` | 通过 |
## 5. 保留边界
- 情绪计算仍在 `application.py`,切片 03 再归位;行情服务只通过继承调用,没有复制。
- 竞价、题材、人气、龙虎榜和问天对公共行情客户端的调用仍可通过兼容别名工作,待各自切片迁移。
- 根级四个数据模块、`DashboardService``ReviewDatabase` 的兼容面在所有消费者完成迁移前保留。
- 没有删除待定代码、没有改动根目录正式数据库、没有切换 Docker/NAS。
Binary file not shown.

After

Width:  |  Height:  |  Size: 92 KiB

@@ -0,0 +1,66 @@
# 切片 03:情绪周期、五类股池与涨停表现
> 基线:`a426432`(切片 02
> 回档标签:`xiaobai-preservation-slice-03-20260731`
> 结论:源码、API、数据库、真实页面和浏览器回归通过;最终视觉仍等待全站人工验收
## 1. 原实现归位
本切片只移动原版副本中的真实实现,没有从 `next/` 取用代码,也没有改写情绪公式、股池数据、
原因补全、表格、样式或交互。
| 原位置 | 新的唯一实现位置 | 原位置兼容 |
|---|---|---|
| `app/backend/application.py` 的 2 个情绪服务方法 | `app/backend/features/sentiment/service.py` | `DashboardService` 继承 `SentimentServiceMixin` |
| `app/backend/application.py` 的 6 个股池原因及事件补全方法 | `app/backend/features/pools/service.py` | `DashboardService` 继承 `PoolServiceMixin` |
| `app/database.py` 的 2 个原因覆盖方法 | `app/backend/features/pools/repository.py` | `ReviewDatabase` 继承 `PoolRepositoryMixin` |
| `app/sentiment_engine.py` | `app/backend/features/sentiment/engine.py` | 根模块为同一模块对象的兼容别名 |
五类股池、涨停梯队和涨停表现仍由切片 02 已归位的原 Tushare 总览实现生成,本切片没有建立第二套
计算或数据来源。
## 2. 等价证据
- `test_preservation_slice_sentiment_pools.py` 对 8 个业务方法和 2 个 Repository 方法逐项执行无位置
信息 AST 比较,全部与根目录原版 `server.py``database.py` 完全相同。
- 新的情绪引擎文件与原版 `sentiment_engine.py` SHA-256 完全相同;根级兼容模块与新模块是同一模块对象。
- 已归位的应用、行情服务、Tushare Provider、演示数据和选股模块直接导入新的唯一实现;Tushare
Provider 仅调整该导入,其全部类和函数 AST 继续与原版一致。
- 原版 `8784` 与迁移版 `8785` 在相同账号、日期和数据库副本上请求 `/api/dashboard`
`/api/sentiment/history`,JSON 状态、字段、值和顺序完全相同。
- 2026-07-30 的同请求结果均为:涨停 56、炸板 23、跌停 83、昨日涨停 81、情绪历史 20 日。
- 原版和迁移版数据库均为 62 个 schema 对象,schema 哈希均为
`17918327f8b919496e6630458293f9f777c7c24662625bb3fc0b64ff0a8fbeef`
- `config/api.config.json`、API 路径、鉴权和 `app/static/` 未修改。
- `app-light-1920x1080.png` 是真实迁移服务载入完成后的情绪周期页面,SHA-256 为
`e387417abbe0667e00875a8d4061b5546748ecf2452a692d06d078d516330dab`
## 3. 真实运行检查
- 迁移副本:`http://127.0.0.1:8785/`,管理员会话与缓存行情载入正常。
- 情绪周期:20 个连续交易日、当前阶段、评分构成和交易日明细均完整显示。
- 股池:涨停池 56 行、炸板池 23 行、跌停池 83 行、昨日涨停 81 行。
- 涨停表现:四档晋级率、市场宽度和今日结论均显示原版结果。
- 1920×1080 下六个页面横向溢出均为 0;日间、夜间背景与面板状态正常;浏览器控制台无迁移错误。
## 4. 自动验证
| 验证 | 结果 |
|---|---:|
| `python -m unittest discover -s tests -q` | 248 项通过 |
| `python -m unittest tests.test_preservation_slice_sentiment_pools -q` | 6 项通过 |
| 情绪、总览、缓存、iFinD 与前端契约专项集合 | 48 项通过 |
| `npx.cmd playwright test --reporter=dot` | 45 项通过 |
| `python -m compileall -q ...` | 通过 |
| `git diff --check` | 通过 |
Windows 下由 Playwright 自行创建临时静态服务器时,45 项完成后子进程无法回收;改为预先启动同一个
`8876` 静态服务器并让 Playwright 复用后,测试以零退出码正常结束,结果为 `45 passed (2.1m)`
## 5. 保留边界
- 板块轮动仍调用情绪历史公共函数,待切片 04 与市场天梯一并归位。
- 竞价、题材、人气和龙虎榜对股池数据的消费保持原调用路径,待切片 05 迁移。
- 根级情绪引擎兼容模块、`DashboardService``ReviewDatabase` 兼容面继续保留;数据库内尚未迁移的
选股统计方法仍走兼容别名,待切片 06 随完整方法一并归位。
- 没有删除待定代码、没有改动根目录正式数据库、没有切换 Docker/NAS。
Binary file not shown.

After

Width:  |  Height:  |  Size: 151 KiB

@@ -0,0 +1,58 @@
# 切片 04:市场天梯与板块轮动
> 基线:`b3555d2`(切片 03
> 回档标签:`xiaobai-preservation-slice-04-20260731`
> 结论:源码、API、真实页面和浏览器回归通过;最终视觉仍等待全站人工验收
## 1. 原实现归位
本切片从原版副本机械移动板块轮动服务,没有从 `next/` 取用代码,也没有修改天梯、轮动的计算、
排序、展开、配色、页面结构或交互。
| 原位置 | 新的唯一实现位置 | 原位置兼容 |
|---|---|---|
| `app/backend/application.py` 的 2 个轮动方法 | `app/backend/features/rotation/service.py` | `DashboardService` 继承 `RotationServiceMixin` |
| Tushare Provider 的天梯与轮动构造函数 | 保持 `app/backend/data/providers/tushare_client.py` | 切片 02 已归位的公共数据实现 |
市场天梯没有独立后端 API 或第二套计算,直接展示 `/api/dashboard` 中原 Tushare 实现生成的
`ladders`;因此没有为目录形式建立空的天梯服务。
## 2. 等价证据
- `test_preservation_slice_ladder_rotation.py` 对 2 个轮动服务方法逐项执行无位置信息 AST 比较,
全部与根目录原版 `server.py` 完全相同。
- `_build_ladders``_build_sector_rotation` 两个原数据构造函数的 AST 与根目录原版完全相同。
- 原版 `8784` 与迁移版 `8785` 在相同账号、日期和数据库副本上返回的天梯数据及 9 日轮动历史
JSON 逐字段完全相同。
- 成分股接口在当前外部网络状态下两版均返回 HTTP 400、`bad_request` 和相同的
`该板块成分股暂不可用:Tushare request failed:`,没有改变错误或增加静默降级。
- `config/api.config.json`、API 路径、鉴权、数据库 schema 和 `app/static/` 未修改。
## 3. 真实运行检查
- 市场天梯:8 个层级(含断层)、18 个首屏股票单元格、3 个结构分析模块正常;1920×1080 下
页面宽度无溢出,首板展开入口保留。
- 板块轮动:9 个交易日、每日 Top 12 共 108 个板块单元格、由远到近/由近到远两个排序入口正常;
1920×1080 下页面宽度无溢出并保持全页滚动。
- 日间模式页面控制台没有错误或警告。
- `app-light-ladder-1920x1080.png` SHA-256
`9e57d18d92e745fd92131f7bf08f21faaaa745476dd942cdaa2a703b9a7a303a`
- `app-light-rotation-1920x1080.png` SHA-256
`03c092bb40bd0eb672136dff853abffc87e9790840107c2f22e6cf31d5f83c09`
## 4. 自动验证
| 验证 | 结果 |
|---|---:|
| `python -m unittest discover -s tests -q` | 252 项通过 |
| `python -m unittest tests.test_preservation_slice_ladder_rotation -q` | 4 项通过 |
| 切片 02 至 04 与总览缓存专项集合 | 24 项通过 |
| `npx.cmd playwright test --reporter=dot` | 45 项通过 |
| `git diff --check` | 通过 |
## 5. 保留边界
- 成分股接口依赖的日行情与因子持久化方法仍由原 `ReviewDatabase` 提供,因其同时服务智能选股,
待切片 06 随完整共享职责归位。
- 天梯和轮动前端资产保持原位,切片 10 再按页面职责归档;当前没有复制或改写。
- 没有删除待定代码、没有改动根目录正式数据库、没有切换 Docker/NAS。
Binary file not shown.

After

Width:  |  Height:  |  Size: 139 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 198 KiB

+5 -5
View File
@@ -1,6 +1,6 @@
{
"schema_version": 1,
"updated_at": "2026-07-31T00:37:00+08:00",
"updated_at": "2026-07-31T01:57:00+08:00",
"status": "active",
"migration_mode": "behavior_preserving_source_migration",
"source_of_truth": "current_original_webapp_runtime_and_source",
@@ -9,10 +9,10 @@
"failed_roots": [
"next"
],
"current_slice": "slice-02-market-search-charts-data",
"last_completed_slice": "slice-01-startup-http-accounts-system",
"last_checkpoint": "xiaobai-preservation-slice-01-20260731",
"next_action": "capture_slice-02_market_search_chart_data_contracts_then_move_original_implementations",
"current_slice": "slice-05-auction-themes-popularity-dragon-tiger",
"last_completed_slice": "slice-04-ladder-rotation",
"last_checkpoint": "xiaobai-preservation-slice-04-20260731",
"next_action": "capture_slice-05_auction_theme_popularity_dragon_tiger_contracts_then_move_original_implementations",
"authoritative_documents": [
"AGENTS.md",
"docs/migration/原版保真迁移总纲.md",
+40 -1
View File
@@ -1,6 +1,6 @@
# 小白复盘保真迁移账本
> 当前状态:正式迁移,切片01“启动、HTTP、账号、会员与系统管理”已完成
> 当前状态:正式迁移,切片04“市场天梯与板块轮动”已完成
本账本是上下文恢复和人工审计的连续记录。任何迁移提交必须在同一提交中更新本文件及
`保真迁移状态.json`
@@ -21,6 +21,9 @@
| 2026-07-30 | `xiaobai-preservation-migration-charter-20260730` | 建立保真迁移总纲、状态和恢复协议 | 尚未开始新迁移 |
| 2026-07-30 | `41329943c4878fc09ed82ec376eb93ab151e4092` | 完成只读资产清查并由用户批准`app/`结构 | 开始切片00 |
| 2026-07-31 | `xiaobai-preservation-slice-01-20260731` | 启动、HTTP、账号、会员与系统管理原实现归位 | 自动差分通过,进入切片02 |
| 2026-07-31 | `xiaobai-preservation-slice-02-20260731` | 公共行情、搜索、详情、图表与数据适配原实现归位 | 自动与浏览器差分通过,进入切片03 |
| 2026-07-31 | `xiaobai-preservation-slice-03-20260731` | 情绪周期、五类股池与涨停表现原实现归位 | 自动、API与浏览器差分通过,进入切片04 |
| 2026-07-31 | `xiaobai-preservation-slice-04-20260731` | 市场天梯与板块轮动原实现归位 | 自动、API与浏览器差分通过,进入切片05 |
## 资产处置登记
@@ -35,6 +38,11 @@
| `commonReviewColumns`等5个前端函数 | 疑似无引用符号 | 未发现静态调用 | 待定 | 待删隔离账本 | 仍需动态注册与浏览器覆盖 | 保留 |
| `wencai_saved_queries`及其方法 | 历史兼容数据 | 当前无前端入口 | 待定 | 数据库兼容区 | 不允许在迁移期破坏旧库 | 保留 |
| 现有7层CSS | 视觉运行资产 | 全部页面和主题 | 原样保留后逐页归档 | `app/frontend/` | 必须通过截图与计算样式差分 | 保留 |
| `sentiment_engine.py` | 情绪周期计算 | 总览、轮动、选股 | 移动并保留兼容别名 | `app/backend/features/sentiment/engine.py` | 文件哈希与原版一致;248项Python与45项Playwright通过 | 已移动 |
| `DashboardService`情绪及股池原因方法 | 业务服务 | 情绪页、五类股池、涨停表现 | 按职责机械移动 | `app/backend/features/sentiment/``app/backend/features/pools/` | 8个方法AST与原版一致;真实API完全一致 | 已移动 |
| `ReviewDatabase`原因覆盖方法 | 持久化 | 股池原因人工覆盖 | 按职责机械移动 | `app/backend/features/pools/repository.py` | 2个方法AST与原版一致;数据库schema哈希一致 | 已移动 |
| `DashboardService`板块轮动方法 | 业务服务 | 板块轮动页 | 按职责机械移动 | `app/backend/features/rotation/service.py` | 2个方法AST、真实API与原版一致 | 已移动 |
| Tushare天梯与轮动构造函数 | 公共数据计算 | 市场天梯、板块轮动 | 原位置保持唯一实现 | `app/backend/data/providers/tushare_client.py` | 2个构造函数AST与原版一致 | 已归位 |
处置只允许:`原样保留``移动``合并重复``待定``确认废弃`
@@ -61,6 +69,37 @@
- 回档:标签`xiaobai-preservation-slice-01-20260731`
- 完整证据:`docs/migration/evidence/slice-01/README.md`
已完成切片:`slice-02-market-search-charts-data`
- 原版基线:提交`4002f09`,即切片01回档点。
- 迁移范围:30个公共行情服务方法、11个行情持久化方法及Tushare/iFinD/图表/实时观察实现。
- 兼容边界:四个根级数据模块保留模块别名;未迁移功能继续使用旧导入且指向同一实现。
- 等价证明:41个方法AST逐项一致,三个数据文件哈希一致,图表定义AST一致,静态资产哈希一致。
- 验收:241项Python测试、6项切片源码等价测试、45项Playwright测试及真实服务搜索/详情流程通过。
- 既存状态:原版和迁移版的实时分时在当前环境均返回相同外部请求失败,不作为迁移回归处理。
- 回档:标签`xiaobai-preservation-slice-02-20260731`
- 完整证据:`docs/migration/evidence/slice-02/README.md`
已完成切片:`slice-03-sentiment-pools-performance`
- 原版基线:提交`a426432`,即切片02回档点。
- 迁移范围:情绪计算引擎、2个情绪服务方法、6个股池原因与iFinD事件补全方法、2个原因覆盖持久化方法。
- 兼容边界:根级`sentiment_engine.py`保留同一模块对象别名;股池生成仍使用切片02的原Tushare总览实现。
- API与数据库:原版`8784`和迁移版`8785`的总览、情绪历史JSON完全一致;两库schema均为62项且哈希一致。
- 验收:248项Python测试、6项切片源码等价测试、45项Playwright测试及六个真实页面流程通过。
- 回档:标签`xiaobai-preservation-slice-03-20260731`
- 完整证据:`docs/migration/evidence/slice-03/README.md`
已完成切片:`slice-04-ladder-rotation`
- 原版基线:提交`b3555d2`,即切片03回档点。
- 迁移范围:2个板块轮动服务方法;市场天梯继续使用切片02已归位的原Tushare数据构造实现。
- 兼容边界:`DashboardService`通过`RotationServiceMixin`保持所有原调用;天梯不制造空服务或第二套计算。
- API与错误:天梯与9日轮动历史JSON完全一致;成分股两版均返回同一Tushare外部失败语义。
- 验收:252项Python测试、4项切片源码等价测试、45项Playwright测试及两个真实页面流程通过。
- 回档:标签`xiaobai-preservation-slice-04-20260731`
- 完整证据:`docs/migration/evidence/slice-04/README.md`
## 决策记录
| 日期 | 决策 | 原因 |