leefer/xiaobai-review
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migration: preserve market data and search slice

Browse Source
This commit is contained in:
leefer
2026-07-31 01:03:43 +08:00
parent 982af4ba92
commit ac94f5285e
22 changed files with 4956 additions and 4628 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/application.py
+4 -927
View File
@@ -18,7 +18,7 @@ from backend.bootstrap.container import build_application_container
from backend.bootstrap.settings import load_runtime_settings from backend.bootstrap.settings import load_runtime_settings
from backend.http import HttpTransportMixin from backend.http import HttpTransportMixin
from backend.llm import LLMGateway, LLMGatewayError from backend.llm import LLMGateway, LLMGatewayError
from chart_data_provider import ChartDataError from backend.features.market import ChartDataError, MarketServiceMixin
from backend.bootstrap.config import ( from backend.bootstrap.config import (
DATA_DIR, DATA_DIR,
MENTOR_SKILLS_DIR, MENTOR_SKILLS_DIR,
@@ -39,7 +39,7 @@ from heaven_engine import (
build_personal_field, build_personal_field,
hexagram_from_lines, hexagram_from_lines,
) )
from ifind_client import IfindError from backend.data.providers.ifind_client import IfindError
from llm_strategy import LLMCompilerError, compile_strategy_with_llm, test_llm_connection from llm_strategy import LLMCompilerError, compile_strategy_with_llm, test_llm_connection
from mentor_agent import MentorAgentError, stream_with_mentor from mentor_agent import MentorAgentError, stream_with_mentor
from market_insights import MarketInsightsService from market_insights import MarketInsightsService
@@ -61,7 +61,7 @@ from sentiment_engine import (
build_sentiment_history, build_sentiment_history,
latest_contiguous_history, latest_contiguous_history,
) )
from tushare_client import TushareClient, TushareError, _sector_coverage_issue from backend.data.providers.tushare_client import TushareClient, TushareError, _sector_coverage_issue
SCREENER_LIBRARY_VERSION = 8 SCREENER_LIBRARY_VERSION = 8
@@ -103,23 +103,6 @@ LEGACY_SECRET_KEYS = {
"LLM_FALLBACK_MODEL", "LLM_FALLBACK_MODEL",
} }
SEARCH_INDEXES = (
{"id": "000001.SH", "code": "000001.SH", "name": "上证指数", "type": "index", "subtitle": "沪市综合指数"},
{"id": "399001.SZ", "code": "399001.SZ", "name": "深证成指", "type": "index", "subtitle": "深市成份指数"},
{"id": "399006.SZ", "code": "399006.SZ", "name": "创业板指", "type": "index", "subtitle": "创业板核心指数"},
)
SEARCH_TYPE_LABELS = {
"stock": "股票",
"sector": "板块",
"theme": "题材",
"index": "指数",
}
THS_SEARCH_TYPES = {
"I": ("sector", "行业板块"),
"R": ("sector", "地域板块"),
"N": ("theme", "概念题材"),
}
MENTOR_DATA_PROFILES = { MENTOR_DATA_PROFILES = {
"emotion": { "emotion": {
"kobe92-perspective", "niepanchongsheng-perspective", "kobe92-perspective", "niepanchongsheng-perspective",
@@ -158,7 +141,7 @@ MENTOR_ETF_UNIVERSE = (
) )
class DashboardService: class DashboardService(MarketServiceMixin):
def __init__(self) -> None: def __init__(self) -> None:
runtime = load_runtime_settings() runtime = load_runtime_settings()
self.vault = SecretVault(runtime.encryption_key) self.vault = SecretVault(runtime.encryption_key)
@@ -213,12 +196,6 @@ class DashboardService:
initial_delay_seconds=3, initial_delay_seconds=3,
) )
def _tushare_client(self) -> TushareClient:
gateway = getattr(self, "data_gateway", None)
if gateway is not None:
return gateway.tushare()
# Compatibility for isolated legacy unit-test service stubs.
return TushareClient(self.token)
def _load_system_credentials(self, environment: dict[str, str]) -> dict[str, Any]: def _load_system_credentials(self, environment: dict[str, str]) -> dict[str, Any]:
encrypted = self.database.get_system_setting("credentials") encrypted = self.database.get_system_setting("credentials")
@@ -756,181 +733,6 @@ class DashboardService:
def _public_personal_profile(personal: dict[str, Any]) -> dict[str, Any]: def _public_personal_profile(personal: dict[str, Any]) -> dict[str, Any]:
return AccountService.public_personal_profile(personal) return AccountService.public_personal_profile(personal)
def get_dashboard(self, trade_date: str, force: bool = False) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
now = datetime.now().astimezone()
if (
normalized_date == now.strftime("%Y%m%d")
and now.time().replace(tzinfo=None) < datetime.strptime("09:15", "%H:%M").time()
):
previous = self.database.get_latest_real_snapshot(normalized_date, strictly_before=True)
if previous:
carried = self._carry_dashboard(previous, normalized_date, "盘前沿用最近交易日收盘行情")
return self._apply_reason_overrides(self._with_storage(carried, cached=True))
if not force:
snapshot = self.database.get_snapshot(normalized_date)
if snapshot and str((snapshot.get("meta") or {}).get("source") or "") != "demo":
snapshot = copy.deepcopy(snapshot)
if normalized_date != now.strftime("%Y%m%d"):
snapshot.setdefault("meta", {}).update(
{"realtime": False, "market_status": "closed"}
)
if not self._dashboard_sentiment_ready(snapshot):
snapshot = self._enrich_dashboard_sentiment(snapshot, normalized_date)
self.database.save_snapshot(
normalized_date,
str((snapshot.get("meta") or {}).get("source") or "tushare"),
snapshot,
)
snapshot.setdefault("meta", {})["requested_date"] = self._display_compact_date(normalized_date)
return self._apply_reason_overrides(self._with_storage(snapshot, cached=True))
resolved = self.database.get_data_snapshot(
"dashboard_request_v1", normalized_date
)
if resolved and str((resolved.get("meta") or {}).get("source") or "") != "demo":
resolved = copy.deepcopy(resolved)
resolved.setdefault("meta", {})["requested_date"] = self._display_compact_date(
normalized_date
)
return self._apply_reason_overrides(
self._with_storage(resolved, cached=True)
)
if datetime.strptime(normalized_date, "%Y%m%d").weekday() >= 5:
previous = self.database.get_latest_real_snapshot(normalized_date)
if previous:
carried = self._carry_dashboard(
previous,
normalized_date,
"非交易日沿用最近交易日收盘行情",
)
self.database.save_data_snapshot(
"dashboard_request_v1", normalized_date, "sqlite", carried
)
return self._apply_reason_overrides(
self._with_storage(carried, cached=True)
)
return self.sync_dashboard(normalized_date)
@staticmethod
def _dashboard_sentiment_ready(dashboard: dict[str, Any]) -> bool:
overview = dashboard.get("overview") or {}
return int(overview.get("sentiment_engine_version") or 0) == SENTIMENT_ENGINE_VERSION and all(
key in overview
for key in (
"sentiment_score",
"sentiment_label",
"sentiment_phase",
"sentiment_direction",
"sentiment_components",
)
)
@staticmethod
def _display_compact_date(compact: str) -> str:
return f"{compact[:4]}-{compact[4:6]}-{compact[6:8]}"
def _carry_dashboard(
self, snapshot: dict[str, Any], requested_date: str, reason: str
) -> dict[str, Any]:
carried = copy.deepcopy(snapshot)
meta = carried.setdefault("meta", {})
meta.update(
{
"requested_date": self._display_compact_date(requested_date),
"carried_forward": True,
"realtime": False,
"market_status": "closed",
"notice": reason,
}
)
return carried
def _realtime_snapshot_due(
self,
normalized_date: str,
snapshot: dict[str, Any],
) -> bool:
if not self.configured or normalized_date != date.today().strftime("%Y%m%d"):
return False
now = datetime.now().astimezone()
local_time = now.time().replace(tzinfo=None)
realtime_start = datetime.strptime("09:15", "%H:%M").time()
morning_end = datetime.strptime("11:35", "%H:%M").time()
afternoon_start = datetime.strptime("12:55", "%H:%M").time()
realtime_end = datetime.strptime("15:05", "%H:%M").time()
in_session = (
realtime_start <= local_time < morning_end
or afternoon_start <= local_time < realtime_end
)
if not in_session:
return False
meta = snapshot.get("meta") or {}
snapshot_trade_date = str(meta.get("trade_date") or "").replace("-", "")
if snapshot_trade_date and snapshot_trade_date != normalized_date:
return False
if not meta.get("realtime"):
return True
try:
updated_at = datetime.fromisoformat(str(meta.get("updated_at") or ""))
if updated_at.tzinfo is None:
updated_at = updated_at.replace(tzinfo=now.tzinfo)
except ValueError:
return True
age_seconds = (now - updated_at.astimezone(now.tzinfo)).total_seconds()
return age_seconds >= 8
def sync_dashboard(self, trade_date: str) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
source = "tushare"
with self.sync_lock:
sync_id = self.database.start_sync(normalized_date, source)
try:
if not self.configured:
raise TushareError("公共行情尚未配置")
dashboard = self._tushare_client().dashboard(normalized_date)
dashboard["meta"]["source"] = source
dashboard["meta"]["requested_date"] = self._display_compact_date(normalized_date)
dashboard = self._enrich_dashboard_sentiment(dashboard, normalized_date)
record_count = self._record_count(dashboard)
actual_date = normalize_date(
str(dashboard.get("meta", {}).get("trade_date") or normalized_date)
)
self.database.save_snapshot(actual_date, source, dashboard)
if actual_date != normalized_date:
dashboard.setdefault("meta", {}).update(
{
"carried_forward": True,
"realtime": False,
"market_status": "closed",
}
)
self.database.save_data_snapshot(
"dashboard_request_v1", normalized_date, source, dashboard
)
self.database.finish_sync(
sync_id,
"success",
record_count,
dashboard.get("meta", {}).get("notice", ""),
source,
)
return self._apply_reason_overrides(self._with_storage(dashboard, cached=False))
except TushareError as exc:
fallback = self.database.get_latest_real_snapshot(normalized_date)
if fallback:
carried = self._carry_dashboard(
fallback, normalized_date, f"最新行情暂不可用，沿用最近收盘快照：{exc}"
)
self.database.finish_sync(
sync_id, "fallback", self._record_count(carried), str(exc), "tushare"
)
return self._apply_reason_overrides(self._with_storage(carried, cached=True))
self.database.finish_sync(sync_id, "failed", message=str(exc))
raise ValueError("暂无可用的真实行情快照，请等待后台完成首次同步。") from exc
except Exception as exc:
self.database.finish_sync(sync_id, "failed", message=str(exc))
raise
def _enrich_dashboard_sentiment( def _enrich_dashboard_sentiment(
self, self,
@@ -1126,9 +928,6 @@ class DashboardService:
**self.database.status(), **self.database.status(),
} }
def realtime_aggregate_health(self, sector: str = "") -> dict[str, Any]:
sector = validate_text(sector, "板块名称", 50)
return self.realtime_aggregator.health_snapshot(sector)
def _market_insights(self) -> MarketInsightsService: def _market_insights(self) -> MarketInsightsService:
if not self.configured: if not self.configured:
@@ -3580,663 +3379,6 @@ class DashboardService:
"rows": [], "rows": [],
} }
def _search_market_directory(self) -> list[dict[str, Any]]:
cached = self.database.get_data_snapshot("search_directory", "ths") or {}
cached_items = list(cached.get("items") or [])
if cached_items and int(cached.get("schema_version") or 0) >= 2:
return cached_items
if not self.configured:
return cached_items
try:
rows = self._tushare_client().query(
"ths_index",
{},
"ts_code,name,count,exchange,list_date,type",
)
except TushareError:
return cached_items
items = []
for row in rows:
mapping = THS_SEARCH_TYPES.get(str(row.get("type") or "").upper())
code = str(row.get("ts_code") or "").strip().upper()
name = str(row.get("name") or "").strip()
if not mapping or not code or not name or str(row.get("exchange") or "").upper() != "A":
continue
entity_type, subtitle = mapping
items.append(
{
"id": code,
"code": code,
"name": name,
"type": entity_type,
"subtitle": subtitle,
"member_count": int(float(row.get("count") or 0)),
}
)
if items:
self.database.save_data_snapshot(
"search_directory", "ths", "tushare", {"schema_version": 2, "items": items}
)
return items
@staticmethod
def _search_match_score(item: dict[str, Any], query: str) -> tuple[int, int, str]:
name = str(item.get("name") or "").casefold()
code = str(item.get("code") or item.get("id") or "").casefold()
needle = query.casefold()
if code == needle:
rank = 0
elif name == needle:
rank = 1
elif code.startswith(needle):
rank = 2
elif name.startswith(needle):
rank = 3
else:
rank = 4
return rank, len(name), code
def search_entities(self, query: str, trade_date: str) -> dict[str, Any]:
needle = str(query or "").strip()
normalized_date = normalize_date(trade_date)
groups: dict[str, list[dict[str, Any]]] = {
"stocks": [],
"sectors": [],
"themes": [],
"indices": [],
}
if not needle:
return {"query": "", "trade_date": normalized_date, "groups": groups}
stocks = []
for row in self.database.search_stock_master(needle, 12):
stocks.append(
{
"id": str(row.get("code") or ""),
"code": str(row.get("code") or ""),
"name": str(row.get("name") or "--"),
"type": "stock",
"type_label": SEARCH_TYPE_LABELS["stock"],
"industry": str(row.get("industry") or "其他"),
"market": str(row.get("market") or ""),
"subtitle": " · ".join(
part for part in (str(row.get("industry") or ""), str(row.get("market") or "")) if part
) or "A股",
}
)
groups["stocks"] = stocks[:8]
market_items = list(self._search_market_directory()) + [dict(item) for item in SEARCH_INDEXES]
matched = [
item for item in market_items
if needle.casefold() in str(item.get("name") or "").casefold()
or needle.casefold() in str(item.get("code") or "").casefold()
]
matched.sort(key=lambda item: self._search_match_score(item, needle))
group_keys = {"sector": "sectors", "theme": "themes", "index": "indices"}
for item in matched:
group_key = group_keys.get(str(item.get("type") or ""))
if not group_key or len(groups[group_key]) >= 8:
continue
groups[group_key].append(
{
**item,
"type_label": SEARCH_TYPE_LABELS[str(item["type"])],
}
)
return {"query": needle, "trade_date": normalized_date, "groups": groups}
def get_search_detail(
self, entity_type: str, identifier: str, trade_date: str
) -> dict[str, Any]:
entity_type = str(entity_type or "").strip().lower()
identifier = str(identifier or "").strip().upper()
normalized_date = normalize_date(trade_date)
if entity_type not in {"sector", "theme", "index"}:
raise ValueError("搜索详情类型不支持。")
if not re.fullmatch(r"[A-Z0-9.]{3,24}", identifier):
raise ValueError("搜索详情标识无效。")
if not self.configured:
raise ValueError("行情数据源尚未配置。")
if entity_type == "index":
index_basic = next((item for item in SEARCH_INDEXES if item["id"] == identifier), None)
if not index_basic:
raise ValueError("暂不支持该指数详情。")
return self._index_search_detail(index_basic, normalized_date)
directory = self._search_market_directory()
basic = next(
(
item for item in directory
if item.get("id") == identifier and item.get("type") == entity_type
),
None,
)
if not basic:
raise ValueError("未找到对应的板块或题材。")
return self._ths_search_detail(basic, normalized_date)
def get_intraday_chart(
self, entity_type: str, identifier: str
) -> dict[str, Any]:
entity_type = str(entity_type or "").strip().lower()
identifier = str(identifier or "").strip().upper()
if entity_type == "stock":
code = validate_stock_code(identifier)
chart = self.chart_data.stock_intraday(code)
type_label = SEARCH_TYPE_LABELS["stock"]
elif entity_type == "index":
basic = next((item for item in SEARCH_INDEXES if item["id"] == identifier), None)
if not basic:
raise ValueError("暂不支持该指数分时行情。")
chart = self.chart_data.index_intraday(identifier)
type_label = SEARCH_TYPE_LABELS["index"]
elif entity_type in {"sector", "theme"}:
basic = next(
(
item for item in self._search_market_directory()
if item.get("id") == identifier and item.get("type") == entity_type
),
None,
)
if not basic:
raise ValueError("未找到对应的板块或题材。")
chart = self.chart_data.board_intraday(identifier, str(basic.get("name") or ""))
type_label = SEARCH_TYPE_LABELS[entity_type]
else:
raise ValueError("分时行情类型不支持。")
return {
"meta": {
"trade_date": str(chart.get("trade_date") or ""),
"previous_close": float(chart.get("previous_close") or 0),
},
"entity": {
"id": identifier,
"code": str(chart.get("code") or identifier),
"name": str(chart.get("name") or ""),
"type": entity_type,
"type_label": type_label,
},
"points": list(chart.get("points") or []),
}
def _ths_search_detail(
self, basic: dict[str, Any], trade_date: str
) -> dict[str, Any]:
client = self._tushare_client()
resolved_date, _ = client.resolve_trade_context(trade_date)
end = datetime.strptime(resolved_date, "%Y%m%d")
start_date = (end - timedelta(days=190)).strftime("%Y%m%d")
identifier = str(basic["id"])
snapshot = client.sector_snapshot(identifier, resolved_date)
rows = client.query(
"ths_daily",
{"ts_code": identifier, "start_date": start_date, "end_date": resolved_date},
"ts_code,trade_date,open,high,low,close,pct_change,vol,turnover_rate,total_mv,float_mv",
)
rows.sort(key=lambda item: str(item.get("trade_date") or ""))
series = [
{
"trade_date": self._display_compact_date(str(row.get("trade_date") or "")),
"open": float(row.get("open") or 0),
"high": float(row.get("high") or 0),
"low": float(row.get("low") or 0),
"close": float(row.get("close") or 0),
"change": float(row.get("pct_change") or 0),
"volume": float(row.get("vol") or 0),
"turnover_rate": float(row.get("turnover_rate") or 0),
}
for row in rows[-90:]
]
try:
chart_series = self.chart_data.board_daily(identifier, resolved_date, 90)
if chart_series:
series = chart_series
except (AttributeError, ChartDataError):
pass
latest = series[-1] if series else {}
snapshot_is_current = str(snapshot.get("trade_date") or "").replace("-", "") == resolved_date
change = float(
snapshot.get("change")
if snapshot_is_current and snapshot.get("change") is not None
else latest.get("change") or 0
)
if latest.get("realtime"):
change = float(latest.get("change") or 0)
turnover_rate = float(
snapshot.get("turnover_rate")
if snapshot_is_current and snapshot.get("turnover_rate") is not None
else latest.get("turnover_rate") or 0
)
metrics = [
{"label": "涨跌幅", "value": round(change, 2), "unit": "%", "tone": "change"},
{"label": "换手率", "value": round(turnover_rate, 2), "unit": "%"},
{"label": "成份数量", "value": int(float(basic.get("member_count") or 0)), "unit": "只"},
]
up_count = int(float(snapshot.get("up_count") or 0))
down_count = int(float(snapshot.get("down_count") or 0))
if up_count or down_count:
metrics.extend(
[
{"label": "上涨家数", "value": up_count, "unit": "家"},
{"label": "下跌家数", "value": down_count, "unit": "家"},
]
)
leader = str(snapshot.get("leader") or "").strip()
if leader and leader != "--":
metrics.extend(
[
{"label": "领涨标的", "value": leader, "unit": ""},
{"label": "领涨幅", "value": round(float(snapshot.get("leading_pct") or 0), 2), "unit": "%", "tone": "change"},
]
)
return {
"meta": {
"trade_date": self._display_compact_date(resolved_date),
"realtime": bool(snapshot.get("realtime")),
},
"entity": {
"id": identifier,
"code": identifier,
"name": str(snapshot.get("name") or basic.get("name") or "--"),
"type": str(basic.get("type") or "sector"),
"type_label": SEARCH_TYPE_LABELS[str(basic.get("type") or "sector")],
"subtitle": str(basic.get("subtitle") or ""),
"value": float(latest.get("close") or 0),
"change": change,
},
"series": series,
"metrics": metrics,
}
def _index_search_detail(
self, basic: dict[str, Any], trade_date: str
) -> dict[str, Any]:
client = self._tushare_client()
resolved_date, _ = client.resolve_trade_context(trade_date)
payload = (
client.realtime_market_indices(resolved_date)
if client.should_use_realtime(trade_date, resolved_date)
else client.market_indices(resolved_date, 90)
)
current = next(
(item for item in payload.get("indices") or [] if item.get("ts_code") == basic["id"]),
None,
)
if not current:
raise ValueError("该指数暂无可用行情。")
end = datetime.strptime(resolved_date, "%Y%m%d")
rows = client.query(
"index_daily",
{
"ts_code": basic["id"],
"start_date": (end - timedelta(days=190)).strftime("%Y%m%d"),
"end_date": resolved_date,
},
"ts_code,trade_date,open,high,low,close,pct_chg,vol,amount",
)
rows.sort(key=lambda item: str(item.get("trade_date") or ""))
series = [
{
"trade_date": self._display_compact_date(str(row.get("trade_date") or "")),
"open": float(row.get("open") or 0),
"high": float(row.get("high") or 0),
"low": float(row.get("low") or 0),
"close": float(row.get("close") or 0),
"change": float(row.get("pct_chg") or 0),
"volume": float(row.get("vol") or 0),
}
for row in rows[-90:]
]
try:
chart_series = self.chart_data.index_daily(str(basic["id"]), resolved_date, 90)
if chart_series:
series = chart_series
except (AttributeError, ChartDataError):
pass
latest = series[-1] if series else {}
latest_close = float(latest.get("close") or current.get("close") or 0)
latest_change = float(latest.get("change") or current.get("pct_chg") or 0)
def series_return(days: int) -> float:
if len(series) <= days:
return 0.0
previous = float(series[-days - 1].get("close") or 0)
return (latest_close / previous - 1) * 100 if previous > 0 else 0.0
return {
"meta": {
"trade_date": self._display_compact_date(str(current.get("trade_date") or resolved_date)),
"realtime": bool(payload.get("realtime")),
},
"entity": {
**basic,
"type_label": SEARCH_TYPE_LABELS["index"],
"value": latest_close,
"change": latest_change,
},
"series": series,
"metrics": [
{"label": "涨跌幅", "value": round(latest_change, 2), "unit": "%", "tone": "change"},
{"label": "近5日", "value": round(series_return(5), 2), "unit": "%", "tone": "change"},
{"label": "近20日", "value": round(series_return(20), 2), "unit": "%", "tone": "change"},
{"label": "成交额", "value": round(float(current.get("amount_billion") or 0), 2), "unit": "亿"},
],
}
def get_stock_detail(
self, code: str, trade_date: str, force: bool = False
) -> dict[str, Any]:
code = validate_stock_code(code)
normalized_date = normalize_date(trade_date)
cache_key = f"{code}:{normalized_date}"
if not force:
cached = self.database.get_data_snapshot("stock_detail", cache_key)
if cached and str((cached.get("meta") or {}).get("source") or "") != "demo":
if not self._stock_detail_cache_needs_refresh(cached, normalized_date):
cached["meta"] = {**cached.get("meta", {}), "cached": True}
return self._prepare_stock_detail(cached, code, normalized_date)
name, sector = self._stock_identity(code, normalized_date)
source = "tushare"
if self.configured:
try:
payload = self._tushare_client().stock_detail(
tushare_code(code), normalized_date
)
if not payload.get("prices"):
raise TushareError("No price history returned")
except TushareError as exc:
payload = self.database.get_latest_data_snapshot(
"stock_detail", f"{code}:", cache_key, exclude_source="demo"
)
if not payload:
raise ValueError(f"暂无 {code} 的真实行情数据：{exc}") from exc
payload = copy.deepcopy(payload)
payload["meta"] = {
**payload.get("meta", {}),
"cached": True,
"notice": "最新行情暂不可用，已沿用最近真实收盘数据。",
}
return self._prepare_stock_detail(payload, code, normalized_date)
else:
payload = self.database.get_latest_data_snapshot(
"stock_detail", f"{code}:", cache_key, exclude_source="demo"
)
if not payload:
raise ValueError(f"暂无 {code} 的真实行情数据，请等待后台完成首次同步。")
payload = copy.deepcopy(payload)
payload["meta"] = {
**payload.get("meta", {}),
"cached": True,
"notice": "公共行情尚未配置，已沿用最近真实收盘数据。",
}
return self._prepare_stock_detail(payload, code, normalized_date)
payload["meta"]["source"] = source
payload["meta"]["cached"] = False
self.database.save_data_snapshot("stock_detail", cache_key, source, payload)
return self._prepare_stock_detail(payload, code, normalized_date)
@staticmethod
def _stock_detail_bar_date(payload: dict[str, Any]) -> str:
prices = list(payload.get("prices") or [])
return str((prices[-1] if prices else {}).get("trade_date") or "").replace("-", "")
def _stock_detail_cache_needs_refresh(
self, payload: dict[str, Any], requested_date: str
) -> bool:
now = datetime.now().astimezone()
return (
requested_date == now.strftime("%Y%m%d")
and now.time().replace(tzinfo=None) >= dt_time(15, 0)
and self._stock_detail_bar_date(payload) < requested_date
)
def _prepare_stock_detail(
self, payload: dict[str, Any], code: str, requested_date: str
) -> dict[str, Any]:
result = copy.deepcopy(payload)
now = datetime.now().astimezone()
try:
result["prices"] = self.chart_data.stock_daily(code, requested_date, 90)
result["meta"] = {**(result.get("meta") or {}), "chart_source": "market_chart"}
except (AttributeError, ChartDataError):
pass
result = self._sanitize_stock_detail_prices(result, now)
actual_date = self._stock_detail_bar_date(result)
if actual_date:
result["meta"] = {
**(result.get("meta") or {}),
"trade_date": f"{actual_date[:4]}-{actual_date[4:6]}-{actual_date[6:]}",
}
today = now.strftime("%Y%m%d")
should_merge = (
requested_date == today
and actual_date <= today
and now.weekday() < 5
and now.time().replace(tzinfo=None) >= dt_time(9, 30)
)
if should_merge:
quote = self._ifind_realtime_stock_quote(code)
if quote and self._valid_realtime_stock_quote(quote, today):
self._merge_realtime_stock_detail(result, quote, requested_date)
elif self.configured and actual_date < today:
client = self._tushare_client()
try:
resolved_date, _ = client.resolve_trade_context(requested_date)
if resolved_date == today:
quote = client.realtime_stock_quote(tushare_code(code), requested_date)
if self._valid_realtime_stock_quote(quote, today):
self._merge_realtime_stock_detail(result, quote, requested_date)
except TushareError:
pass
return self._enrich_stock_detail(result)
@staticmethod
def _sanitize_stock_detail_prices(
payload: dict[str, Any], market_now: datetime
) -> dict[str, Any]:
result = copy.deepcopy(payload)
raw_prices = list(result.get("prices") or [])
raw_latest_date = str(
(raw_prices[-1] if raw_prices else {}).get("trade_date") or ""
).replace("-", "")
prices = []
for bar in raw_prices:
open_price = float(bar.get("open") or 0)
high = float(bar.get("high") or 0)
low = float(bar.get("low") or 0)
close = float(bar.get("close") or 0)
if (
open_price > 0
and high >= max(open_price, close)
and 0 < low <= min(open_price, close)
and close > 0
):
prices.append(bar)
today = market_now.strftime("%Y%m%d")
market_open = (
market_now.weekday() < 5
and market_now.time().replace(tzinfo=None) >= dt_time(9, 30)
)
if prices and str(prices[-1].get("trade_date") or "").replace("-", "") == today:
current = prices[-1]
has_market_activity = (
float(current.get("volume") or 0) > 0
or float(current.get("amount_billion") or 0) > 0
)
if not market_open or not has_market_activity:
prices.pop()
if raw_latest_date == today and (
not prices
or str(prices[-1].get("trade_date") or "").replace("-", "") != today
):
result["meta"] = {**(result.get("meta") or {}), "realtime": False}
result["prices"] = prices
if prices:
latest = prices[-1]
stock = dict(result.get("stock") or {})
stock.update(
{
"price": float(latest.get("close") or 0),
"change": float(latest.get("change") or 0),
"amount_billion": float(latest.get("amount_billion") or 0),
}
)
result["stock"] = stock
return result
@staticmethod
def _valid_realtime_stock_quote(quote: dict[str, Any], trade_date: str) -> bool:
price = float(quote.get("price") or 0)
open_price = float(quote.get("open") or 0)
high = float(quote.get("high") or 0)
low = float(quote.get("low") or 0)
volume = float(quote.get("volume") or 0)
amount = float(quote.get("amount_billion") or 0)
quote_date = str(quote.get("quote_time") or "")[:10].replace("-", "")
return (
price > 0
and open_price > 0
and high >= max(open_price, price)
and 0 < low <= min(open_price, price)
and (volume > 0 or amount > 0)
and (not quote_date or quote_date == trade_date)
)
def _ifind_realtime_stock_quote(self, code: str) -> dict[str, Any] | None:
ifind = getattr(self, "ifind", None)
if not ifind or not ifind.configured:
return None
try:
rows = ifind.real_time(
tushare_code(code),
[
"open", "high", "low", "latest", "preClose",
"volume", "amount", "turnoverRatio",
],
cache_ttl=10,
)
except IfindError:
return None
row = rows[0] if rows else {}
price = float(row.get("latest") or 0)
previous_close = float(row.get("preClose") or 0)
if price <= 0:
return None
change = (price / previous_close - 1) * 100 if previous_close > 0 else 0.0
stock = self._stock_identity(code, date.today().strftime("%Y%m%d"))
return {
"name": stock[0],
"sector": stock[1],
"price": price,
"open": float(row.get("open") or price),
"high": float(row.get("high") or price),
"low": float(row.get("low") or price),
"change": round(change, 4),
"volume": float(row.get("volume") or 0),
"volume_unit": "lots",
"amount_billion": float(row.get("amount") or 0) / 100_000_000,
"turnover_rate": float(row.get("turnoverRatio") or 0),
"quote_time": str(row.get("time") or ""),
}
@staticmethod
def _merge_realtime_stock_detail(
payload: dict[str, Any], quote: dict[str, Any], trade_date: str
) -> None:
display_date = f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:]}"
realtime_bar = {
"trade_date": display_date,
"open": quote["open"],
"high": quote["high"],
"low": quote["low"],
"close": quote["price"],
"change": quote["change"],
"volume": quote["volume"] if quote.get("volume_unit") == "lots" else quote["volume"] / 100,
"amount_billion": quote["amount_billion"],
"realtime": True,
}
prices = list(payload.get("prices") or [])
if prices and str(prices[-1].get("trade_date") or "").replace("-", "") == trade_date:
prices[-1] = realtime_bar
else:
prices.append(realtime_bar)
payload["prices"] = prices[-90:]
stock = dict(payload.get("stock") or {})
stock.update(
{
"name": quote["name"],
"industry": quote["sector"],
"price": quote["price"],
"change": quote["change"],
"amount_billion": quote["amount_billion"],
"turnover_rate": quote["turnover_rate"],
}
)
payload["stock"] = stock
payload["meta"] = {
**(payload.get("meta") or {}),
"trade_date": display_date,
"realtime": True,
"updated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
}
def get_stock_preview(
self, code: str, trade_date: str, force: bool = False
) -> dict[str, Any]:
code = validate_stock_code(code)
# Hover previews deliberately follow the latest market day, independent
# from the review date selected by the page.
detail = self.get_stock_detail(code, date.today().strftime("%Y%m%d"), force)
detail_meta = detail.get("meta") or {}
resolved_date = str(detail_meta.get("trade_date") or trade_date)
intraday_points: list[dict[str, Any]] = []
intraday_status = "unavailable"
intraday_notice = "分时行情暂不可用。"
intraday_trade_date = ""
intraday_previous_close = 0.0
try:
intraday = self.chart_data.stock_intraday(code)
intraday_points = list(intraday.get("points") or [])
intraday_trade_date = str(intraday.get("trade_date") or "")
intraday_previous_close = float(intraday.get("previous_close") or 0)
if intraday_points:
intraday_status = "available"
intraday_notice = ""
else:
intraday_status = "empty"
intraday_notice = "最近交易日暂无分时数据。"
except ChartDataError:
intraday_status = "unavailable"
intraday_notice = "分时行情暂不可用，请稍后重试。"
prices = list(detail.get("prices") or [])[-60:]
stock = dict(detail.get("stock") or {"code": code})
realtime = bool(detail_meta.get("realtime"))
return {
"meta": {
"trade_date": resolved_date,
"source": detail_meta.get("source") or "unavailable",
"notice": detail_meta.get("notice") or "",
"intraday_status": intraday_status,
"intraday_notice": intraday_notice,
"intraday_trade_date": intraday_trade_date,
"intraday_previous_close": intraday_previous_close,
"realtime": realtime,
"refresh_interval_seconds": 10 if realtime else 0,
},
"stock": stock,
"prices": prices,
"intraday": intraday_points,
}
def save_reason(self, trade_date: str, code: str, reason: str) -> None: def save_reason(self, trade_date: str, code: str, reason: str) -> None:
normalized_date = normalize_date(trade_date) normalized_date = normalize_date(trade_date)
@@ -4246,55 +3388,6 @@ class DashboardService:
raise ValueError("涨停原因应为 1 至 200 个字符。") raise ValueError("涨停原因应为 1 至 200 个字符。")
self.database.save_reason_override(normalized_date, code, reason) self.database.save_reason_override(normalized_date, code, reason)
def backfill(self, start_date: str, end_date: str) -> list[dict[str, Any]]:
start = datetime.strptime(normalize_date(start_date), "%Y%m%d").date()
end = datetime.strptime(normalize_date(end_date), "%Y%m%d").date()
if start > end:
raise ValueError("开始日期不能晚于结束日期。")
weekdays = []
current = start
while current <= end:
if current.weekday() < 5:
weekdays.append(current)
current += timedelta(days=1)
if len(weekdays) > 15:
raise ValueError("单次最多回补 15 个工作日。")
results = []
for day in weekdays:
dashboard = self.sync_dashboard(day.strftime("%Y%m%d"))
results.append(
{
"requested_date": day.isoformat(),
"trade_date": dashboard["meta"]["trade_date"],
"source": dashboard["meta"]["source"],
"records": self._record_count(dashboard),
}
)
return results
def _stock_identity(self, code: str, trade_date: str) -> tuple[str, str]:
snapshot = self.database.get_snapshot(trade_date) or {}
for key in ("limits", "broken", "down_limits"):
for row in snapshot.get(key) or []:
if str(row.get("code")) == code:
return row.get("name") or "--", row.get("sector") or "其他"
for item in self.database.list_watchlist(self.current_user_id):
if item["code"] == code:
return item["name"], item["sector"] or "其他"
return "--", "其他"
def _enrich_stock_detail(self, payload: dict[str, Any]) -> dict[str, Any]:
result = dict(payload)
stock = dict(payload.get("stock") or {})
code = str(stock.get("code") or "")
watched = {
item["code"]: item
for item in self.database.list_watchlist(self.current_user_id)
}
stock["watchlist"] = watched.get(code)
result["stock"] = stock
result["notes"] = self.database.list_notes(self.current_user_id, code=code)
return result
def _apply_reason_overrides(self, dashboard: dict[str, Any]) -> dict[str, Any]: def _apply_reason_overrides(self, dashboard: dict[str, Any]) -> dict[str, Any]:
trade_date = str(dashboard.get("meta", {}).get("trade_date", "")).replace("-", "") trade_date = str(dashboard.get("meta", {}).get("trade_date", "")).replace("-", "")
@@ -4578,22 +3671,6 @@ class DashboardService:
} }
return result return result
def _with_storage(self, dashboard: dict[str, Any], cached: bool) -> dict[str, Any]:
result = dict(dashboard)
result["meta"] = {
**dashboard.get("meta", {}),
"storage": "sqlite",
"cached": cached,
}
return result
@staticmethod
def _record_count(dashboard: dict[str, Any]) -> int:
return sum(
len(dashboard.get(key) or [])
for key in ("limits", "broken", "down_limits", "yesterday_limits")
)
SERVICE = DashboardService() SERVICE = DashboardService()
backend/bootstrap/container.py
+3 -3
View File
@@ -10,12 +10,12 @@ from backend.features.alerts import AlertService
from backend.features.review import TradeJournalService from backend.features.review import TradeJournalService
from backend.features.screener import StrategyTrackingService from backend.features.screener import StrategyTrackingService
from backend.jobs import InProcessJobRunner, JobRegistry, SQLiteJobRunRepository from backend.jobs import InProcessJobRunner, JobRegistry, SQLiteJobRunRepository
from chart_data_provider import MarketChartClient
from database import ReviewDatabase from database import ReviewDatabase
from ifind_client import IfindHttpClient
from mentor_agent import MentorSkillRegistry from mentor_agent import MentorSkillRegistry
from realtime_aggregator import WebRealtimeAggregator
from screener import ScreenerEngine from screener import ScreenerEngine
from backend.data.providers.ifind_client import IfindHttpClient
from backend.data.realtime import WebRealtimeAggregator
from backend.features.market.charts import MarketChartClient
@dataclass(frozen=True) @dataclass(frozen=True)
backend/data/__init__.py
+8 -1
View File
@@ -1,4 +1,3 @@
from .gateway import DataGateway, build_data_gateway
from .policy import DataPolicyError, DataSourcePolicy from .policy import DataPolicyError, DataSourcePolicy
from .quality import DataQualityError, DataQualityGate, QualityEvidence, QualityReport from .quality import DataQualityError, DataQualityGate, QualityEvidence, QualityReport
@@ -12,3 +11,11 @@ __all__ = [
"QualityReport", "QualityReport",
"build_data_gateway", "build_data_gateway",
] ]
def __getattr__(name: str):
if name in {"DataGateway", "build_data_gateway"}:
from .gateway import DataGateway, build_data_gateway
return {"DataGateway": DataGateway, "build_data_gateway": build_data_gateway}[name]
raise AttributeError(name)
backend/data/gateway.py
+4 -4
View File
@@ -8,10 +8,10 @@ from backend.data.contracts import DataUsage
from backend.data.policy import DataSourcePolicy from backend.data.policy import DataSourcePolicy
from backend.data.providers import IfindProvider, TushareProvider from backend.data.providers import IfindProvider, TushareProvider
from backend.data.quality import DataQualityGate, QualityEvidence, QualityReport from backend.data.quality import DataQualityGate, QualityEvidence, QualityReport
from chart_data_provider import EastmoneyChartClient, MarketChartClient from backend.data.providers.ifind_client import IfindHttpClient
from ifind_client import IfindHttpClient from backend.data.providers.tushare_client import TushareClient
from realtime_aggregator import WebRealtimeAggregator from backend.data.realtime import WebRealtimeAggregator
from tushare_client import TushareClient from backend.features.market.charts import EastmoneyChartClient, MarketChartClient
@dataclass(frozen=True) @dataclass(frozen=True)
backend/data/providers/ifind.py
+1 -1
View File
@@ -1,6 +1,6 @@
from __future__ import annotations from __future__ import annotations
from ifind_client import IfindHttpClient from backend.data.providers.ifind_client import IfindHttpClient
class IfindProvider: class IfindProvider:
backend/data/providers/ifind_client.py
+385
View File
@@ -0,0 +1,385 @@
from __future__ import annotations
import copy
import json
import threading
import time
import urllib.error
import urllib.request
from datetime import datetime, timedelta
from typing import Any
class IfindError(RuntimeError):
pass
class IfindHttpClient:
BASE_URL = "https://quantapi.51ifind.com/api/v1"
AUTH_ENDPOINT = "get_access_token"
AUTH_ERROR_CODES = {-1302, -1303, -1304, -4302, -4303}
def __init__(
self,
refresh_token: str = "",
access_token: str = "",
timeout: int = 15,
) -> None:
self.timeout = max(3, int(timeout))
self._refresh_token = str(refresh_token or "").strip()
self._access_token = str(access_token or "").strip()
self._access_expires_at: datetime | None = None
self._token_lock = threading.Lock()
self._cache_lock = threading.Lock()
self._cache: dict[str, dict[str, Any]] = {}
@property
def configured(self) -> bool:
return bool(self._refresh_token or self._access_token)
def set_credentials(self, refresh_token: str, access_token: str = "") -> None:
refresh_token = str(refresh_token or "").strip()
access_token = str(access_token or "").strip()
with self._token_lock:
refresh_changed = refresh_token != self._refresh_token
self._refresh_token = refresh_token
if access_token or refresh_changed:
self._access_token = access_token
self._access_expires_at = None
if refresh_changed:
with self._cache_lock:
self._cache.clear()
def status(self) -> dict[str, Any]:
return {
"configured": self.configured,
"access_ready": bool(self._access_token),
"access_expires_at": (
self._access_expires_at.isoformat(timespec="seconds")
if self._access_expires_at
else ""
),
}
def test_connection(self) -> dict[str, Any]:
payload = self.real_time(
"000001.SH",
["open", "high", "low", "latest", "preClose"],
cache_ttl=0,
)
return {
"ok": bool(payload),
"sample_time": str(payload[0].get("time") or "") if payload else "",
}
def real_time(
self,
codes: str | list[str],
indicators: list[str],
cache_ttl: int = 10,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"real_time_quotation",
{"codes": code_text, "indicators": ",".join(indicators)},
cache_key=f"rq:{code_text}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def history(
self,
codes: str | list[str],
indicators: list[str],
start_date: str,
end_date: str,
cache_ttl: int = 300,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"cmd_history_quotation",
{
"codes": code_text,
"indicators": ",".join(indicators),
"startdate": self._display_date(start_date),
"enddate": self._display_date(end_date),
"functionpara": {"CPS": "forward1", "Fill": "Omit"},
},
cache_key=f"hq:{code_text}:{start_date}:{end_date}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def intraday(
self,
code: str,
start_time: str,
end_time: str,
cache_ttl: int = 20,
) -> list[dict[str, Any]]:
indicators = ["open", "high", "low", "close", "volume", "amount", "avgPrice"]
payload = self._request(
"high_frequency",
{
"codes": self._codes(code),
"indicators": ",".join(indicators),
"starttime": start_time,
"endtime": end_time,
"functionpara": {
"CPS": "forward1",
"Fill": "Previous",
"Timeformat": "LocalTime",
"Interval": "1",
"Limitstart": "09:30:00",
"Limitend": "15:00:00",
},
},
cache_key=f"hf:{code}:{start_time}:{end_time}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def snapshots(
self,
codes: str | list[str],
indicators: list[str],
start_time: str,
end_time: str,
cache_ttl: int = 8,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"snap_shot",
{
"codes": code_text,
"indicators": ",".join(indicators),
"starttime": start_time,
"endtime": end_time,
},
cache_key=f"ss:{code_text}:{start_time}:{end_time}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def wencai(self, query: str, search_type: str = "stock", cache_ttl: int = 300) -> list[dict[str, Any]]:
normalized = " ".join(str(query or "").split())
if not normalized:
raise IfindError("问财查询不能为空。")
payload = self._request(
"smart_stock_picking",
{"searchstring": normalized, "searchtype": search_type},
cache_key=f"wc:{search_type}:{normalized}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def report_query(
self,
codes: str | list[str],
begin_date: str,
end_date: str,
cache_ttl: int = 300,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"report_query",
{
"codes": code_text,
"beginrDate": self._display_date(begin_date),
"endrDate": self._display_date(end_date),
"outputpara": (
"reportDate:Y,thscode:Y,secName:Y,ctime:Y,"
"reportTitle:Y,pdfURL:Y,seq:Y"
),
},
cache_key=f"report:{code_text}:{begin_date}:{end_date}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def _request(
self,
endpoint: str,
body: dict[str, Any],
cache_key: str = "",
cache_ttl: int = 0,
) -> dict[str, Any]:
if not self.configured:
raise IfindError("iFinD 尚未配置。")
if cache_key and cache_ttl > 0:
cached = self._cached(cache_key, cache_ttl)
if cached is not None:
return cached
payload = self._post(endpoint, body, self._ensure_access_token())
if self._is_auth_error(payload) and self._refresh_token:
self._invalidate_access_token()
payload = self._post(endpoint, body, self._ensure_access_token(force=True))
self._validate_payload(payload)
if cache_key and cache_ttl > 0:
with self._cache_lock:
self._cache[cache_key] = {
"created_at": time.time(),
"payload": copy.deepcopy(payload),
}
return payload
def _ensure_access_token(self, force: bool = False) -> str:
with self._token_lock:
now = datetime.now().astimezone().replace(tzinfo=None)
token_valid = bool(self._access_token) and (
self._access_expires_at is None
or self._access_expires_at > now + timedelta(minutes=2)
)
if token_valid and not force:
return self._access_token
if not self._refresh_token:
if self._access_token:
return self._access_token
raise IfindError("iFinD Refresh Token 尚未配置。")
payload = self._post(self.AUTH_ENDPOINT, {}, "", self._refresh_token)
self._validate_payload(payload)
data = payload.get("data") or {}
token = str(data.get("access_token") or "").strip()
if not token:
raise IfindError("iFinD 未返回 Access Token。")
expires_at = self._parse_datetime(data.get("expired_time"))
self._access_token = token
self._access_expires_at = expires_at
return token
def _post(
self,
endpoint: str,
body: dict[str, Any],
access_token: str,
refresh_token: str = "",
) -> dict[str, Any]:
headers = {
"Accept": "application/json",
"Content-Type": "application/json",
"User-Agent": "XiaobaiReviewWeb/1.0",
"ifindlang": "cn",
}
if access_token:
headers["access_token"] = access_token
if refresh_token:
headers["refresh_token"] = refresh_token
request = urllib.request.Request(
f"{self.BASE_URL}/{endpoint}",
data=json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
headers=headers,
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
payload = json.loads(response.read().decode("utf-8"))
except urllib.error.HTTPError as exc:
detail = ""
try:
detail_payload = json.loads(exc.read().decode("utf-8", errors="replace"))
detail = str(detail_payload.get("errmsg") or detail_payload.get("message") or "")
except (json.JSONDecodeError, OSError):
pass
raise IfindError(f"iFinD HTTP {exc.code}{f'：{detail[:160]}' if detail else ''}") from exc
except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
raise IfindError("iFinD 数据请求失败。") from exc
if not isinstance(payload, dict):
raise IfindError("iFinD 返回格式不正确。")
return payload
def _cached(self, key: str, ttl: int) -> dict[str, Any] | None:
with self._cache_lock:
cached = self._cache.get(key)
if not cached:
return None
if time.time() - float(cached.get("created_at") or 0) > ttl:
self._cache.pop(key, None)
return None
return copy.deepcopy(cached["payload"])
def _invalidate_access_token(self) -> None:
with self._token_lock:
self._access_token = ""
self._access_expires_at = None
@classmethod
def _validate_payload(cls, payload: dict[str, Any]) -> None:
try:
error_code = int(payload.get("errorcode") or 0)
except (TypeError, ValueError):
error_code = -1
if error_code != 0:
message = str(payload.get("errmsg") or "未知错误")
raise IfindError(f"iFinD 返回错误：{message[:200]}")
@classmethod
def _is_auth_error(cls, payload: dict[str, Any]) -> bool:
try:
error_code = int(payload.get("errorcode") or 0)
except (TypeError, ValueError):
error_code = 0
message = str(payload.get("errmsg") or "").casefold()
return error_code in cls.AUTH_ERROR_CODES or "token" in message or "鉴权" in message
@staticmethod
def _table_rows(payload: dict[str, Any]) -> list[dict[str, Any]]:
tables = payload.get("tables") or []
if isinstance(tables, dict):
tables = [tables]
rows: list[dict[str, Any]] = []
for block in tables if isinstance(tables, list) else []:
if not isinstance(block, dict):
continue
table = block.get("table") or {}
if not isinstance(table, dict):
continue
times = block.get("time") or []
codes = block.get("thscode") or block.get("thscodes") or []
if isinstance(codes, str):
codes = [codes]
lengths = [len(value) for value in table.values() if isinstance(value, list)]
row_count = max(lengths or [len(times) if isinstance(times, list) else 0, 1 if table else 0])
for index in range(row_count):
row: dict[str, Any] = {}
if isinstance(times, list) and index < len(times):
row["time"] = times[index]
if codes:
row["thscode"] = codes[index] if index < len(codes) else codes[0]
for field, values in table.items():
if isinstance(values, list):
row[field] = values[index] if index < len(values) else None
elif index == 0:
row[field] = values
rows.append(row)
return rows
@staticmethod
def _codes(codes: str | list[str]) -> str:
if isinstance(codes, list):
values = [str(code or "").strip().upper() for code in codes]
else:
values = [part.strip().upper() for part in str(codes or "").split(",")]
values = [value for value in values if value]
if not values:
raise IfindError("iFinD 证券代码不能为空。")
if len(values) > 100:
raise IfindError("iFinD 单次证券代码过多。")
return ",".join(values)
@staticmethod
def _display_date(value: str) -> str:
compact = str(value or "").replace("-", "")
if len(compact) != 8 or not compact.isdigit():
raise IfindError("iFinD 日期格式不正确。")
return f"{compact[:4]}-{compact[4:6]}-{compact[6:]}"
@staticmethod
def _parse_datetime(value: Any) -> datetime | None:
text = str(value or "").strip()
if not text:
return None
try:
return datetime.fromisoformat(text)
except ValueError:
return None
backend/data/providers/tushare.py
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
from collections.abc import Callable from collections.abc import Callable
from tushare_client import TushareClient from backend.data.providers.tushare_client import TushareClient
class TushareProvider: class TushareProvider:
backend/data/providers/tushare_client.py
+2175
View File
File diff suppressed because it is too large Load Diff
backend/data/realtime.py
+426
View File
@@ -0,0 +1,426 @@
from __future__ import annotations
import copy
import http.client
import json
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from datetime import datetime
from threading import Lock
from typing import Any, ClassVar
class RealtimeAggregateError(RuntimeError):
pass
EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get"
EASTMONEY_SECTOR_URL = "https://push2.eastmoney.com/api/qt/clist/get"
TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006"
THS_LIMIT_URL = "https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool"
XGB_POOL_URL = "https://flash-api.xuangubao.cn/api/pool/detail"
BROWSER_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/138.0.0.0 Safari/537.36"
)
@dataclass
class WebRealtimeAggregator:
timeout: int = 8
retry_attempts: int = 3
retry_delay_seconds: float = 0.2
response_cache_ttl_seconds: int = 90
_sector_cache: ClassVar[dict[str, Any]] = {}
_sector_cache_lock: ClassVar[Lock] = Lock()
_response_cache: ClassVar[dict[str, dict[str, Any]]] = {}
_response_cache_lock: ClassVar[Lock] = Lock()
def health_snapshot(self, sector: str = "") -> dict[str, Any]:
started = time.perf_counter()
sources: dict[str, dict[str, Any]] = {}
indices: list[dict[str, Any]] = []
sector_payload: dict[str, Any] | None = None
indices, sources["eastmoney_indices"] = self._capture(self.eastmoney_indices)
if sector.strip():
sector_payload, sources["eastmoney_sector"] = self._capture(
lambda: self.eastmoney_sector(sector)
)
ths_observation, sources["ths_limit_pool"] = self._capture(self.ths_limit_pool)
xgb_observation, sources["xgb_limit_pool"] = self._capture(self.xgb_limit_pool)
index_times = [int(item.get("quote_time_epoch") or 0) for item in indices or []]
now = datetime.now().astimezone()
max_skew = 120 if now.hour >= 15 else 15
index_consistent = bool(index_times) and max(index_times) - min(index_times) <= max_skew
ready = (
bool(indices)
and len(indices) == 3
and index_consistent
and (not sector.strip() or bool(sector_payload))
)
return {
"ready": ready,
"isolated": True,
"generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"indices": indices or [],
"index_consistent": index_consistent,
"sector": sector_payload,
"sources": sources,
"observations": {
"ths_limit_pool": ths_observation,
"xgb_limit_pool": xgb_observation,
},
"policy": {
"integration": "heaven_realtime_fallback",
"max_index_time_skew_seconds": max_skew,
"notice": "聚合源仅作为盘中观势的实时指数与板块外显，主行情快照仍由Tushare维护。",
},
}
def eastmoney_indices(self) -> list[dict[str, Any]]:
try:
payload = self._get_json(
EASTMONEY_INDEX_URL,
{
"secids": "1.000001,0.399001,0.399006",
"fltt": "2",
"invt": "2",
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124",
},
referer="https://quote.eastmoney.com/",
)
except RealtimeAggregateError:
return self.tencent_indices()
cache_meta = payload.get("_aggregate_cache") or {}
rows = list((payload.get("data") or {}).get("diff") or [])
result = []
for row in rows:
code = str(row.get("f12") or "")
if code not in {"000001", "399001", "399006"}:
continue
epoch = int(_number(row.get("f124")))
result.append(
{
"code": code,
"name": row.get("f14") or code,
"price": _number(row.get("f2")),
"change": _number(row.get("f3")),
"change_amount": _number(row.get("f4")),
"open": _number(row.get("f17")),
"high": _number(row.get("f15")),
"low": _number(row.get("f16")),
"previous_close": _number(row.get("f18")),
"amount_billion": round(_number(row.get("f6")) / 100000000, 2),
"quote_time_epoch": epoch,
"quote_time": (
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
if epoch else ""
),
"source": (
"eastmoney_push2_cache" if cache_meta else "eastmoney_push2"
),
"cache_age_seconds": cache_meta.get("age_seconds", 0),
}
)
if len(result) != 3:
raise RealtimeAggregateError(f"Eastmoney returned {len(result)}/3 indices")
return result
def tencent_indices(self) -> list[dict[str, Any]]:
raw, cache_age = self._get_text(
TENCENT_INDEX_URL,
referer="https://gu.qq.com/",
encoding="gb18030",
)
result = []
for line in raw.splitlines():
if '="' not in line:
continue
fields = line.split('="', 1)[1].rsplit('";', 1)[0].split("~")
if len(fields) < 38:
continue
code = fields[2]
if code not in {"000001", "399001", "399006"}:
continue
try:
quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S").astimezone()
except ValueError as exc:
raise RealtimeAggregateError(
f"Tencent returned invalid quote time for {code}"
) from exc
result.append(
{
"code": code,
"name": fields[1] or code,
"price": _number(fields[3]),
"change": _number(fields[32]),
"change_amount": _number(fields[31]),
"open": _number(fields[5]),
"high": _number(fields[33]),
"low": _number(fields[34]),
"previous_close": _number(fields[4]),
"amount_billion": round(_number(fields[37]) / 10000, 2),
"quote_time_epoch": int(quote_time.timestamp()),
"quote_time": quote_time.isoformat(timespec="seconds"),
"source": "tencent_qt_cache" if cache_age else "tencent_qt",
"cache_age_seconds": cache_age,
}
)
if len(result) != 3:
raise RealtimeAggregateError(f"Tencent returned {len(result)}/3 indices")
return result
def eastmoney_sector(self, query: str) -> dict[str, Any]:
target = _normalize_sector(query)
candidates = self._eastmoney_sector_catalog()
matched = _match_sector(candidates, target)
if not matched:
raise RealtimeAggregateError(f"Eastmoney sector not found: {query}")
epoch = int(_number(matched.get("f124")))
return {
"code": matched.get("f12") or "",
"name": matched.get("f14") or query,
"price": _number(matched.get("f2")),
"change": _number(matched.get("f3")),
"change_amount": _number(matched.get("f4")),
"turnover_rate": _number(matched.get("f8")),
"up_count": int(_number(matched.get("f104"))),
"down_count": int(_number(matched.get("f105"))),
"leader": matched.get("f128") or "--",
"leader_code": matched.get("f140") or "",
"leading_pct": _number(matched.get("f136")),
"quote_time_epoch": epoch,
"quote_time": (
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
if epoch else ""
),
"source": "eastmoney_push2",
"match_query": query,
}
def _eastmoney_sector_catalog(self) -> list[dict[str, Any]]:
now = time.time()
with self._sector_cache_lock:
cached = self._sector_cache.get("eastmoney")
if cached and now - float(cached.get("created_at") or 0) < 600:
return list(cached.get("rows") or [])
def load_page(page: int) -> list[dict[str, Any]]:
payload = self._get_json(
EASTMONEY_SECTOR_URL,
{
"pn": str(page),
"pz": "100",
"po": "1",
"np": "1",
"fltt": "2",
"invt": "2",
"fid": "f3",
"fs": "m:90+t:2",
"fields": "f12,f14,f2,f3,f4,f8,f104,f105,f128,f136,f140,f124",
},
referer="https://quote.eastmoney.com/center/boardlist.html",
)
return list((payload.get("data") or {}).get("diff") or [])
with ThreadPoolExecutor(max_workers=5) as executor:
pages = list(executor.map(load_page, range(1, 6)))
rows = [row for page in pages for row in page]
if not rows:
raise RealtimeAggregateError("Eastmoney sector catalog is empty")
with self._sector_cache_lock:
self._sector_cache["eastmoney"] = {"created_at": now, "rows": rows}
return rows
def ths_limit_pool(self) -> dict[str, Any]:
payload = self._get_json(
THS_LIMIT_URL,
{"page": "1", "limit": "3", "field": "199112"},
referer="https://data.10jqka.com.cn/limit_up/",
)
data = payload.get("data") or payload
return {
"available": True,
"keys": sorted(str(key) for key in data.keys()) if isinstance(data, dict) else [],
"source": "ths_web_dataapi",
}
def xgb_limit_pool(self) -> dict[str, Any]:
payload = self._get_json(
XGB_POOL_URL,
{"pool_name": "limit_up"},
referer="https://xuangubao.cn/",
)
data = payload.get("data") or {}
rows = data if isinstance(data, list) else data.get("pool") or data.get("list") or []
return {
"available": True,
"count": len(rows) if isinstance(rows, list) else 0,
"source": "xuangubao_web_api",
}
def _capture(self, operation):
started = time.perf_counter()
try:
value = operation()
return value, {
"ok": True,
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"error": "",
}
except Exception as exc:
return None, {
"ok": False,
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"error": str(exc)[:500],
}
def _get_json(
self,
url: str,
params: dict[str, str],
referer: str,
) -> dict[str, Any]:
request_url = f"{url}?{urllib.parse.urlencode(params)}"
last_error: Exception | None = None
attempts = max(1, int(self.retry_attempts))
for attempt in range(attempts):
request = urllib.request.Request(
request_url,
headers={
"Accept": "application/json,text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
content_type = response.headers.get("Content-Type", "")
raw = response.read().decode("utf-8", errors="replace")
if "json" not in content_type.lower() and not raw.lstrip().startswith(("{", "[")):
raise RealtimeAggregateError(
f"non-JSON response: {raw[:120].strip()}"
)
payload = json.loads(raw)
if not isinstance(payload, dict):
raise RealtimeAggregateError("unexpected response shape")
if payload.get("rc") not in (None, 0):
raise RealtimeAggregateError(f"provider rc={payload.get('rc')}")
with self._response_cache_lock:
self._response_cache[request_url] = {
"created_at": time.time(),
"payload": copy.deepcopy(payload),
}
return payload
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
json.JSONDecodeError,
RealtimeAggregateError,
) as exc:
last_error = exc
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
time.sleep(self.retry_delay_seconds * (attempt + 1))
now = time.time()
with self._response_cache_lock:
cached = self._response_cache.get(request_url)
cache_age = now - float((cached or {}).get("created_at") or 0)
if cached and cache_age <= self.response_cache_ttl_seconds:
payload = copy.deepcopy(cached.get("payload") or {})
payload["_aggregate_cache"] = {"age_seconds": round(cache_age, 1)}
return payload
raise RealtimeAggregateError(f"request failed after {attempts} attempts: {last_error}") from last_error
def _get_text(
self,
request_url: str,
referer: str,
encoding: str = "utf-8",
) -> tuple[str, float]:
cache_key = f"text:{request_url}"
last_error: Exception | None = None
attempts = max(1, int(self.retry_attempts))
for attempt in range(attempts):
request = urllib.request.Request(
request_url,
headers={
"Accept": "text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
raw = response.read().decode(encoding, errors="replace")
if not raw.strip():
raise RealtimeAggregateError("empty text response")
with self._response_cache_lock:
self._response_cache[cache_key] = {
"created_at": time.time(),
"payload": raw,
}
return raw, 0
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
RealtimeAggregateError,
) as exc:
last_error = exc
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
time.sleep(self.retry_delay_seconds * (attempt + 1))
now = time.time()
with self._response_cache_lock:
cached = self._response_cache.get(cache_key)
cache_age = now - float((cached or {}).get("created_at") or 0)
if cached and cache_age <= self.response_cache_ttl_seconds:
return str(cached.get("payload") or ""), round(cache_age, 1)
raise RealtimeAggregateError(
f"text request failed after {attempts} attempts: {last_error}"
) from last_error
def _normalize_sector(value: Any) -> str:
text = str(value or "").strip().replace(" ", "")
for suffix in ("板块", "概念", "行业", "Ⅱ", "Ⅲ", "(A股)", "（A股）"):
text = text.replace(suffix, "")
aliases = {"元器件": "元件", "电子元器件": "元件"}
return aliases.get(text, text)
def _match_sector(rows: list[dict[str, Any]], target: str) -> dict[str, Any] | None:
exact = [row for row in rows if _normalize_sector(row.get("f14")) == target]
if exact:
return min(exact, key=lambda row: len(str(row.get("f14") or "")))
fuzzy = [
row for row in rows
if target and (
target in _normalize_sector(row.get("f14"))
or _normalize_sector(row.get("f14")) in target
)
]
return min(fuzzy, key=lambda row: len(_normalize_sector(row.get("f14")))) if fuzzy else None
def _number(value: Any, default: float = 0.0) -> float:
try:
return float(value)
except (TypeError, ValueError):
return default
backend/features/market/__init__.py
+13
View File
@@ -0,0 +1,13 @@
"""Public market data, search, detail and chart feature."""
from .charts import ChartDataError, EastmoneyChartClient, MarketChartClient
from .repository import MarketRepositoryMixin
from .service import MarketServiceMixin
__all__ = [
"ChartDataError",
"EastmoneyChartClient",
"MarketChartClient",
"MarketRepositoryMixin",
"MarketServiceMixin",
]
backend/features/market/charts.py
+497
View File
@@ -0,0 +1,497 @@
from __future__ import annotations
import http.client
import json
import re
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from datetime import datetime, time as dt_time, timedelta
from threading import Lock
from typing import Any, ClassVar
from backend.data.providers.ifind_client import IfindError, IfindHttpClient
class ChartDataError(RuntimeError):
pass
TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get"
BOARD_LIST_URL = "https://push2delay.eastmoney.com/api/qt/clist/get"
BROWSER_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/138.0.0.0 Safari/537.36"
)
INDEX_SECIDS = {
"000001.SH": "1.000001",
"399001.SZ": "0.399001",
"399006.SZ": "0.399006",
}
class MarketChartClient:
"""Prefer iFinD for display charts and retain Eastmoney as a last resort."""
def __init__(self, ifind: IfindHttpClient, fallback: "EastmoneyChartClient") -> None:
self.ifind = ifind
self.fallback = fallback
def stock_intraday(self, code: str) -> dict[str, Any]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
ifind_code = _stock_market_code(normalized)
try:
return self._ifind_intraday(ifind_code, "stock", normalized)
except (IfindError, ChartDataError):
return self.fallback.stock_intraday(normalized)
def stock_daily(self, code: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
return self._ifind_daily(_stock_market_code(normalized), end_date, limit)
def index_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(identifier or "").strip().upper()
if normalized not in INDEX_SECIDS:
raise ChartDataError("Unsupported index")
return self._ifind_daily(normalized, end_date, limit)
def board_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(identifier or "").strip().upper()
if not normalized:
raise ChartDataError("Invalid board code")
return self._ifind_daily(normalized, end_date, limit)
def index_intraday(self, identifier: str) -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
if normalized not in INDEX_SECIDS:
raise ChartDataError("Unsupported index")
try:
return self._ifind_intraday(normalized, "index", normalized)
except (IfindError, ChartDataError):
return self.fallback.index_intraday(normalized)
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
try:
return self._ifind_intraday(normalized, "board", normalized, name)
except (IfindError, ChartDataError):
return self.fallback.board_intraday(normalized, name)
def _ifind_intraday(
self,
ifind_code: str,
entity_type: str,
identifier: str,
name: str = "",
) -> dict[str, Any]:
if not self.ifind.configured:
raise ChartDataError("iFinD is not configured")
now = datetime.now().astimezone()
rows: list[dict[str, Any]] = []
for offset in range(0, 8):
candidate = now.date() - timedelta(days=offset)
if candidate.weekday() >= 5:
continue
display_date = candidate.isoformat()
rows = self.ifind.intraday(
ifind_code,
f"{display_date} 09:30:00",
f"{display_date} 15:00:00",
cache_ttl=20 if offset == 0 else 6 * 60 * 60,
)
if rows:
break
points = [point for row in rows if (point := _ifind_point(row))]
if not points:
raise ChartDataError("No iFinD intraday chart data returned")
latest_date = points[-1]["date"]
points = [point for point in points if point["date"] == latest_date]
previous_close = self._previous_close(ifind_code, latest_date, points[0]["open"])
return {
"entity_type": entity_type,
"identifier": identifier,
"name": name,
"code": identifier,
"trade_date": latest_date,
"previous_close": previous_close,
"points": points,
"source": "ifind",
}
def _ifind_daily(
self, ifind_code: str, end_date: str, limit: int
) -> list[dict[str, Any]]:
if not self.ifind.configured:
raise ChartDataError("iFinD is not configured")
compact_end = str(end_date or "").replace("-", "")
if not re.fullmatch(r"\d{8}", compact_end):
raise ChartDataError("Invalid chart end date")
end = datetime.strptime(compact_end, "%Y%m%d")
start = (end - timedelta(days=max(190, limit * 3))).strftime("%Y%m%d")
try:
rows = self.ifind.history(
ifind_code,
["open", "high", "low", "close", "volume", "amount"],
start,
compact_end,
cache_ttl=300,
)
except IfindError as exc:
raise ChartDataError("No iFinD daily chart data returned") from exc
normalized = []
for row in rows:
stamp = str(row.get("time") or "").strip()
trade_date = stamp[:10]
close = _number(row.get("close"))
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", trade_date) or close <= 0:
continue
normalized.append(
{
"trade_date": trade_date,
"open": _number(row.get("open")),
"high": _number(row.get("high")),
"low": _number(row.get("low")),
"close": close,
"volume": _number(row.get("volume")),
"amount_billion": _number(row.get("amount")) / 100_000_000,
}
)
normalized.sort(key=lambda row: row["trade_date"])
for index, row in enumerate(normalized):
previous = normalized[index - 1]["close"] if index > 0 else 0
row["change"] = round((row["close"] / previous - 1) * 100, 4) if previous else 0.0
market_now = datetime.now().astimezone()
today = market_now.strftime("%Y%m%d")
market_open = (
market_now.weekday() < 5
and market_now.time().replace(tzinfo=None) >= dt_time(9, 30)
)
today_display = market_now.date().isoformat()
if normalized and normalized[-1]["trade_date"] == today_display:
current_bar = normalized[-1]
current_bar_is_valid = (
current_bar["open"] > 0
and current_bar["high"] >= max(current_bar["open"], current_bar["close"])
and 0 < current_bar["low"] <= min(current_bar["open"], current_bar["close"])
and (current_bar["volume"] > 0 or current_bar["amount_billion"] > 0)
)
if not market_open or not current_bar_is_valid:
normalized.pop()
if compact_end == today and market_open:
try:
quote_rows = self.ifind.real_time(
ifind_code,
["open", "high", "low", "latest", "preClose", "volume", "amount"],
cache_ttl=10,
)
quote = quote_rows[0] if quote_rows else {}
latest = _number(quote.get("latest"))
previous = _number(quote.get("preClose"))
open_price = _number(quote.get("open"))
high = _number(quote.get("high"))
low = _number(quote.get("low"))
volume = _number(quote.get("volume"))
amount = _number(quote.get("amount"))
quote_date = str(quote.get("time") or "")[:10].replace("-", "")
quote_is_current = not quote_date or quote_date == today
has_market_activity = volume > 0 or amount > 0
if (
latest > 0
and open_price > 0
and high >= max(open_price, latest)
and 0 < low <= min(open_price, latest)
and has_market_activity
and quote_is_current
):
realtime = {
"trade_date": end.strftime("%Y-%m-%d"),
"open": open_price,
"high": high,
"low": low,
"close": latest,
"change": round((latest / previous - 1) * 100, 4) if previous else 0.0,
"volume": volume,
"amount_billion": amount / 100_000_000,
"realtime": True,
}
if normalized and normalized[-1]["trade_date"] == realtime["trade_date"]:
normalized[-1] = realtime
else:
normalized.append(realtime)
except IfindError:
pass
if not normalized:
raise ChartDataError("No iFinD daily chart data returned")
return normalized[-max(20, min(180, int(limit))):]
def _previous_close(self, code: str, trade_date: str, fallback: float) -> float:
today = datetime.now().astimezone().date().isoformat()
if trade_date == today:
try:
quote = self.ifind.real_time(code, ["preClose"], cache_ttl=20)
value = _number((quote[0] if quote else {}).get("preClose"))
if value > 0:
return value
except IfindError:
pass
end = datetime.strptime(trade_date, "%Y-%m-%d")
try:
rows = self.ifind.history(
code,
["close"],
(end - timedelta(days=12)).strftime("%Y%m%d"),
end.strftime("%Y%m%d"),
cache_ttl=6 * 60 * 60,
)
closes = [_number(row.get("close")) for row in rows if _number(row.get("close")) > 0]
if len(closes) >= 2:
return closes[-2]
except IfindError:
pass
return fallback
@dataclass
class EastmoneyChartClient:
"""Isolated display-only minute chart source.
The returned data must not be used by market snapshots, scoring, screening,
or divination. Its only consumer is a chart-rendering endpoint.
"""
timeout: int = 6
cache_ttl_seconds: int = 20
retry_attempts: int = 2
_cache: ClassVar[dict[str, dict[str, Any]]] = {}
_cache_lock: ClassVar[Lock] = Lock()
_board_catalog: ClassVar[dict[str, dict[str, str]]] = {}
_board_catalog_at: ClassVar[float] = 0.0
_board_catalog_lock: ClassVar[Lock] = Lock()
def stock_intraday(self, code: str) -> dict[str, Any]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
market = "1" if normalized.startswith(("5", "6", "9")) else "0"
return self._intraday(f"{market}.{normalized}", "stock", normalized)
def index_intraday(self, identifier: str) -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
secid = INDEX_SECIDS.get(normalized)
if not secid:
raise ChartDataError("Unsupported index")
return self._intraday(secid, "index", normalized)
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
if re.fullmatch(r"BK\d{4}", normalized):
board_code = normalized
else:
board_code = self._resolve_board_code(name or identifier)
return self._intraday(f"90.{board_code}", "board", board_code)
def _intraday(self, secid: str, entity_type: str, identifier: str) -> dict[str, Any]:
cache_key = f"{entity_type}:{identifier}"
cached = self._get_cached(cache_key)
if cached is not None:
return cached
payload = self._request_json(
TRENDS_URL,
{
"secid": secid,
"fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13",
"fields2": "f51,f52,f53,f54,f55,f56,f57,f58",
"iscr": "0",
"ndays": "1",
},
"https://quote.eastmoney.com/",
)
data = payload.get("data") or {}
points = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))]
if not points:
raise ChartDataError("No intraday chart data returned")
result = {
"entity_type": entity_type,
"identifier": identifier,
"name": str(data.get("name") or ""),
"code": str(data.get("code") or identifier),
"trade_date": points[-1]["date"],
"previous_close": _number(data.get("preClose")),
"points": points,
}
with self._cache_lock:
self._cache[cache_key] = {"created_at": time.time(), "payload": result}
return result
def _get_cached(self, cache_key: str) -> dict[str, Any] | None:
with self._cache_lock:
cached = self._cache.get(cache_key)
if not cached:
return None
if time.time() - float(cached.get("created_at") or 0) > self.cache_ttl_seconds:
with self._cache_lock:
self._cache.pop(cache_key, None)
return None
return dict(cached["payload"])
def _resolve_board_code(self, name: str) -> str:
normalized = _normalize_name(name)
if not normalized:
raise ChartDataError("Board name is required")
catalog = self._load_board_catalog()
item = catalog.get(normalized)
if not item:
raise ChartDataError("No matching chart board")
return item["code"]
def _load_board_catalog(self) -> dict[str, dict[str, str]]:
now = time.time()
with self._board_catalog_lock:
if self._board_catalog and now - self._board_catalog_at < 6 * 60 * 60:
return dict(self._board_catalog)
rows: list[dict[str, Any]] = []
for board_type in ("1", "2", "3"):
for page in range(1, 6):
payload = self._request_json(
BOARD_LIST_URL,
{
"pn": str(page),
"pz": "100",
"po": "1",
"np": "1",
"fltt": "2",
"invt": "2",
"fid": "f3",
"fs": f"m:90+t:{board_type}",
"fields": "f12,f14",
},
"https://quote.eastmoney.com/center/boardlist.html",
)
page_rows = (payload.get("data") or {}).get("diff") or []
rows.extend(page_rows)
if len(page_rows) < 100:
break
catalog: dict[str, dict[str, str]] = {}
for row in rows:
code = str(row.get("f12") or "").strip().upper()
board_name = str(row.get("f14") or "").strip()
if re.fullmatch(r"BK\d{4}", code) and board_name:
catalog.setdefault(_normalize_name(board_name), {"code": code, "name": board_name})
if not catalog:
raise ChartDataError("Board chart directory is unavailable")
with self._board_catalog_lock:
type(self)._board_catalog = catalog
type(self)._board_catalog_at = now
return dict(catalog)
def _request_json(
self, url: str, params: dict[str, str], referer: str
) -> dict[str, Any]:
request_url = f"{url}?{urllib.parse.urlencode(params)}"
last_error: Exception | None = None
for attempt in range(max(1, int(self.retry_attempts))):
request = urllib.request.Request(
request_url,
headers={
"Accept": "application/json,text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
payload = json.loads(response.read().decode("utf-8"))
if not isinstance(payload, dict):
raise ChartDataError("Invalid intraday chart response")
return payload
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
json.JSONDecodeError,
ChartDataError,
) as exc:
last_error = exc
if attempt + 1 < self.retry_attempts:
time.sleep(0.12)
raise ChartDataError("Intraday chart request failed") from last_error
def _parse_trend(raw: Any) -> dict[str, Any] | None:
fields = str(raw or "").split(",")
if len(fields) < 8 or " " not in fields[0]:
return None
stamp = fields[0].strip()
trade_date, trade_time = stamp.split(" ", 1)
close = _number(fields[2])
if close <= 0:
return None
return {
"date": trade_date,
"time": trade_time[:5],
"open": _number(fields[1]),
"close": close,
"high": _number(fields[3]),
"low": _number(fields[4]),
"volume": _number(fields[5]),
"amount": _number(fields[6]),
"average": _number(fields[7]),
}
def _ifind_point(row: dict[str, Any]) -> dict[str, Any] | None:
stamp = str(row.get("time") or "").strip()
if " " not in stamp:
return None
trade_date, trade_time = stamp.split(" ", 1)
close = _number(row.get("close"))
if close <= 0:
return None
return {
"date": trade_date,
"time": trade_time[:5],
"open": _number(row.get("open")),
"close": close,
"high": _number(row.get("high")),
"low": _number(row.get("low")),
"volume": _number(row.get("volume")),
"amount": _number(row.get("amount")),
"average": _number(row.get("avgPrice")),
}
def _stock_market_code(code: str) -> str:
if code.startswith(("4", "8", "9")):
suffix = "BJ"
elif code.startswith("6"):
suffix = "SH"
else:
suffix = "SZ"
return f"{code}.{suffix}"
def _number(value: Any) -> float:
try:
return float(value or 0)
except (TypeError, ValueError):
return 0.0
def _normalize_name(value: Any) -> str:
normalized = re.sub(r"[\s·・（）()\-_/]", "", str(value or "")).casefold()
return re.sub(r"(?:概念|行业|[ⅠⅡⅢ])$", "", normalized)
backend/features/market/repository.py
+222
View File
@@ -0,0 +1,222 @@
from __future__ import annotations
import json
from datetime import datetime
from typing import Any
class MarketRepositoryMixin:
def get_snapshot(self, trade_date: str) -> dict[str, Any] | None:
with self.connect() as connection:
row = connection.execute(
"SELECT payload FROM dashboard_snapshots WHERE trade_date = ?",
(trade_date,),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def get_latest_real_snapshot(
self, trade_date: str, strictly_before: bool = False
) -> dict[str, Any] | None:
operator = "<" if strictly_before else "<="
with self.connect() as connection:
row = connection.execute(
f"""
SELECT payload FROM dashboard_snapshots
WHERE trade_date {operator} ? AND source != 'demo'
ORDER BY trade_date DESC LIMIT 1
""",
(trade_date,),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def save_snapshot(self, trade_date: str, source: str, payload: dict[str, Any]) -> None:
updated_at = datetime.now().astimezone().isoformat(timespec="seconds")
record_count = sum(
len(payload.get(key) or [])
for key in ("limits", "broken", "down_limits", "yesterday_limits")
)
content = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
with self.connect() as connection:
connection.execute(
"""
INSERT INTO dashboard_snapshots
(trade_date, source, payload, record_count, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(trade_date) DO UPDATE SET
source = excluded.source,
payload = excluded.payload,
record_count = excluded.record_count,
updated_at = excluded.updated_at
""",
(trade_date, source, content, record_count, updated_at),
)
def get_data_snapshot(self, kind: str, cache_key: str) -> dict[str, Any] | None:
with self.connect() as connection:
row = connection.execute(
"SELECT payload FROM data_snapshots WHERE kind = ? AND cache_key = ?",
(kind, cache_key),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def get_latest_data_snapshot(
self,
kind: str,
cache_key_prefix: str,
maximum_cache_key: str,
exclude_source: str = "",
) -> dict[str, Any] | None:
source_clause = " AND source != ?" if exclude_source else ""
parameters: list[Any] = [kind, f"{cache_key_prefix}%", maximum_cache_key]
if exclude_source:
parameters.append(exclude_source)
with self.connect() as connection:
row = connection.execute(
f"""
SELECT payload FROM data_snapshots
WHERE kind = ? AND cache_key LIKE ? AND cache_key <= ?{source_clause}
ORDER BY cache_key DESC LIMIT 1
""",
parameters,
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def save_data_snapshot(
self, kind: str, cache_key: str, source: str, payload: dict[str, Any]
) -> None:
updated_at = datetime.now().astimezone().isoformat(timespec="seconds")
content = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
with self.connect() as connection:
connection.execute(
"""
INSERT INTO data_snapshots (kind, cache_key, source, payload, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(kind, cache_key) DO UPDATE SET
source = excluded.source,
payload = excluded.payload,
updated_at = excluded.updated_at
""",
(kind, cache_key, source, content, updated_at),
)
def search_stock_master(self, query: str, limit: int = 12) -> list[dict[str, Any]]:
text = str(query or "").strip()
if not text:
return []
escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
with self.connect() as connection:
rows = connection.execute(
"""
SELECT ts_code, code, name, industry, market, list_date
FROM stock_master
WHERE code = ? OR name = ? OR name LIKE ? ESCAPE '\\'
ORDER BY
CASE WHEN code = ? THEN 0 WHEN name = ? THEN 1 ELSE 2 END,
list_date DESC,
code
LIMIT ?
""",
(text, text, f"%{escaped}%", text, text, max(1, min(30, int(limit)))),
).fetchall()
return [dict(row) for row in rows]
def list_snapshot_payloads(self, end_date: str, limit: int = 260) -> list[dict[str, Any]]:
with self.connect() as connection:
rows = connection.execute(
"""
SELECT trade_date, payload FROM dashboard_snapshots
WHERE trade_date <= ? ORDER BY trade_date DESC LIMIT ?
""",
(end_date, limit),
).fetchall()
result: list[dict[str, Any]] = []
for row in reversed(rows):
try:
payload = json.loads(row["payload"])
except json.JSONDecodeError:
continue
payload["_snapshot_date"] = row["trade_date"]
result.append(payload)
return result
def start_sync(self, trade_date: str, source: str) -> int:
started_at = datetime.now().astimezone().isoformat(timespec="seconds")
with self.connect() as connection:
cursor = connection.execute(
"""
INSERT INTO sync_runs (trade_date, source, status, started_at)
VALUES (?, ?, 'running', ?)
""",
(trade_date, source, started_at),
)
return int(cursor.lastrowid)
def finish_sync(
self,
sync_id: int,
status: str,
record_count: int = 0,
message: str = "",
source: str | None = None,
) -> None:
finished_at = datetime.now().astimezone().isoformat(timespec="seconds")
with self.connect() as connection:
connection.execute(
"""
UPDATE sync_runs
SET status = ?, finished_at = ?, record_count = ?, message = ?,
source = COALESCE(?, source)
WHERE id = ?
""",
(status, finished_at, record_count, message[:1000], source, sync_id),
)
def status(self) -> dict[str, Any]:
with self.connect() as connection:
last_sync = connection.execute(
"""
SELECT id, trade_date, source, status, started_at, finished_at,
record_count, message
FROM sync_runs ORDER BY id DESC LIMIT 1
"""
).fetchone()
snapshot_stats = connection.execute(
"""
SELECT COUNT(*) AS dates, COALESCE(SUM(record_count), 0) AS records,
MAX(updated_at) AS updated_at
FROM dashboard_snapshots
"""
).fetchone()
watchlist_count = connection.execute("SELECT COUNT(*) FROM watchlist").fetchone()[0]
note_count = connection.execute("SELECT COUNT(*) FROM review_notes").fetchone()[0]
return {
"database": str(self.path.name),
"snapshot_dates": int(snapshot_stats["dates"]),
"snapshot_records": int(snapshot_stats["records"]),
"updated_at": snapshot_stats["updated_at"],
"last_sync": dict(last_sync) if last_sync else None,
"watchlist_count": int(watchlist_count),
"note_count": int(note_count),
}
backend/features/market/service.py
+949
View File
@@ -0,0 +1,949 @@
from __future__ import annotations
import copy
import re
from datetime import date, datetime, time as dt_time, timedelta
from typing import Any
from backend.bootstrap.config import (
normalize_date,
tushare_code,
validate_stock_code,
validate_text,
)
from backend.data.providers.ifind_client import IfindError
from backend.data.providers.tushare_client import TushareClient, TushareError
from backend.features.market.charts import ChartDataError
from sentiment_engine import SENTIMENT_ENGINE_VERSION
SEARCH_INDEXES = (
{"id": "000001.SH", "code": "000001.SH", "name": "上证指数", "type": "index", "subtitle": "沪市综合指数"},
{"id": "399001.SZ", "code": "399001.SZ", "name": "深证成指", "type": "index", "subtitle": "深市成份指数"},
{"id": "399006.SZ", "code": "399006.SZ", "name": "创业板指", "type": "index", "subtitle": "创业板核心指数"},
)
SEARCH_TYPE_LABELS = {
"stock": "股票",
"sector": "板块",
"theme": "题材",
"index": "指数",
}
THS_SEARCH_TYPES = {
"I": ("sector", "行业板块"),
"R": ("sector", "地域板块"),
"N": ("theme", "概念题材"),
}
class MarketServiceMixin:
def _tushare_client(self) -> TushareClient:
gateway = getattr(self, "data_gateway", None)
if gateway is not None:
return gateway.tushare()
# Compatibility for isolated legacy unit-test service stubs.
return TushareClient(self.token)
def get_dashboard(self, trade_date: str, force: bool = False) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
now = datetime.now().astimezone()
if (
normalized_date == now.strftime("%Y%m%d")
and now.time().replace(tzinfo=None) < datetime.strptime("09:15", "%H:%M").time()
):
previous = self.database.get_latest_real_snapshot(normalized_date, strictly_before=True)
if previous:
carried = self._carry_dashboard(previous, normalized_date, "盘前沿用最近交易日收盘行情")
return self._apply_reason_overrides(self._with_storage(carried, cached=True))
if not force:
snapshot = self.database.get_snapshot(normalized_date)
if snapshot and str((snapshot.get("meta") or {}).get("source") or "") != "demo":
snapshot = copy.deepcopy(snapshot)
if normalized_date != now.strftime("%Y%m%d"):
snapshot.setdefault("meta", {}).update(
{"realtime": False, "market_status": "closed"}
)
if not self._dashboard_sentiment_ready(snapshot):
snapshot = self._enrich_dashboard_sentiment(snapshot, normalized_date)
self.database.save_snapshot(
normalized_date,
str((snapshot.get("meta") or {}).get("source") or "tushare"),
snapshot,
)
snapshot.setdefault("meta", {})["requested_date"] = self._display_compact_date(normalized_date)
return self._apply_reason_overrides(self._with_storage(snapshot, cached=True))
resolved = self.database.get_data_snapshot(
"dashboard_request_v1", normalized_date
)
if resolved and str((resolved.get("meta") or {}).get("source") or "") != "demo":
resolved = copy.deepcopy(resolved)
resolved.setdefault("meta", {})["requested_date"] = self._display_compact_date(
normalized_date
)
return self._apply_reason_overrides(
self._with_storage(resolved, cached=True)
)
if datetime.strptime(normalized_date, "%Y%m%d").weekday() >= 5:
previous = self.database.get_latest_real_snapshot(normalized_date)
if previous:
carried = self._carry_dashboard(
previous,
normalized_date,
"非交易日沿用最近交易日收盘行情",
)
self.database.save_data_snapshot(
"dashboard_request_v1", normalized_date, "sqlite", carried
)
return self._apply_reason_overrides(
self._with_storage(carried, cached=True)
)
return self.sync_dashboard(normalized_date)
@staticmethod
def _dashboard_sentiment_ready(dashboard: dict[str, Any]) -> bool:
overview = dashboard.get("overview") or {}
return int(overview.get("sentiment_engine_version") or 0) == SENTIMENT_ENGINE_VERSION and all(
key in overview
for key in (
"sentiment_score",
"sentiment_label",
"sentiment_phase",
"sentiment_direction",
"sentiment_components",
)
)
@staticmethod
def _display_compact_date(compact: str) -> str:
return f"{compact[:4]}-{compact[4:6]}-{compact[6:8]}"
def _carry_dashboard(
self, snapshot: dict[str, Any], requested_date: str, reason: str
) -> dict[str, Any]:
carried = copy.deepcopy(snapshot)
meta = carried.setdefault("meta", {})
meta.update(
{
"requested_date": self._display_compact_date(requested_date),
"carried_forward": True,
"realtime": False,
"market_status": "closed",
"notice": reason,
}
)
return carried
def _realtime_snapshot_due(
self,
normalized_date: str,
snapshot: dict[str, Any],
) -> bool:
if not self.configured or normalized_date != date.today().strftime("%Y%m%d"):
return False
now = datetime.now().astimezone()
local_time = now.time().replace(tzinfo=None)
realtime_start = datetime.strptime("09:15", "%H:%M").time()
morning_end = datetime.strptime("11:35", "%H:%M").time()
afternoon_start = datetime.strptime("12:55", "%H:%M").time()
realtime_end = datetime.strptime("15:05", "%H:%M").time()
in_session = (
realtime_start <= local_time < morning_end
or afternoon_start <= local_time < realtime_end
)
if not in_session:
return False
meta = snapshot.get("meta") or {}
snapshot_trade_date = str(meta.get("trade_date") or "").replace("-", "")
if snapshot_trade_date and snapshot_trade_date != normalized_date:
return False
if not meta.get("realtime"):
return True
try:
updated_at = datetime.fromisoformat(str(meta.get("updated_at") or ""))
if updated_at.tzinfo is None:
updated_at = updated_at.replace(tzinfo=now.tzinfo)
except ValueError:
return True
age_seconds = (now - updated_at.astimezone(now.tzinfo)).total_seconds()
return age_seconds >= 8
def sync_dashboard(self, trade_date: str) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
source = "tushare"
with self.sync_lock:
sync_id = self.database.start_sync(normalized_date, source)
try:
if not self.configured:
raise TushareError("公共行情尚未配置")
dashboard = self._tushare_client().dashboard(normalized_date)
dashboard["meta"]["source"] = source
dashboard["meta"]["requested_date"] = self._display_compact_date(normalized_date)
dashboard = self._enrich_dashboard_sentiment(dashboard, normalized_date)
record_count = self._record_count(dashboard)
actual_date = normalize_date(
str(dashboard.get("meta", {}).get("trade_date") or normalized_date)
)
self.database.save_snapshot(actual_date, source, dashboard)
if actual_date != normalized_date:
dashboard.setdefault("meta", {}).update(
{
"carried_forward": True,
"realtime": False,
"market_status": "closed",
}
)
self.database.save_data_snapshot(
"dashboard_request_v1", normalized_date, source, dashboard
)
self.database.finish_sync(
sync_id,
"success",
record_count,
dashboard.get("meta", {}).get("notice", ""),
source,
)
return self._apply_reason_overrides(self._with_storage(dashboard, cached=False))
except TushareError as exc:
fallback = self.database.get_latest_real_snapshot(normalized_date)
if fallback:
carried = self._carry_dashboard(
fallback, normalized_date, f"最新行情暂不可用，沿用最近收盘快照：{exc}"
)
self.database.finish_sync(
sync_id, "fallback", self._record_count(carried), str(exc), "tushare"
)
return self._apply_reason_overrides(self._with_storage(carried, cached=True))
self.database.finish_sync(sync_id, "failed", message=str(exc))
raise ValueError("暂无可用的真实行情快照，请等待后台完成首次同步。") from exc
except Exception as exc:
self.database.finish_sync(sync_id, "failed", message=str(exc))
raise
def realtime_aggregate_health(self, sector: str = "") -> dict[str, Any]:
sector = validate_text(sector, "板块名称", 50)
return self.realtime_aggregator.health_snapshot(sector)
def _search_market_directory(self) -> list[dict[str, Any]]:
cached = self.database.get_data_snapshot("search_directory", "ths") or {}
cached_items = list(cached.get("items") or [])
if cached_items and int(cached.get("schema_version") or 0) >= 2:
return cached_items
if not self.configured:
return cached_items
try:
rows = self._tushare_client().query(
"ths_index",
{},
"ts_code,name,count,exchange,list_date,type",
)
except TushareError:
return cached_items
items = []
for row in rows:
mapping = THS_SEARCH_TYPES.get(str(row.get("type") or "").upper())
code = str(row.get("ts_code") or "").strip().upper()
name = str(row.get("name") or "").strip()
if not mapping or not code or not name or str(row.get("exchange") or "").upper() != "A":
continue
entity_type, subtitle = mapping
items.append(
{
"id": code,
"code": code,
"name": name,
"type": entity_type,
"subtitle": subtitle,
"member_count": int(float(row.get("count") or 0)),
}
)
if items:
self.database.save_data_snapshot(
"search_directory", "ths", "tushare", {"schema_version": 2, "items": items}
)
return items
@staticmethod
def _search_match_score(item: dict[str, Any], query: str) -> tuple[int, int, str]:
name = str(item.get("name") or "").casefold()
code = str(item.get("code") or item.get("id") or "").casefold()
needle = query.casefold()
if code == needle:
rank = 0
elif name == needle:
rank = 1
elif code.startswith(needle):
rank = 2
elif name.startswith(needle):
rank = 3
else:
rank = 4
return rank, len(name), code
def search_entities(self, query: str, trade_date: str) -> dict[str, Any]:
needle = str(query or "").strip()
normalized_date = normalize_date(trade_date)
groups: dict[str, list[dict[str, Any]]] = {
"stocks": [],
"sectors": [],
"themes": [],
"indices": [],
}
if not needle:
return {"query": "", "trade_date": normalized_date, "groups": groups}
stocks = []
for row in self.database.search_stock_master(needle, 12):
stocks.append(
{
"id": str(row.get("code") or ""),
"code": str(row.get("code") or ""),
"name": str(row.get("name") or "--"),
"type": "stock",
"type_label": SEARCH_TYPE_LABELS["stock"],
"industry": str(row.get("industry") or "其他"),
"market": str(row.get("market") or ""),
"subtitle": " · ".join(
part for part in (str(row.get("industry") or ""), str(row.get("market") or "")) if part
) or "A股",
}
)
groups["stocks"] = stocks[:8]
market_items = list(self._search_market_directory()) + [dict(item) for item in SEARCH_INDEXES]
matched = [
item for item in market_items
if needle.casefold() in str(item.get("name") or "").casefold()
or needle.casefold() in str(item.get("code") or "").casefold()
]
matched.sort(key=lambda item: self._search_match_score(item, needle))
group_keys = {"sector": "sectors", "theme": "themes", "index": "indices"}
for item in matched:
group_key = group_keys.get(str(item.get("type") or ""))
if not group_key or len(groups[group_key]) >= 8:
continue
groups[group_key].append(
{
**item,
"type_label": SEARCH_TYPE_LABELS[str(item["type"])],
}
)
return {"query": needle, "trade_date": normalized_date, "groups": groups}
def get_search_detail(
self, entity_type: str, identifier: str, trade_date: str
) -> dict[str, Any]:
entity_type = str(entity_type or "").strip().lower()
identifier = str(identifier or "").strip().upper()
normalized_date = normalize_date(trade_date)
if entity_type not in {"sector", "theme", "index"}:
raise ValueError("搜索详情类型不支持。")
if not re.fullmatch(r"[A-Z0-9.]{3,24}", identifier):
raise ValueError("搜索详情标识无效。")
if not self.configured:
raise ValueError("行情数据源尚未配置。")
if entity_type == "index":
index_basic = next((item for item in SEARCH_INDEXES if item["id"] == identifier), None)
if not index_basic:
raise ValueError("暂不支持该指数详情。")
return self._index_search_detail(index_basic, normalized_date)
directory = self._search_market_directory()
basic = next(
(
item for item in directory
if item.get("id") == identifier and item.get("type") == entity_type
),
None,
)
if not basic:
raise ValueError("未找到对应的板块或题材。")
return self._ths_search_detail(basic, normalized_date)
def get_intraday_chart(
self, entity_type: str, identifier: str
) -> dict[str, Any]:
entity_type = str(entity_type or "").strip().lower()
identifier = str(identifier or "").strip().upper()
if entity_type == "stock":
code = validate_stock_code(identifier)
chart = self.chart_data.stock_intraday(code)
type_label = SEARCH_TYPE_LABELS["stock"]
elif entity_type == "index":
basic = next((item for item in SEARCH_INDEXES if item["id"] == identifier), None)
if not basic:
raise ValueError("暂不支持该指数分时行情。")
chart = self.chart_data.index_intraday(identifier)
type_label = SEARCH_TYPE_LABELS["index"]
elif entity_type in {"sector", "theme"}:
basic = next(
(
item for item in self._search_market_directory()
if item.get("id") == identifier and item.get("type") == entity_type
),
None,
)
if not basic:
raise ValueError("未找到对应的板块或题材。")
chart = self.chart_data.board_intraday(identifier, str(basic.get("name") or ""))
type_label = SEARCH_TYPE_LABELS[entity_type]
else:
raise ValueError("分时行情类型不支持。")
return {
"meta": {
"trade_date": str(chart.get("trade_date") or ""),
"previous_close": float(chart.get("previous_close") or 0),
},
"entity": {
"id": identifier,
"code": str(chart.get("code") or identifier),
"name": str(chart.get("name") or ""),
"type": entity_type,
"type_label": type_label,
},
"points": list(chart.get("points") or []),
}
def _ths_search_detail(
self, basic: dict[str, Any], trade_date: str
) -> dict[str, Any]:
client = self._tushare_client()
resolved_date, _ = client.resolve_trade_context(trade_date)
end = datetime.strptime(resolved_date, "%Y%m%d")
start_date = (end - timedelta(days=190)).strftime("%Y%m%d")
identifier = str(basic["id"])
snapshot = client.sector_snapshot(identifier, resolved_date)
rows = client.query(
"ths_daily",
{"ts_code": identifier, "start_date": start_date, "end_date": resolved_date},
"ts_code,trade_date,open,high,low,close,pct_change,vol,turnover_rate,total_mv,float_mv",
)
rows.sort(key=lambda item: str(item.get("trade_date") or ""))
series = [
{
"trade_date": self._display_compact_date(str(row.get("trade_date") or "")),
"open": float(row.get("open") or 0),
"high": float(row.get("high") or 0),
"low": float(row.get("low") or 0),
"close": float(row.get("close") or 0),
"change": float(row.get("pct_change") or 0),
"volume": float(row.get("vol") or 0),
"turnover_rate": float(row.get("turnover_rate") or 0),
}
for row in rows[-90:]
]
try:
chart_series = self.chart_data.board_daily(identifier, resolved_date, 90)
if chart_series:
series = chart_series
except (AttributeError, ChartDataError):
pass
latest = series[-1] if series else {}
snapshot_is_current = str(snapshot.get("trade_date") or "").replace("-", "") == resolved_date
change = float(
snapshot.get("change")
if snapshot_is_current and snapshot.get("change") is not None
else latest.get("change") or 0
)
if latest.get("realtime"):
change = float(latest.get("change") or 0)
turnover_rate = float(
snapshot.get("turnover_rate")
if snapshot_is_current and snapshot.get("turnover_rate") is not None
else latest.get("turnover_rate") or 0
)
metrics = [
{"label": "涨跌幅", "value": round(change, 2), "unit": "%", "tone": "change"},
{"label": "换手率", "value": round(turnover_rate, 2), "unit": "%"},
{"label": "成份数量", "value": int(float(basic.get("member_count") or 0)), "unit": "只"},
]
up_count = int(float(snapshot.get("up_count") or 0))
down_count = int(float(snapshot.get("down_count") or 0))
if up_count or down_count:
metrics.extend(
[
{"label": "上涨家数", "value": up_count, "unit": "家"},
{"label": "下跌家数", "value": down_count, "unit": "家"},
]
)
leader = str(snapshot.get("leader") or "").strip()
if leader and leader != "--":
metrics.extend(
[
{"label": "领涨标的", "value": leader, "unit": ""},
{"label": "领涨幅", "value": round(float(snapshot.get("leading_pct") or 0), 2), "unit": "%", "tone": "change"},
]
)
return {
"meta": {
"trade_date": self._display_compact_date(resolved_date),
"realtime": bool(snapshot.get("realtime")),
},
"entity": {
"id": identifier,
"code": identifier,
"name": str(snapshot.get("name") or basic.get("name") or "--"),
"type": str(basic.get("type") or "sector"),
"type_label": SEARCH_TYPE_LABELS[str(basic.get("type") or "sector")],
"subtitle": str(basic.get("subtitle") or ""),
"value": float(latest.get("close") or 0),
"change": change,
},
"series": series,
"metrics": metrics,
}
def _index_search_detail(
self, basic: dict[str, Any], trade_date: str
) -> dict[str, Any]:
client = self._tushare_client()
resolved_date, _ = client.resolve_trade_context(trade_date)
payload = (
client.realtime_market_indices(resolved_date)
if client.should_use_realtime(trade_date, resolved_date)
else client.market_indices(resolved_date, 90)
)
current = next(
(item for item in payload.get("indices") or [] if item.get("ts_code") == basic["id"]),
None,
)
if not current:
raise ValueError("该指数暂无可用行情。")
end = datetime.strptime(resolved_date, "%Y%m%d")
rows = client.query(
"index_daily",
{
"ts_code": basic["id"],
"start_date": (end - timedelta(days=190)).strftime("%Y%m%d"),
"end_date": resolved_date,
},
"ts_code,trade_date,open,high,low,close,pct_chg,vol,amount",
)
rows.sort(key=lambda item: str(item.get("trade_date") or ""))
series = [
{
"trade_date": self._display_compact_date(str(row.get("trade_date") or "")),
"open": float(row.get("open") or 0),
"high": float(row.get("high") or 0),
"low": float(row.get("low") or 0),
"close": float(row.get("close") or 0),
"change": float(row.get("pct_chg") or 0),
"volume": float(row.get("vol") or 0),
}
for row in rows[-90:]
]
try:
chart_series = self.chart_data.index_daily(str(basic["id"]), resolved_date, 90)
if chart_series:
series = chart_series
except (AttributeError, ChartDataError):
pass
latest = series[-1] if series else {}
latest_close = float(latest.get("close") or current.get("close") or 0)
latest_change = float(latest.get("change") or current.get("pct_chg") or 0)
def series_return(days: int) -> float:
if len(series) <= days:
return 0.0
previous = float(series[-days - 1].get("close") or 0)
return (latest_close / previous - 1) * 100 if previous > 0 else 0.0
return {
"meta": {
"trade_date": self._display_compact_date(str(current.get("trade_date") or resolved_date)),
"realtime": bool(payload.get("realtime")),
},
"entity": {
**basic,
"type_label": SEARCH_TYPE_LABELS["index"],
"value": latest_close,
"change": latest_change,
},
"series": series,
"metrics": [
{"label": "涨跌幅", "value": round(latest_change, 2), "unit": "%", "tone": "change"},
{"label": "近5日", "value": round(series_return(5), 2), "unit": "%", "tone": "change"},
{"label": "近20日", "value": round(series_return(20), 2), "unit": "%", "tone": "change"},
{"label": "成交额", "value": round(float(current.get("amount_billion") or 0), 2), "unit": "亿"},
],
}
def get_stock_detail(
self, code: str, trade_date: str, force: bool = False
) -> dict[str, Any]:
code = validate_stock_code(code)
normalized_date = normalize_date(trade_date)
cache_key = f"{code}:{normalized_date}"
if not force:
cached = self.database.get_data_snapshot("stock_detail", cache_key)
if cached and str((cached.get("meta") or {}).get("source") or "") != "demo":
if not self._stock_detail_cache_needs_refresh(cached, normalized_date):
cached["meta"] = {**cached.get("meta", {}), "cached": True}
return self._prepare_stock_detail(cached, code, normalized_date)
name, sector = self._stock_identity(code, normalized_date)
source = "tushare"
if self.configured:
try:
payload = self._tushare_client().stock_detail(
tushare_code(code), normalized_date
)
if not payload.get("prices"):
raise TushareError("No price history returned")
except TushareError as exc:
payload = self.database.get_latest_data_snapshot(
"stock_detail", f"{code}:", cache_key, exclude_source="demo"
)
if not payload:
raise ValueError(f"暂无 {code} 的真实行情数据：{exc}") from exc
payload = copy.deepcopy(payload)
payload["meta"] = {
**payload.get("meta", {}),
"cached": True,
"notice": "最新行情暂不可用，已沿用最近真实收盘数据。",
}
return self._prepare_stock_detail(payload, code, normalized_date)
else:
payload = self.database.get_latest_data_snapshot(
"stock_detail", f"{code}:", cache_key, exclude_source="demo"
)
if not payload:
raise ValueError(f"暂无 {code} 的真实行情数据，请等待后台完成首次同步。")
payload = copy.deepcopy(payload)
payload["meta"] = {
**payload.get("meta", {}),
"cached": True,
"notice": "公共行情尚未配置，已沿用最近真实收盘数据。",
}
return self._prepare_stock_detail(payload, code, normalized_date)
payload["meta"]["source"] = source
payload["meta"]["cached"] = False
self.database.save_data_snapshot("stock_detail", cache_key, source, payload)
return self._prepare_stock_detail(payload, code, normalized_date)
@staticmethod
def _stock_detail_bar_date(payload: dict[str, Any]) -> str:
prices = list(payload.get("prices") or [])
return str((prices[-1] if prices else {}).get("trade_date") or "").replace("-", "")
def _stock_detail_cache_needs_refresh(
self, payload: dict[str, Any], requested_date: str
) -> bool:
now = datetime.now().astimezone()
return (
requested_date == now.strftime("%Y%m%d")
and now.time().replace(tzinfo=None) >= dt_time(15, 0)
and self._stock_detail_bar_date(payload) < requested_date
)
def _prepare_stock_detail(
self, payload: dict[str, Any], code: str, requested_date: str
) -> dict[str, Any]:
result = copy.deepcopy(payload)
now = datetime.now().astimezone()
try:
result["prices"] = self.chart_data.stock_daily(code, requested_date, 90)
result["meta"] = {**(result.get("meta") or {}), "chart_source": "market_chart"}
except (AttributeError, ChartDataError):
pass
result = self._sanitize_stock_detail_prices(result, now)
actual_date = self._stock_detail_bar_date(result)
if actual_date:
result["meta"] = {
**(result.get("meta") or {}),
"trade_date": f"{actual_date[:4]}-{actual_date[4:6]}-{actual_date[6:]}",
}
today = now.strftime("%Y%m%d")
should_merge = (
requested_date == today
and actual_date <= today
and now.weekday() < 5
and now.time().replace(tzinfo=None) >= dt_time(9, 30)
)
if should_merge:
quote = self._ifind_realtime_stock_quote(code)
if quote and self._valid_realtime_stock_quote(quote, today):
self._merge_realtime_stock_detail(result, quote, requested_date)
elif self.configured and actual_date < today:
client = self._tushare_client()
try:
resolved_date, _ = client.resolve_trade_context(requested_date)
if resolved_date == today:
quote = client.realtime_stock_quote(tushare_code(code), requested_date)
if self._valid_realtime_stock_quote(quote, today):
self._merge_realtime_stock_detail(result, quote, requested_date)
except TushareError:
pass
return self._enrich_stock_detail(result)
@staticmethod
def _sanitize_stock_detail_prices(
payload: dict[str, Any], market_now: datetime
) -> dict[str, Any]:
result = copy.deepcopy(payload)
raw_prices = list(result.get("prices") or [])
raw_latest_date = str(
(raw_prices[-1] if raw_prices else {}).get("trade_date") or ""
).replace("-", "")
prices = []
for bar in raw_prices:
open_price = float(bar.get("open") or 0)
high = float(bar.get("high") or 0)
low = float(bar.get("low") or 0)
close = float(bar.get("close") or 0)
if (
open_price > 0
and high >= max(open_price, close)
and 0 < low <= min(open_price, close)
and close > 0
):
prices.append(bar)
today = market_now.strftime("%Y%m%d")
market_open = (
market_now.weekday() < 5
and market_now.time().replace(tzinfo=None) >= dt_time(9, 30)
)
if prices and str(prices[-1].get("trade_date") or "").replace("-", "") == today:
current = prices[-1]
has_market_activity = (
float(current.get("volume") or 0) > 0
or float(current.get("amount_billion") or 0) > 0
)
if not market_open or not has_market_activity:
prices.pop()
if raw_latest_date == today and (
not prices
or str(prices[-1].get("trade_date") or "").replace("-", "") != today
):
result["meta"] = {**(result.get("meta") or {}), "realtime": False}
result["prices"] = prices
if prices:
latest = prices[-1]
stock = dict(result.get("stock") or {})
stock.update(
{
"price": float(latest.get("close") or 0),
"change": float(latest.get("change") or 0),
"amount_billion": float(latest.get("amount_billion") or 0),
}
)
result["stock"] = stock
return result
@staticmethod
def _valid_realtime_stock_quote(quote: dict[str, Any], trade_date: str) -> bool:
price = float(quote.get("price") or 0)
open_price = float(quote.get("open") or 0)
high = float(quote.get("high") or 0)
low = float(quote.get("low") or 0)
volume = float(quote.get("volume") or 0)
amount = float(quote.get("amount_billion") or 0)
quote_date = str(quote.get("quote_time") or "")[:10].replace("-", "")
return (
price > 0
and open_price > 0
and high >= max(open_price, price)
and 0 < low <= min(open_price, price)
and (volume > 0 or amount > 0)
and (not quote_date or quote_date == trade_date)
)
def _ifind_realtime_stock_quote(self, code: str) -> dict[str, Any] | None:
ifind = getattr(self, "ifind", None)
if not ifind or not ifind.configured:
return None
try:
rows = ifind.real_time(
tushare_code(code),
[
"open", "high", "low", "latest", "preClose",
"volume", "amount", "turnoverRatio",
],
cache_ttl=10,
)
except IfindError:
return None
row = rows[0] if rows else {}
price = float(row.get("latest") or 0)
previous_close = float(row.get("preClose") or 0)
if price <= 0:
return None
change = (price / previous_close - 1) * 100 if previous_close > 0 else 0.0
stock = self._stock_identity(code, date.today().strftime("%Y%m%d"))
return {
"name": stock[0],
"sector": stock[1],
"price": price,
"open": float(row.get("open") or price),
"high": float(row.get("high") or price),
"low": float(row.get("low") or price),
"change": round(change, 4),
"volume": float(row.get("volume") or 0),
"volume_unit": "lots",
"amount_billion": float(row.get("amount") or 0) / 100_000_000,
"turnover_rate": float(row.get("turnoverRatio") or 0),
"quote_time": str(row.get("time") or ""),
}
@staticmethod
def _merge_realtime_stock_detail(
payload: dict[str, Any], quote: dict[str, Any], trade_date: str
) -> None:
display_date = f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:]}"
realtime_bar = {
"trade_date": display_date,
"open": quote["open"],
"high": quote["high"],
"low": quote["low"],
"close": quote["price"],
"change": quote["change"],
"volume": quote["volume"] if quote.get("volume_unit") == "lots" else quote["volume"] / 100,
"amount_billion": quote["amount_billion"],
"realtime": True,
}
prices = list(payload.get("prices") or [])
if prices and str(prices[-1].get("trade_date") or "").replace("-", "") == trade_date:
prices[-1] = realtime_bar
else:
prices.append(realtime_bar)
payload["prices"] = prices[-90:]
stock = dict(payload.get("stock") or {})
stock.update(
{
"name": quote["name"],
"industry": quote["sector"],
"price": quote["price"],
"change": quote["change"],
"amount_billion": quote["amount_billion"],
"turnover_rate": quote["turnover_rate"],
}
)
payload["stock"] = stock
payload["meta"] = {
**(payload.get("meta") or {}),
"trade_date": display_date,
"realtime": True,
"updated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
}
def get_stock_preview(
self, code: str, trade_date: str, force: bool = False
) -> dict[str, Any]:
code = validate_stock_code(code)
# Hover previews deliberately follow the latest market day, independent
# from the review date selected by the page.
detail = self.get_stock_detail(code, date.today().strftime("%Y%m%d"), force)
detail_meta = detail.get("meta") or {}
resolved_date = str(detail_meta.get("trade_date") or trade_date)
intraday_points: list[dict[str, Any]] = []
intraday_status = "unavailable"
intraday_notice = "分时行情暂不可用。"
intraday_trade_date = ""
intraday_previous_close = 0.0
try:
intraday = self.chart_data.stock_intraday(code)
intraday_points = list(intraday.get("points") or [])
intraday_trade_date = str(intraday.get("trade_date") or "")
intraday_previous_close = float(intraday.get("previous_close") or 0)
if intraday_points:
intraday_status = "available"
intraday_notice = ""
else:
intraday_status = "empty"
intraday_notice = "最近交易日暂无分时数据。"
except ChartDataError:
intraday_status = "unavailable"
intraday_notice = "分时行情暂不可用，请稍后重试。"
prices = list(detail.get("prices") or [])[-60:]
stock = dict(detail.get("stock") or {"code": code})
realtime = bool(detail_meta.get("realtime"))
return {
"meta": {
"trade_date": resolved_date,
"source": detail_meta.get("source") or "unavailable",
"notice": detail_meta.get("notice") or "",
"intraday_status": intraday_status,
"intraday_notice": intraday_notice,
"intraday_trade_date": intraday_trade_date,
"intraday_previous_close": intraday_previous_close,
"realtime": realtime,
"refresh_interval_seconds": 10 if realtime else 0,
},
"stock": stock,
"prices": prices,
"intraday": intraday_points,
}
def backfill(self, start_date: str, end_date: str) -> list[dict[str, Any]]:
start = datetime.strptime(normalize_date(start_date), "%Y%m%d").date()
end = datetime.strptime(normalize_date(end_date), "%Y%m%d").date()
if start > end:
raise ValueError("开始日期不能晚于结束日期。")
weekdays = []
current = start
while current <= end:
if current.weekday() < 5:
weekdays.append(current)
current += timedelta(days=1)
if len(weekdays) > 15:
raise ValueError("单次最多回补 15 个工作日。")
results = []
for day in weekdays:
dashboard = self.sync_dashboard(day.strftime("%Y%m%d"))
results.append(
{
"requested_date": day.isoformat(),
"trade_date": dashboard["meta"]["trade_date"],
"source": dashboard["meta"]["source"],
"records": self._record_count(dashboard),
}
)
return results
def _stock_identity(self, code: str, trade_date: str) -> tuple[str, str]:
snapshot = self.database.get_snapshot(trade_date) or {}
for key in ("limits", "broken", "down_limits"):
for row in snapshot.get(key) or []:
if str(row.get("code")) == code:
return row.get("name") or "--", row.get("sector") or "其他"
for item in self.database.list_watchlist(self.current_user_id):
if item["code"] == code:
return item["name"], item["sector"] or "其他"
return "--", "其他"
def _enrich_stock_detail(self, payload: dict[str, Any]) -> dict[str, Any]:
result = dict(payload)
stock = dict(payload.get("stock") or {})
code = str(stock.get("code") or "")
watched = {
item["code"]: item
for item in self.database.list_watchlist(self.current_user_id)
}
stock["watchlist"] = watched.get(code)
result["stock"] = stock
result["notes"] = self.database.list_notes(self.current_user_id, code=code)
return result
def _with_storage(self, dashboard: dict[str, Any], cached: bool) -> dict[str, Any]:
result = dict(dashboard)
result["meta"] = {
**dashboard.get("meta", {}),
"storage": "sqlite",
"cached": cached,
}
return result
@staticmethod
def _record_count(dashboard: dict[str, Any]) -> int:
return sum(
len(dashboard.get(key) or [])
for key in ("limits", "broken", "down_limits", "yesterday_limits")
)
chart_data_provider.py
+4 -494
View File
@@ -1,497 +1,7 @@
from __future__ import annotations """Compatibility alias for the canonical market chart clients."""
import http.client import sys
import json
import re
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass
from datetime import datetime, time as dt_time, timedelta
from threading import Lock
from typing import Any, ClassVar
from ifind_client import IfindError, IfindHttpClient from backend.features.market import charts as _implementation
sys.modules[__name__] = _implementation
class ChartDataError(RuntimeError):
pass
TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get"
BOARD_LIST_URL = "https://push2delay.eastmoney.com/api/qt/clist/get"
BROWSER_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/138.0.0.0 Safari/537.36"
)
INDEX_SECIDS = {
"000001.SH": "1.000001",
"399001.SZ": "0.399001",
"399006.SZ": "0.399006",
}
class MarketChartClient:
"""Prefer iFinD for display charts and retain Eastmoney as a last resort."""
def __init__(self, ifind: IfindHttpClient, fallback: "EastmoneyChartClient") -> None:
self.ifind = ifind
self.fallback = fallback
def stock_intraday(self, code: str) -> dict[str, Any]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
ifind_code = _stock_market_code(normalized)
try:
return self._ifind_intraday(ifind_code, "stock", normalized)
except (IfindError, ChartDataError):
return self.fallback.stock_intraday(normalized)
def stock_daily(self, code: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
return self._ifind_daily(_stock_market_code(normalized), end_date, limit)
def index_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(identifier or "").strip().upper()
if normalized not in INDEX_SECIDS:
raise ChartDataError("Unsupported index")
return self._ifind_daily(normalized, end_date, limit)
def board_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
normalized = str(identifier or "").strip().upper()
if not normalized:
raise ChartDataError("Invalid board code")
return self._ifind_daily(normalized, end_date, limit)
def index_intraday(self, identifier: str) -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
if normalized not in INDEX_SECIDS:
raise ChartDataError("Unsupported index")
try:
return self._ifind_intraday(normalized, "index", normalized)
except (IfindError, ChartDataError):
return self.fallback.index_intraday(normalized)
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
try:
return self._ifind_intraday(normalized, "board", normalized, name)
except (IfindError, ChartDataError):
return self.fallback.board_intraday(normalized, name)
def _ifind_intraday(
self,
ifind_code: str,
entity_type: str,
identifier: str,
name: str = "",
) -> dict[str, Any]:
if not self.ifind.configured:
raise ChartDataError("iFinD is not configured")
now = datetime.now().astimezone()
rows: list[dict[str, Any]] = []
for offset in range(0, 8):
candidate = now.date() - timedelta(days=offset)
if candidate.weekday() >= 5:
continue
display_date = candidate.isoformat()
rows = self.ifind.intraday(
ifind_code,
f"{display_date} 09:30:00",
f"{display_date} 15:00:00",
cache_ttl=20 if offset == 0 else 6 * 60 * 60,
)
if rows:
break
points = [point for row in rows if (point := _ifind_point(row))]
if not points:
raise ChartDataError("No iFinD intraday chart data returned")
latest_date = points[-1]["date"]
points = [point for point in points if point["date"] == latest_date]
previous_close = self._previous_close(ifind_code, latest_date, points[0]["open"])
return {
"entity_type": entity_type,
"identifier": identifier,
"name": name,
"code": identifier,
"trade_date": latest_date,
"previous_close": previous_close,
"points": points,
"source": "ifind",
}
def _ifind_daily(
self, ifind_code: str, end_date: str, limit: int
) -> list[dict[str, Any]]:
if not self.ifind.configured:
raise ChartDataError("iFinD is not configured")
compact_end = str(end_date or "").replace("-", "")
if not re.fullmatch(r"\d{8}", compact_end):
raise ChartDataError("Invalid chart end date")
end = datetime.strptime(compact_end, "%Y%m%d")
start = (end - timedelta(days=max(190, limit * 3))).strftime("%Y%m%d")
try:
rows = self.ifind.history(
ifind_code,
["open", "high", "low", "close", "volume", "amount"],
start,
compact_end,
cache_ttl=300,
)
except IfindError as exc:
raise ChartDataError("No iFinD daily chart data returned") from exc
normalized = []
for row in rows:
stamp = str(row.get("time") or "").strip()
trade_date = stamp[:10]
close = _number(row.get("close"))
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", trade_date) or close <= 0:
continue
normalized.append(
{
"trade_date": trade_date,
"open": _number(row.get("open")),
"high": _number(row.get("high")),
"low": _number(row.get("low")),
"close": close,
"volume": _number(row.get("volume")),
"amount_billion": _number(row.get("amount")) / 100_000_000,
}
)
normalized.sort(key=lambda row: row["trade_date"])
for index, row in enumerate(normalized):
previous = normalized[index - 1]["close"] if index > 0 else 0
row["change"] = round((row["close"] / previous - 1) * 100, 4) if previous else 0.0
market_now = datetime.now().astimezone()
today = market_now.strftime("%Y%m%d")
market_open = (
market_now.weekday() < 5
and market_now.time().replace(tzinfo=None) >= dt_time(9, 30)
)
today_display = market_now.date().isoformat()
if normalized and normalized[-1]["trade_date"] == today_display:
current_bar = normalized[-1]
current_bar_is_valid = (
current_bar["open"] > 0
and current_bar["high"] >= max(current_bar["open"], current_bar["close"])
and 0 < current_bar["low"] <= min(current_bar["open"], current_bar["close"])
and (current_bar["volume"] > 0 or current_bar["amount_billion"] > 0)
)
if not market_open or not current_bar_is_valid:
normalized.pop()
if compact_end == today and market_open:
try:
quote_rows = self.ifind.real_time(
ifind_code,
["open", "high", "low", "latest", "preClose", "volume", "amount"],
cache_ttl=10,
)
quote = quote_rows[0] if quote_rows else {}
latest = _number(quote.get("latest"))
previous = _number(quote.get("preClose"))
open_price = _number(quote.get("open"))
high = _number(quote.get("high"))
low = _number(quote.get("low"))
volume = _number(quote.get("volume"))
amount = _number(quote.get("amount"))
quote_date = str(quote.get("time") or "")[:10].replace("-", "")
quote_is_current = not quote_date or quote_date == today
has_market_activity = volume > 0 or amount > 0
if (
latest > 0
and open_price > 0
and high >= max(open_price, latest)
and 0 < low <= min(open_price, latest)
and has_market_activity
and quote_is_current
):
realtime = {
"trade_date": end.strftime("%Y-%m-%d"),
"open": open_price,
"high": high,
"low": low,
"close": latest,
"change": round((latest / previous - 1) * 100, 4) if previous else 0.0,
"volume": volume,
"amount_billion": amount / 100_000_000,
"realtime": True,
}
if normalized and normalized[-1]["trade_date"] == realtime["trade_date"]:
normalized[-1] = realtime
else:
normalized.append(realtime)
except IfindError:
pass
if not normalized:
raise ChartDataError("No iFinD daily chart data returned")
return normalized[-max(20, min(180, int(limit))):]
def _previous_close(self, code: str, trade_date: str, fallback: float) -> float:
today = datetime.now().astimezone().date().isoformat()
if trade_date == today:
try:
quote = self.ifind.real_time(code, ["preClose"], cache_ttl=20)
value = _number((quote[0] if quote else {}).get("preClose"))
if value > 0:
return value
except IfindError:
pass
end = datetime.strptime(trade_date, "%Y-%m-%d")
try:
rows = self.ifind.history(
code,
["close"],
(end - timedelta(days=12)).strftime("%Y%m%d"),
end.strftime("%Y%m%d"),
cache_ttl=6 * 60 * 60,
)
closes = [_number(row.get("close")) for row in rows if _number(row.get("close")) > 0]
if len(closes) >= 2:
return closes[-2]
except IfindError:
pass
return fallback
@dataclass
class EastmoneyChartClient:
"""Isolated display-only minute chart source.
The returned data must not be used by market snapshots, scoring, screening,
or divination. Its only consumer is a chart-rendering endpoint.
"""
timeout: int = 6
cache_ttl_seconds: int = 20
retry_attempts: int = 2
_cache: ClassVar[dict[str, dict[str, Any]]] = {}
_cache_lock: ClassVar[Lock] = Lock()
_board_catalog: ClassVar[dict[str, dict[str, str]]] = {}
_board_catalog_at: ClassVar[float] = 0.0
_board_catalog_lock: ClassVar[Lock] = Lock()
def stock_intraday(self, code: str) -> dict[str, Any]:
normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code")
market = "1" if normalized.startswith(("5", "6", "9")) else "0"
return self._intraday(f"{market}.{normalized}", "stock", normalized)
def index_intraday(self, identifier: str) -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
secid = INDEX_SECIDS.get(normalized)
if not secid:
raise ChartDataError("Unsupported index")
return self._intraday(secid, "index", normalized)
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
normalized = str(identifier or "").strip().upper()
if re.fullmatch(r"BK\d{4}", normalized):
board_code = normalized
else:
board_code = self._resolve_board_code(name or identifier)
return self._intraday(f"90.{board_code}", "board", board_code)
def _intraday(self, secid: str, entity_type: str, identifier: str) -> dict[str, Any]:
cache_key = f"{entity_type}:{identifier}"
cached = self._get_cached(cache_key)
if cached is not None:
return cached
payload = self._request_json(
TRENDS_URL,
{
"secid": secid,
"fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13",
"fields2": "f51,f52,f53,f54,f55,f56,f57,f58",
"iscr": "0",
"ndays": "1",
},
"https://quote.eastmoney.com/",
)
data = payload.get("data") or {}
points = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))]
if not points:
raise ChartDataError("No intraday chart data returned")
result = {
"entity_type": entity_type,
"identifier": identifier,
"name": str(data.get("name") or ""),
"code": str(data.get("code") or identifier),
"trade_date": points[-1]["date"],
"previous_close": _number(data.get("preClose")),
"points": points,
}
with self._cache_lock:
self._cache[cache_key] = {"created_at": time.time(), "payload": result}
return result
def _get_cached(self, cache_key: str) -> dict[str, Any] | None:
with self._cache_lock:
cached = self._cache.get(cache_key)
if not cached:
return None
if time.time() - float(cached.get("created_at") or 0) > self.cache_ttl_seconds:
with self._cache_lock:
self._cache.pop(cache_key, None)
return None
return dict(cached["payload"])
def _resolve_board_code(self, name: str) -> str:
normalized = _normalize_name(name)
if not normalized:
raise ChartDataError("Board name is required")
catalog = self._load_board_catalog()
item = catalog.get(normalized)
if not item:
raise ChartDataError("No matching chart board")
return item["code"]
def _load_board_catalog(self) -> dict[str, dict[str, str]]:
now = time.time()
with self._board_catalog_lock:
if self._board_catalog and now - self._board_catalog_at < 6 * 60 * 60:
return dict(self._board_catalog)
rows: list[dict[str, Any]] = []
for board_type in ("1", "2", "3"):
for page in range(1, 6):
payload = self._request_json(
BOARD_LIST_URL,
{
"pn": str(page),
"pz": "100",
"po": "1",
"np": "1",
"fltt": "2",
"invt": "2",
"fid": "f3",
"fs": f"m:90+t:{board_type}",
"fields": "f12,f14",
},
"https://quote.eastmoney.com/center/boardlist.html",
)
page_rows = (payload.get("data") or {}).get("diff") or []
rows.extend(page_rows)
if len(page_rows) < 100:
break
catalog: dict[str, dict[str, str]] = {}
for row in rows:
code = str(row.get("f12") or "").strip().upper()
board_name = str(row.get("f14") or "").strip()
if re.fullmatch(r"BK\d{4}", code) and board_name:
catalog.setdefault(_normalize_name(board_name), {"code": code, "name": board_name})
if not catalog:
raise ChartDataError("Board chart directory is unavailable")
with self._board_catalog_lock:
type(self)._board_catalog = catalog
type(self)._board_catalog_at = now
return dict(catalog)
def _request_json(
self, url: str, params: dict[str, str], referer: str
) -> dict[str, Any]:
request_url = f"{url}?{urllib.parse.urlencode(params)}"
last_error: Exception | None = None
for attempt in range(max(1, int(self.retry_attempts))):
request = urllib.request.Request(
request_url,
headers={
"Accept": "application/json,text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
payload = json.loads(response.read().decode("utf-8"))
if not isinstance(payload, dict):
raise ChartDataError("Invalid intraday chart response")
return payload
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
json.JSONDecodeError,
ChartDataError,
) as exc:
last_error = exc
if attempt + 1 < self.retry_attempts:
time.sleep(0.12)
raise ChartDataError("Intraday chart request failed") from last_error
def _parse_trend(raw: Any) -> dict[str, Any] | None:
fields = str(raw or "").split(",")
if len(fields) < 8 or " " not in fields[0]:
return None
stamp = fields[0].strip()
trade_date, trade_time = stamp.split(" ", 1)
close = _number(fields[2])
if close <= 0:
return None
return {
"date": trade_date,
"time": trade_time[:5],
"open": _number(fields[1]),
"close": close,
"high": _number(fields[3]),
"low": _number(fields[4]),
"volume": _number(fields[5]),
"amount": _number(fields[6]),
"average": _number(fields[7]),
}
def _ifind_point(row: dict[str, Any]) -> dict[str, Any] | None:
stamp = str(row.get("time") or "").strip()
if " " not in stamp:
return None
trade_date, trade_time = stamp.split(" ", 1)
close = _number(row.get("close"))
if close <= 0:
return None
return {
"date": trade_date,
"time": trade_time[:5],
"open": _number(row.get("open")),
"close": close,
"high": _number(row.get("high")),
"low": _number(row.get("low")),
"volume": _number(row.get("volume")),
"amount": _number(row.get("amount")),
"average": _number(row.get("avgPrice")),
}
def _stock_market_code(code: str) -> str:
if code.startswith(("4", "8", "9")):
suffix = "BJ"
elif code.startswith("6"):
suffix = "SH"
else:
suffix = "SZ"
return f"{code}.{suffix}"
def _number(value: Any) -> float:
try:
return float(value or 0)
except (TypeError, ValueError):
return 0.0
def _normalize_name(value: Any) -> str:
normalized = re.sub(r"[\s·・（）()\-_/]", "", str(value or "")).casefold()
return re.sub(r"(?:概念|行业|[ⅠⅡⅢ])$", "", normalized)
database.py
+6 -213
View File
@@ -8,6 +8,7 @@ from typing import Any
from backend.database import MIGRATIONS, MigrationRunner, SQLiteConnectionFactory from backend.database import MIGRATIONS, MigrationRunner, SQLiteConnectionFactory
from backend.features.accounts.repository import AccountRepositoryMixin from backend.features.accounts.repository import AccountRepositoryMixin
from backend.features.market.repository import MarketRepositoryMixin
from backend.features.system.repository import SystemSettingsRepositoryMixin from backend.features.system.repository import SystemSettingsRepositoryMixin
@@ -20,7 +21,11 @@ def _optional_float(value: Any) -> float | None:
return None return None
class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin): class ReviewDatabase(
AccountRepositoryMixin,
MarketRepositoryMixin,
SystemSettingsRepositoryMixin,
):
def __init__(self, path: Path) -> None: def __init__(self, path: Path) -> None:
self.path = path self.path = path
self.path.parent.mkdir(parents=True, exist_ok=True) self.path.parent.mkdir(parents=True, exist_ok=True)
@@ -690,118 +695,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
).fetchone() ).fetchone()
return int(row["total"] if row else 0) return int(row["total"] if row else 0)
def get_snapshot(self, trade_date: str) -> dict[str, Any] | None:
with self.connect() as connection:
row = connection.execute(
"SELECT payload FROM dashboard_snapshots WHERE trade_date = ?",
(trade_date,),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def get_latest_real_snapshot(
self, trade_date: str, strictly_before: bool = False
) -> dict[str, Any] | None:
operator = "<" if strictly_before else "<="
with self.connect() as connection:
row = connection.execute(
f"""
SELECT payload FROM dashboard_snapshots
WHERE trade_date {operator} ? AND source != 'demo'
ORDER BY trade_date DESC LIMIT 1
""",
(trade_date,),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def save_snapshot(self, trade_date: str, source: str, payload: dict[str, Any]) -> None:
updated_at = datetime.now().astimezone().isoformat(timespec="seconds")
record_count = sum(
len(payload.get(key) or [])
for key in ("limits", "broken", "down_limits", "yesterday_limits")
)
content = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
with self.connect() as connection:
connection.execute(
"""
INSERT INTO dashboard_snapshots
(trade_date, source, payload, record_count, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(trade_date) DO UPDATE SET
source = excluded.source,
payload = excluded.payload,
record_count = excluded.record_count,
updated_at = excluded.updated_at
""",
(trade_date, source, content, record_count, updated_at),
)
def get_data_snapshot(self, kind: str, cache_key: str) -> dict[str, Any] | None:
with self.connect() as connection:
row = connection.execute(
"SELECT payload FROM data_snapshots WHERE kind = ? AND cache_key = ?",
(kind, cache_key),
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def get_latest_data_snapshot(
self,
kind: str,
cache_key_prefix: str,
maximum_cache_key: str,
exclude_source: str = "",
) -> dict[str, Any] | None:
source_clause = " AND source != ?" if exclude_source else ""
parameters: list[Any] = [kind, f"{cache_key_prefix}%", maximum_cache_key]
if exclude_source:
parameters.append(exclude_source)
with self.connect() as connection:
row = connection.execute(
f"""
SELECT payload FROM data_snapshots
WHERE kind = ? AND cache_key LIKE ? AND cache_key <= ?{source_clause}
ORDER BY cache_key DESC LIMIT 1
""",
parameters,
).fetchone()
if not row:
return None
try:
return json.loads(row["payload"])
except json.JSONDecodeError:
return None
def save_data_snapshot(
self, kind: str, cache_key: str, source: str, payload: dict[str, Any]
) -> None:
updated_at = datetime.now().astimezone().isoformat(timespec="seconds")
content = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
with self.connect() as connection:
connection.execute(
"""
INSERT INTO data_snapshots (kind, cache_key, source, payload, updated_at)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(kind, cache_key) DO UPDATE SET
source = excluded.source,
payload = excluded.payload,
updated_at = excluded.updated_at
""",
(kind, cache_key, source, content, updated_at),
)
def list_watchlist(self, user_id: int) -> list[dict[str, Any]]: def list_watchlist(self, user_id: int) -> list[dict[str, Any]]:
with self.connect() as connection: with self.connect() as connection:
@@ -869,7 +762,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
(int(user_id), code), (int(user_id), code),
) )
return cursor.rowcount > 0 return cursor.rowcount > 0
def list_notes( def list_notes(
self, self,
user_id: int, user_id: int,
@@ -1042,26 +934,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
) )
return len(values) return len(values)
def search_stock_master(self, query: str, limit: int = 12) -> list[dict[str, Any]]:
text = str(query or "").strip()
if not text:
return []
escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
with self.connect() as connection:
rows = connection.execute(
"""
SELECT ts_code, code, name, industry, market, list_date
FROM stock_master
WHERE code = ? OR name = ? OR name LIKE ? ESCAPE '\\'
ORDER BY
CASE WHEN code = ? THEN 0 WHEN name = ? THEN 1 ELSE 2 END,
list_date DESC,
code
LIMIT ?
""",
(text, text, f"%{escaped}%", text, text, max(1, min(30, int(limit)))),
).fetchall()
return [dict(row) for row in rows]
def list_stock_master(self) -> list[dict[str, Any]]: def list_stock_master(self) -> list[dict[str, Any]]:
with self.connect() as connection: with self.connect() as connection:
@@ -1666,24 +1538,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
for row in series[-limit:] for row in series[-limit:]
] ]
def list_snapshot_payloads(self, end_date: str, limit: int = 260) -> list[dict[str, Any]]:
with self.connect() as connection:
rows = connection.execute(
"""
SELECT trade_date, payload FROM dashboard_snapshots
WHERE trade_date <= ? ORDER BY trade_date DESC LIMIT ?
""",
(end_date, limit),
).fetchall()
result: list[dict[str, Any]] = []
for row in reversed(rows):
try:
payload = json.loads(row["payload"])
except json.JSONDecodeError:
continue
payload["_snapshot_date"] = row["trade_date"]
result.append(payload)
return result
def save_screener_strategy( def save_screener_strategy(
self, user_id: int | None, name: str, description: str, regimes: list[str], formula: dict[str, Any], self, user_id: int | None, name: str, description: str, regimes: list[str], formula: dict[str, Any],
@@ -2529,64 +2383,3 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin):
(int(reading_id), int(user_id)), (int(reading_id), int(user_id)),
) )
return cursor.rowcount > 0 return cursor.rowcount > 0
def start_sync(self, trade_date: str, source: str) -> int:
started_at = datetime.now().astimezone().isoformat(timespec="seconds")
with self.connect() as connection:
cursor = connection.execute(
"""
INSERT INTO sync_runs (trade_date, source, status, started_at)
VALUES (?, ?, 'running', ?)
""",
(trade_date, source, started_at),
)
return int(cursor.lastrowid)
def finish_sync(
self,
sync_id: int,
status: str,
record_count: int = 0,
message: str = "",
source: str | None = None,
) -> None:
finished_at = datetime.now().astimezone().isoformat(timespec="seconds")
with self.connect() as connection:
connection.execute(
"""
UPDATE sync_runs
SET status = ?, finished_at = ?, record_count = ?, message = ?,
source = COALESCE(?, source)
WHERE id = ?
""",
(status, finished_at, record_count, message[:1000], source, sync_id),
)
def status(self) -> dict[str, Any]:
with self.connect() as connection:
last_sync = connection.execute(
"""
SELECT id, trade_date, source, status, started_at, finished_at,
record_count, message
FROM sync_runs ORDER BY id DESC LIMIT 1
"""
).fetchone()
snapshot_stats = connection.execute(
"""
SELECT COUNT(*) AS dates, COALESCE(SUM(record_count), 0) AS records,
MAX(updated_at) AS updated_at
FROM dashboard_snapshots
"""
).fetchone()
watchlist_count = connection.execute("SELECT COUNT(*) FROM watchlist").fetchone()[0]
note_count = connection.execute("SELECT COUNT(*) FROM review_notes").fetchone()[0]
return {
"database": str(self.path.name),
"snapshot_dates": int(snapshot_stats["dates"]),
"snapshot_records": int(snapshot_stats["records"]),
"updated_at": snapshot_stats["updated_at"],
"last_sync": dict(last_sync) if last_sync else None,
"watchlist_count": int(watchlist_count),
"note_count": int(note_count),
}
ifind_client.py
+4 -382
View File
@@ -1,385 +1,7 @@
from __future__ import annotations """Compatibility alias for the canonical iFinD provider implementation."""
import copy import sys
import json
import threading
import time
import urllib.error
import urllib.request
from datetime import datetime, timedelta
from typing import Any
from backend.data.providers import ifind_client as _implementation
class IfindError(RuntimeError): sys.modules[__name__] = _implementation
pass
class IfindHttpClient:
BASE_URL = "https://quantapi.51ifind.com/api/v1"
AUTH_ENDPOINT = "get_access_token"
AUTH_ERROR_CODES = {-1302, -1303, -1304, -4302, -4303}
def __init__(
self,
refresh_token: str = "",
access_token: str = "",
timeout: int = 15,
) -> None:
self.timeout = max(3, int(timeout))
self._refresh_token = str(refresh_token or "").strip()
self._access_token = str(access_token or "").strip()
self._access_expires_at: datetime | None = None
self._token_lock = threading.Lock()
self._cache_lock = threading.Lock()
self._cache: dict[str, dict[str, Any]] = {}
@property
def configured(self) -> bool:
return bool(self._refresh_token or self._access_token)
def set_credentials(self, refresh_token: str, access_token: str = "") -> None:
refresh_token = str(refresh_token or "").strip()
access_token = str(access_token or "").strip()
with self._token_lock:
refresh_changed = refresh_token != self._refresh_token
self._refresh_token = refresh_token
if access_token or refresh_changed:
self._access_token = access_token
self._access_expires_at = None
if refresh_changed:
with self._cache_lock:
self._cache.clear()
def status(self) -> dict[str, Any]:
return {
"configured": self.configured,
"access_ready": bool(self._access_token),
"access_expires_at": (
self._access_expires_at.isoformat(timespec="seconds")
if self._access_expires_at
else ""
),
}
def test_connection(self) -> dict[str, Any]:
payload = self.real_time(
"000001.SH",
["open", "high", "low", "latest", "preClose"],
cache_ttl=0,
)
return {
"ok": bool(payload),
"sample_time": str(payload[0].get("time") or "") if payload else "",
}
def real_time(
self,
codes: str | list[str],
indicators: list[str],
cache_ttl: int = 10,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"real_time_quotation",
{"codes": code_text, "indicators": ",".join(indicators)},
cache_key=f"rq:{code_text}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def history(
self,
codes: str | list[str],
indicators: list[str],
start_date: str,
end_date: str,
cache_ttl: int = 300,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"cmd_history_quotation",
{
"codes": code_text,
"indicators": ",".join(indicators),
"startdate": self._display_date(start_date),
"enddate": self._display_date(end_date),
"functionpara": {"CPS": "forward1", "Fill": "Omit"},
},
cache_key=f"hq:{code_text}:{start_date}:{end_date}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def intraday(
self,
code: str,
start_time: str,
end_time: str,
cache_ttl: int = 20,
) -> list[dict[str, Any]]:
indicators = ["open", "high", "low", "close", "volume", "amount", "avgPrice"]
payload = self._request(
"high_frequency",
{
"codes": self._codes(code),
"indicators": ",".join(indicators),
"starttime": start_time,
"endtime": end_time,
"functionpara": {
"CPS": "forward1",
"Fill": "Previous",
"Timeformat": "LocalTime",
"Interval": "1",
"Limitstart": "09:30:00",
"Limitend": "15:00:00",
},
},
cache_key=f"hf:{code}:{start_time}:{end_time}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def snapshots(
self,
codes: str | list[str],
indicators: list[str],
start_time: str,
end_time: str,
cache_ttl: int = 8,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"snap_shot",
{
"codes": code_text,
"indicators": ",".join(indicators),
"starttime": start_time,
"endtime": end_time,
},
cache_key=f"ss:{code_text}:{start_time}:{end_time}:{','.join(indicators)}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def wencai(self, query: str, search_type: str = "stock", cache_ttl: int = 300) -> list[dict[str, Any]]:
normalized = " ".join(str(query or "").split())
if not normalized:
raise IfindError("问财查询不能为空。")
payload = self._request(
"smart_stock_picking",
{"searchstring": normalized, "searchtype": search_type},
cache_key=f"wc:{search_type}:{normalized}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def report_query(
self,
codes: str | list[str],
begin_date: str,
end_date: str,
cache_ttl: int = 300,
) -> list[dict[str, Any]]:
code_text = self._codes(codes)
payload = self._request(
"report_query",
{
"codes": code_text,
"beginrDate": self._display_date(begin_date),
"endrDate": self._display_date(end_date),
"outputpara": (
"reportDate:Y,thscode:Y,secName:Y,ctime:Y,"
"reportTitle:Y,pdfURL:Y,seq:Y"
),
},
cache_key=f"report:{code_text}:{begin_date}:{end_date}",
cache_ttl=cache_ttl,
)
return self._table_rows(payload)
def _request(
self,
endpoint: str,
body: dict[str, Any],
cache_key: str = "",
cache_ttl: int = 0,
) -> dict[str, Any]:
if not self.configured:
raise IfindError("iFinD 尚未配置。")
if cache_key and cache_ttl > 0:
cached = self._cached(cache_key, cache_ttl)
if cached is not None:
return cached
payload = self._post(endpoint, body, self._ensure_access_token())
if self._is_auth_error(payload) and self._refresh_token:
self._invalidate_access_token()
payload = self._post(endpoint, body, self._ensure_access_token(force=True))
self._validate_payload(payload)
if cache_key and cache_ttl > 0:
with self._cache_lock:
self._cache[cache_key] = {
"created_at": time.time(),
"payload": copy.deepcopy(payload),
}
return payload
def _ensure_access_token(self, force: bool = False) -> str:
with self._token_lock:
now = datetime.now().astimezone().replace(tzinfo=None)
token_valid = bool(self._access_token) and (
self._access_expires_at is None
or self._access_expires_at > now + timedelta(minutes=2)
)
if token_valid and not force:
return self._access_token
if not self._refresh_token:
if self._access_token:
return self._access_token
raise IfindError("iFinD Refresh Token 尚未配置。")
payload = self._post(self.AUTH_ENDPOINT, {}, "", self._refresh_token)
self._validate_payload(payload)
data = payload.get("data") or {}
token = str(data.get("access_token") or "").strip()
if not token:
raise IfindError("iFinD 未返回 Access Token。")
expires_at = self._parse_datetime(data.get("expired_time"))
self._access_token = token
self._access_expires_at = expires_at
return token
def _post(
self,
endpoint: str,
body: dict[str, Any],
access_token: str,
refresh_token: str = "",
) -> dict[str, Any]:
headers = {
"Accept": "application/json",
"Content-Type": "application/json",
"User-Agent": "XiaobaiReviewWeb/1.0",
"ifindlang": "cn",
}
if access_token:
headers["access_token"] = access_token
if refresh_token:
headers["refresh_token"] = refresh_token
request = urllib.request.Request(
f"{self.BASE_URL}/{endpoint}",
data=json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
headers=headers,
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
payload = json.loads(response.read().decode("utf-8"))
except urllib.error.HTTPError as exc:
detail = ""
try:
detail_payload = json.loads(exc.read().decode("utf-8", errors="replace"))
detail = str(detail_payload.get("errmsg") or detail_payload.get("message") or "")
except (json.JSONDecodeError, OSError):
pass
raise IfindError(f"iFinD HTTP {exc.code}{f'：{detail[:160]}' if detail else ''}") from exc
except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
raise IfindError("iFinD 数据请求失败。") from exc
if not isinstance(payload, dict):
raise IfindError("iFinD 返回格式不正确。")
return payload
def _cached(self, key: str, ttl: int) -> dict[str, Any] | None:
with self._cache_lock:
cached = self._cache.get(key)
if not cached:
return None
if time.time() - float(cached.get("created_at") or 0) > ttl:
self._cache.pop(key, None)
return None
return copy.deepcopy(cached["payload"])
def _invalidate_access_token(self) -> None:
with self._token_lock:
self._access_token = ""
self._access_expires_at = None
@classmethod
def _validate_payload(cls, payload: dict[str, Any]) -> None:
try:
error_code = int(payload.get("errorcode") or 0)
except (TypeError, ValueError):
error_code = -1
if error_code != 0:
message = str(payload.get("errmsg") or "未知错误")
raise IfindError(f"iFinD 返回错误：{message[:200]}")
@classmethod
def _is_auth_error(cls, payload: dict[str, Any]) -> bool:
try:
error_code = int(payload.get("errorcode") or 0)
except (TypeError, ValueError):
error_code = 0
message = str(payload.get("errmsg") or "").casefold()
return error_code in cls.AUTH_ERROR_CODES or "token" in message or "鉴权" in message
@staticmethod
def _table_rows(payload: dict[str, Any]) -> list[dict[str, Any]]:
tables = payload.get("tables") or []
if isinstance(tables, dict):
tables = [tables]
rows: list[dict[str, Any]] = []
for block in tables if isinstance(tables, list) else []:
if not isinstance(block, dict):
continue
table = block.get("table") or {}
if not isinstance(table, dict):
continue
times = block.get("time") or []
codes = block.get("thscode") or block.get("thscodes") or []
if isinstance(codes, str):
codes = [codes]
lengths = [len(value) for value in table.values() if isinstance(value, list)]
row_count = max(lengths or [len(times) if isinstance(times, list) else 0, 1 if table else 0])
for index in range(row_count):
row: dict[str, Any] = {}
if isinstance(times, list) and index < len(times):
row["time"] = times[index]
if codes:
row["thscode"] = codes[index] if index < len(codes) else codes[0]
for field, values in table.items():
if isinstance(values, list):
row[field] = values[index] if index < len(values) else None
elif index == 0:
row[field] = values
rows.append(row)
return rows
@staticmethod
def _codes(codes: str | list[str]) -> str:
if isinstance(codes, list):
values = [str(code or "").strip().upper() for code in codes]
else:
values = [part.strip().upper() for part in str(codes or "").split(",")]
values = [value for value in values if value]
if not values:
raise IfindError("iFinD 证券代码不能为空。")
if len(values) > 100:
raise IfindError("iFinD 单次证券代码过多。")
return ",".join(values)
@staticmethod
def _display_date(value: str) -> str:
compact = str(value or "").replace("-", "")
if len(compact) != 8 or not compact.isdigit():
raise IfindError("iFinD 日期格式不正确。")
return f"{compact[:4]}-{compact[4:6]}-{compact[6:]}"
@staticmethod
def _parse_datetime(value: Any) -> datetime | None:
text = str(value or "").strip()
if not text:
return None
try:
return datetime.fromisoformat(text)
except ValueError:
return None
realtime_aggregator.py
+4 -423
View File
@@ -1,426 +1,7 @@
from __future__ import annotations """Compatibility alias for the canonical display-only realtime observer."""
import copy import sys
import http.client
import json
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from datetime import datetime
from threading import Lock
from typing import Any, ClassVar
from backend.data import realtime as _implementation
class RealtimeAggregateError(RuntimeError): sys.modules[__name__] = _implementation
pass
EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get"
EASTMONEY_SECTOR_URL = "https://push2.eastmoney.com/api/qt/clist/get"
TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006"
THS_LIMIT_URL = "https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool"
XGB_POOL_URL = "https://flash-api.xuangubao.cn/api/pool/detail"
BROWSER_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/138.0.0.0 Safari/537.36"
)
@dataclass
class WebRealtimeAggregator:
timeout: int = 8
retry_attempts: int = 3
retry_delay_seconds: float = 0.2
response_cache_ttl_seconds: int = 90
_sector_cache: ClassVar[dict[str, Any]] = {}
_sector_cache_lock: ClassVar[Lock] = Lock()
_response_cache: ClassVar[dict[str, dict[str, Any]]] = {}
_response_cache_lock: ClassVar[Lock] = Lock()
def health_snapshot(self, sector: str = "") -> dict[str, Any]:
started = time.perf_counter()
sources: dict[str, dict[str, Any]] = {}
indices: list[dict[str, Any]] = []
sector_payload: dict[str, Any] | None = None
indices, sources["eastmoney_indices"] = self._capture(self.eastmoney_indices)
if sector.strip():
sector_payload, sources["eastmoney_sector"] = self._capture(
lambda: self.eastmoney_sector(sector)
)
ths_observation, sources["ths_limit_pool"] = self._capture(self.ths_limit_pool)
xgb_observation, sources["xgb_limit_pool"] = self._capture(self.xgb_limit_pool)
index_times = [int(item.get("quote_time_epoch") or 0) for item in indices or []]
now = datetime.now().astimezone()
max_skew = 120 if now.hour >= 15 else 15
index_consistent = bool(index_times) and max(index_times) - min(index_times) <= max_skew
ready = (
bool(indices)
and len(indices) == 3
and index_consistent
and (not sector.strip() or bool(sector_payload))
)
return {
"ready": ready,
"isolated": True,
"generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"indices": indices or [],
"index_consistent": index_consistent,
"sector": sector_payload,
"sources": sources,
"observations": {
"ths_limit_pool": ths_observation,
"xgb_limit_pool": xgb_observation,
},
"policy": {
"integration": "heaven_realtime_fallback",
"max_index_time_skew_seconds": max_skew,
"notice": "聚合源仅作为盘中观势的实时指数与板块外显，主行情快照仍由Tushare维护。",
},
}
def eastmoney_indices(self) -> list[dict[str, Any]]:
try:
payload = self._get_json(
EASTMONEY_INDEX_URL,
{
"secids": "1.000001,0.399001,0.399006",
"fltt": "2",
"invt": "2",
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124",
},
referer="https://quote.eastmoney.com/",
)
except RealtimeAggregateError:
return self.tencent_indices()
cache_meta = payload.get("_aggregate_cache") or {}
rows = list((payload.get("data") or {}).get("diff") or [])
result = []
for row in rows:
code = str(row.get("f12") or "")
if code not in {"000001", "399001", "399006"}:
continue
epoch = int(_number(row.get("f124")))
result.append(
{
"code": code,
"name": row.get("f14") or code,
"price": _number(row.get("f2")),
"change": _number(row.get("f3")),
"change_amount": _number(row.get("f4")),
"open": _number(row.get("f17")),
"high": _number(row.get("f15")),
"low": _number(row.get("f16")),
"previous_close": _number(row.get("f18")),
"amount_billion": round(_number(row.get("f6")) / 100000000, 2),
"quote_time_epoch": epoch,
"quote_time": (
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
if epoch else ""
),
"source": (
"eastmoney_push2_cache" if cache_meta else "eastmoney_push2"
),
"cache_age_seconds": cache_meta.get("age_seconds", 0),
}
)
if len(result) != 3:
raise RealtimeAggregateError(f"Eastmoney returned {len(result)}/3 indices")
return result
def tencent_indices(self) -> list[dict[str, Any]]:
raw, cache_age = self._get_text(
TENCENT_INDEX_URL,
referer="https://gu.qq.com/",
encoding="gb18030",
)
result = []
for line in raw.splitlines():
if '="' not in line:
continue
fields = line.split('="', 1)[1].rsplit('";', 1)[0].split("~")
if len(fields) < 38:
continue
code = fields[2]
if code not in {"000001", "399001", "399006"}:
continue
try:
quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S").astimezone()
except ValueError as exc:
raise RealtimeAggregateError(
f"Tencent returned invalid quote time for {code}"
) from exc
result.append(
{
"code": code,
"name": fields[1] or code,
"price": _number(fields[3]),
"change": _number(fields[32]),
"change_amount": _number(fields[31]),
"open": _number(fields[5]),
"high": _number(fields[33]),
"low": _number(fields[34]),
"previous_close": _number(fields[4]),
"amount_billion": round(_number(fields[37]) / 10000, 2),
"quote_time_epoch": int(quote_time.timestamp()),
"quote_time": quote_time.isoformat(timespec="seconds"),
"source": "tencent_qt_cache" if cache_age else "tencent_qt",
"cache_age_seconds": cache_age,
}
)
if len(result) != 3:
raise RealtimeAggregateError(f"Tencent returned {len(result)}/3 indices")
return result
def eastmoney_sector(self, query: str) -> dict[str, Any]:
target = _normalize_sector(query)
candidates = self._eastmoney_sector_catalog()
matched = _match_sector(candidates, target)
if not matched:
raise RealtimeAggregateError(f"Eastmoney sector not found: {query}")
epoch = int(_number(matched.get("f124")))
return {
"code": matched.get("f12") or "",
"name": matched.get("f14") or query,
"price": _number(matched.get("f2")),
"change": _number(matched.get("f3")),
"change_amount": _number(matched.get("f4")),
"turnover_rate": _number(matched.get("f8")),
"up_count": int(_number(matched.get("f104"))),
"down_count": int(_number(matched.get("f105"))),
"leader": matched.get("f128") or "--",
"leader_code": matched.get("f140") or "",
"leading_pct": _number(matched.get("f136")),
"quote_time_epoch": epoch,
"quote_time": (
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
if epoch else ""
),
"source": "eastmoney_push2",
"match_query": query,
}
def _eastmoney_sector_catalog(self) -> list[dict[str, Any]]:
now = time.time()
with self._sector_cache_lock:
cached = self._sector_cache.get("eastmoney")
if cached and now - float(cached.get("created_at") or 0) < 600:
return list(cached.get("rows") or [])
def load_page(page: int) -> list[dict[str, Any]]:
payload = self._get_json(
EASTMONEY_SECTOR_URL,
{
"pn": str(page),
"pz": "100",
"po": "1",
"np": "1",
"fltt": "2",
"invt": "2",
"fid": "f3",
"fs": "m:90+t:2",
"fields": "f12,f14,f2,f3,f4,f8,f104,f105,f128,f136,f140,f124",
},
referer="https://quote.eastmoney.com/center/boardlist.html",
)
return list((payload.get("data") or {}).get("diff") or [])
with ThreadPoolExecutor(max_workers=5) as executor:
pages = list(executor.map(load_page, range(1, 6)))
rows = [row for page in pages for row in page]
if not rows:
raise RealtimeAggregateError("Eastmoney sector catalog is empty")
with self._sector_cache_lock:
self._sector_cache["eastmoney"] = {"created_at": now, "rows": rows}
return rows
def ths_limit_pool(self) -> dict[str, Any]:
payload = self._get_json(
THS_LIMIT_URL,
{"page": "1", "limit": "3", "field": "199112"},
referer="https://data.10jqka.com.cn/limit_up/",
)
data = payload.get("data") or payload
return {
"available": True,
"keys": sorted(str(key) for key in data.keys()) if isinstance(data, dict) else [],
"source": "ths_web_dataapi",
}
def xgb_limit_pool(self) -> dict[str, Any]:
payload = self._get_json(
XGB_POOL_URL,
{"pool_name": "limit_up"},
referer="https://xuangubao.cn/",
)
data = payload.get("data") or {}
rows = data if isinstance(data, list) else data.get("pool") or data.get("list") or []
return {
"available": True,
"count": len(rows) if isinstance(rows, list) else 0,
"source": "xuangubao_web_api",
}
def _capture(self, operation):
started = time.perf_counter()
try:
value = operation()
return value, {
"ok": True,
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"error": "",
}
except Exception as exc:
return None, {
"ok": False,
"elapsed_ms": round((time.perf_counter() - started) * 1000),
"error": str(exc)[:500],
}
def _get_json(
self,
url: str,
params: dict[str, str],
referer: str,
) -> dict[str, Any]:
request_url = f"{url}?{urllib.parse.urlencode(params)}"
last_error: Exception | None = None
attempts = max(1, int(self.retry_attempts))
for attempt in range(attempts):
request = urllib.request.Request(
request_url,
headers={
"Accept": "application/json,text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
content_type = response.headers.get("Content-Type", "")
raw = response.read().decode("utf-8", errors="replace")
if "json" not in content_type.lower() and not raw.lstrip().startswith(("{", "[")):
raise RealtimeAggregateError(
f"non-JSON response: {raw[:120].strip()}"
)
payload = json.loads(raw)
if not isinstance(payload, dict):
raise RealtimeAggregateError("unexpected response shape")
if payload.get("rc") not in (None, 0):
raise RealtimeAggregateError(f"provider rc={payload.get('rc')}")
with self._response_cache_lock:
self._response_cache[request_url] = {
"created_at": time.time(),
"payload": copy.deepcopy(payload),
}
return payload
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
json.JSONDecodeError,
RealtimeAggregateError,
) as exc:
last_error = exc
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
time.sleep(self.retry_delay_seconds * (attempt + 1))
now = time.time()
with self._response_cache_lock:
cached = self._response_cache.get(request_url)
cache_age = now - float((cached or {}).get("created_at") or 0)
if cached and cache_age <= self.response_cache_ttl_seconds:
payload = copy.deepcopy(cached.get("payload") or {})
payload["_aggregate_cache"] = {"age_seconds": round(cache_age, 1)}
return payload
raise RealtimeAggregateError(f"request failed after {attempts} attempts: {last_error}") from last_error
def _get_text(
self,
request_url: str,
referer: str,
encoding: str = "utf-8",
) -> tuple[str, float]:
cache_key = f"text:{request_url}"
last_error: Exception | None = None
attempts = max(1, int(self.retry_attempts))
for attempt in range(attempts):
request = urllib.request.Request(
request_url,
headers={
"Accept": "text/plain,*/*",
"Connection": "close",
"Referer": referer,
"User-Agent": BROWSER_USER_AGENT,
},
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
raw = response.read().decode(encoding, errors="replace")
if not raw.strip():
raise RealtimeAggregateError("empty text response")
with self._response_cache_lock:
self._response_cache[cache_key] = {
"created_at": time.time(),
"payload": raw,
}
return raw, 0
except (
urllib.error.URLError,
TimeoutError,
ConnectionError,
OSError,
http.client.HTTPException,
RealtimeAggregateError,
) as exc:
last_error = exc
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
time.sleep(self.retry_delay_seconds * (attempt + 1))
now = time.time()
with self._response_cache_lock:
cached = self._response_cache.get(cache_key)
cache_age = now - float((cached or {}).get("created_at") or 0)
if cached and cache_age <= self.response_cache_ttl_seconds:
return str(cached.get("payload") or ""), round(cache_age, 1)
raise RealtimeAggregateError(
f"text request failed after {attempts} attempts: {last_error}"
) from last_error
def _normalize_sector(value: Any) -> str:
text = str(value or "").strip().replace(" ", "")
for suffix in ("板块", "概念", "行业", "Ⅱ", "Ⅲ", "(A股)", "（A股）"):
text = text.replace(suffix, "")
aliases = {"元器件": "元件", "电子元器件": "元件"}
return aliases.get(text, text)
def _match_sector(rows: list[dict[str, Any]], target: str) -> dict[str, Any] | None:
exact = [row for row in rows if _normalize_sector(row.get("f14")) == target]
if exact:
return min(exact, key=lambda row: len(str(row.get("f14") or "")))
fuzzy = [
row for row in rows
if target and (
target in _normalize_sector(row.get("f14"))
or _normalize_sector(row.get("f14")) in target
)
]
return min(fuzzy, key=lambda row: len(_normalize_sector(row.get("f14")))) if fuzzy else None
def _number(value: Any, default: float = 0.0) -> float:
try:
return float(value)
except (TypeError, ValueError):
return default
tests/test_data_gateway.py
+5 -1
View File
@@ -48,7 +48,11 @@ class DataGatewayTests(unittest.TestCase):
from pathlib import Path from pathlib import Path
source = ( source = (
Path(__file__).resolve().parents[1] / "backend" / "application.py" Path(__file__).resolve().parents[1]
/ "backend"
/ "features"
/ "market"
/ "service.py"
).read_text(encoding="utf-8") ).read_text(encoding="utf-8")
self.assertEqual(source.count("TushareClient(self.token)"), 1) self.assertEqual(source.count("TushareClient(self.token)"), 1)
self.assertIn("return gateway.tushare()", source) self.assertIn("return gateway.tushare()", source)
tests/test_preservation_slice_market.py
+159
View File
@@ -0,0 +1,159 @@
from __future__ import annotations
import ast
import hashlib
import unittest
from pathlib import Path
import chart_data_provider
import ifind_client
import realtime_aggregator
import tushare_client
from backend.data import realtime
from backend.data.providers import ifind_client as canonical_ifind
from backend.data.providers import tushare_client as canonical_tushare
from backend.features.market import charts
APP_ROOT = Path(__file__).resolve().parents[1]
ORIGINAL_ROOT = APP_ROOT.parent
MARKET_METHODS = {
"_tushare_client",
"get_dashboard",
"_dashboard_sentiment_ready",
"_display_compact_date",
"_carry_dashboard",
"_realtime_snapshot_due",
"sync_dashboard",
"realtime_aggregate_health",
"_search_market_directory",
"_search_match_score",
"search_entities",
"get_search_detail",
"get_intraday_chart",
"_ths_search_detail",
"_index_search_detail",
"get_stock_detail",
"_stock_detail_bar_date",
"_stock_detail_cache_needs_refresh",
"_prepare_stock_detail",
"_sanitize_stock_detail_prices",
"_valid_realtime_stock_quote",
"_ifind_realtime_stock_quote",
"_merge_realtime_stock_detail",
"get_stock_preview",
"backfill",
"_stock_identity",
"_enrich_stock_detail",
"_with_storage",
"_record_count",
}
MARKET_REPOSITORY_METHODS = {
"get_snapshot",
"get_latest_real_snapshot",
"save_snapshot",
"get_data_snapshot",
"get_latest_data_snapshot",
"save_data_snapshot",
"search_stock_master",
"list_snapshot_payloads",
"start_sync",
"finish_sync",
"status",
}
def class_methods(path: Path, class_name: str) -> dict[str, str]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
owner = next(
node
for node in tree.body
if isinstance(node, ast.ClassDef) and node.name == class_name
)
return {
node.name: ast.dump(node, include_attributes=False)
for node in owner.body
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
}
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def top_level_definitions(path: Path) -> dict[str, str]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
return {
node.name: ast.dump(node, include_attributes=False)
for node in tree.body
if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef))
}
class MarketSliceSourceEquivalenceTests(unittest.TestCase):
def test_market_service_methods_are_exact_original_ast(self) -> None:
original = class_methods(ORIGINAL_ROOT / "server.py", "DashboardService")
migrated = class_methods(
APP_ROOT / "backend" / "features" / "market" / "service.py",
"MarketServiceMixin",
)
self.assertEqual(set(migrated), MARKET_METHODS)
for name in sorted(MARKET_METHODS):
self.assertEqual(migrated[name], original[name], name)
def test_market_repository_methods_are_exact_original_ast(self) -> None:
original = class_methods(ORIGINAL_ROOT / "database.py", "ReviewDatabase")
migrated = class_methods(
APP_ROOT / "backend" / "features" / "market" / "repository.py",
"MarketRepositoryMixin",
)
self.assertEqual(set(migrated), MARKET_REPOSITORY_METHODS)
for name in sorted(MARKET_REPOSITORY_METHODS):
self.assertEqual(migrated[name], original[name], name)
def test_original_classes_no_longer_duplicate_moved_methods(self) -> None:
remaining_service = class_methods(APP_ROOT / "backend" / "application.py", "DashboardService")
remaining_database = class_methods(APP_ROOT / "database.py", "ReviewDatabase")
self.assertTrue(MARKET_METHODS.isdisjoint(remaining_service))
self.assertTrue(MARKET_REPOSITORY_METHODS.isdisjoint(remaining_database))
def test_provider_compatibility_modules_are_canonical_aliases(self) -> None:
self.assertIs(tushare_client.TushareClient, canonical_tushare.TushareClient)
self.assertIs(ifind_client.IfindHttpClient, canonical_ifind.IfindHttpClient)
self.assertIs(realtime_aggregator.WebRealtimeAggregator, realtime.WebRealtimeAggregator)
self.assertIs(chart_data_provider.MarketChartClient, charts.MarketChartClient)
def test_provider_logic_is_the_original_implementation(self) -> None:
exact_moves = (
("tushare_client.py", "backend/data/providers/tushare_client.py"),
("ifind_client.py", "backend/data/providers/ifind_client.py"),
("realtime_aggregator.py", "backend/data/realtime.py"),
)
for original, migrated in exact_moves:
self.assertEqual(sha256(ORIGINAL_ROOT / original), sha256(APP_ROOT / migrated))
self.assertEqual(
top_level_definitions(ORIGINAL_ROOT / "chart_data_provider.py"),
top_level_definitions(APP_ROOT / "backend/features/market/charts.py"),
)
def test_unchanged_frontend_assets_match_the_original(self) -> None:
for relative in (
"index.html",
"app.js",
"styles.css",
"renovation.css",
"redesign-v2.css",
"theme.css",
"wentian-v2.css",
):
self.assertEqual(
sha256(APP_ROOT / "static" / relative),
sha256(ORIGINAL_ROOT / "static" / relative),
relative,
)
if __name__ == "__main__":
unittest.main()
tests/test_stock_detail_realtime.py
+6 -6
View File
@@ -90,8 +90,8 @@ class StockDetailRealtimeTests(unittest.TestCase):
"moneyflow": {}, "moneyflow": {},
} }
with patch("backend.application.datetime", FixedMarketDatetime), patch( with patch("backend.features.market.service.datetime", FixedMarketDatetime), patch(
"backend.application.TushareClient", RealtimeClientStub "backend.features.market.service.TushareClient", RealtimeClientStub
): ):
result = self.service._prepare_stock_detail(cached, "002141", today) result = self.service._prepare_stock_detail(cached, "002141", today)
@@ -112,8 +112,8 @@ class StockDetailRealtimeTests(unittest.TestCase):
"stock": {"code": "002141", "price": 10, "change": 1.2}, "stock": {"code": "002141", "price": 10, "change": 1.2},
"prices": [{"trade_date": historical, "close": 10, "change": 1.2}], "prices": [{"trade_date": historical, "close": 10, "change": 1.2}],
} }
with patch("backend.application.datetime", FixedMarketDatetime), patch( with patch("backend.features.market.service.datetime", FixedMarketDatetime), patch(
"backend.application.TushareClient", RealtimeClientStub "backend.features.market.service.TushareClient", RealtimeClientStub
): ):
result = self.service._prepare_stock_detail(payload, "002141", historical) result = self.service._prepare_stock_detail(payload, "002141", historical)
@@ -151,8 +151,8 @@ class StockDetailRealtimeTests(unittest.TestCase):
}, },
], ],
} }
with patch("backend.application.datetime", FixedPreopenDatetime), patch( with patch("backend.features.market.service.datetime", FixedPreopenDatetime), patch(
"backend.application.TushareClient", RealtimeClientStub "backend.features.market.service.TushareClient", RealtimeClientStub
): ):
result = self.service._prepare_stock_detail(payload, "002141", today) result = self.service._prepare_stock_detail(payload, "002141", today)
tools/move_class_methods.py
+76
View File
@@ -0,0 +1,76 @@
from __future__ import annotations
import argparse
import ast
from pathlib import Path
MARKER = " # PRESERVATION_METHODS\n"
def method_span(node: ast.FunctionDef | ast.AsyncFunctionDef) -> tuple[int, int]:
start = min((decorator.lineno for decorator in node.decorator_list), default=node.lineno)
if node.end_lineno is None:
raise ValueError(f"Missing end position for {node.name}")
return start - 1, node.end_lineno
def move_methods(
source_path: Path,
class_name: str,
target_path: Path,
method_names: list[str],
) -> None:
source = source_path.read_text(encoding="utf-8")
tree = ast.parse(source, filename=str(source_path))
owner = next(
(
node
for node in tree.body
if isinstance(node, ast.ClassDef) and node.name == class_name
),
None,
)
if owner is None:
raise ValueError(f"Class not found: {class_name}")
methods = {
node.name: node
for node in owner.body
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
}
missing = [name for name in method_names if name not in methods]
if missing:
raise ValueError(f"Methods not found in {class_name}: {', '.join(missing)}")
lines = source.splitlines(keepends=True)
ordered = sorted((methods[name] for name in method_names), key=lambda node: node.lineno)
blocks = ["".join(lines[start:end]).rstrip() for start, end in map(method_span, ordered)]
for start, end in sorted(map(method_span, ordered), reverse=True):
del lines[start:end]
while start < len(lines) - 1 and lines[start] == "\n" and lines[start + 1] == "\n":
del lines[start]
target = target_path.read_text(encoding="utf-8")
if target.count(MARKER) != 1:
raise ValueError(f"Target must contain exactly one method marker: {target_path}")
target = target.replace(MARKER, "\n\n".join(blocks) + "\n")
source_path.write_text("".join(lines), encoding="utf-8")
target_path.write_text(target, encoding="utf-8")
def main() -> None:
parser = argparse.ArgumentParser(description="Mechanically move class methods between modules")
parser.add_argument("--source", type=Path, required=True)
parser.add_argument("--class-name", required=True)
parser.add_argument("--target", type=Path, required=True)
parser.add_argument("methods", nargs="+")
args = parser.parse_args()
move_methods(args.source, args.class_name, args.target, args.methods)
if __name__ == "__main__":
main()
tushare_client.py
+4 -2172
View File
File diff suppressed because it is too large Load Diff