Compare commits
36
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4a90c32fcc | ||
|
|
5385bad9ec | ||
|
|
955785b9fd | ||
|
|
88411efa1c | ||
|
|
c3881af708 | ||
|
|
5f343a6bae | ||
|
|
abee8306be | ||
|
|
cb45d742da | ||
|
|
16843bdc7e | ||
|
|
b640e77264 | ||
|
|
b950ea4344 | ||
|
|
41f8509a98 | ||
|
|
c9e2d30780 | ||
|
|
8a7d1f3698 | ||
|
|
100752f43c | ||
|
|
0b8419abca | ||
|
|
ef13d6feb5 | ||
|
|
b5d65ecb41 | ||
|
|
3e828b346c | ||
|
|
c8a9376adb | ||
|
|
1c2f2ac057 | ||
|
|
5d3465987d | ||
|
|
dd89a09643 | ||
|
|
a043bc9eb1 | ||
|
|
acde4de40d | ||
|
|
3d2c1252f1 | ||
|
|
605f97e5df | ||
|
|
16ba83ec01 | ||
|
|
1c740a9d48 | ||
|
|
75c2e33b68 | ||
|
|
32f565ecb9 | ||
|
|
16841e9ae3 | ||
|
|
bed6450992 | ||
|
|
c9892050c3 | ||
|
|
a836cda1b2 | ||
|
|
25ff6bbe06 |
+9
-7
@@ -1,18 +1,20 @@
|
||||
# Generated automatically when omitted. Back it up together with the database.
|
||||
APP_ENCRYPTION_KEY=
|
||||
|
||||
# Initial shared market-data credential. After first launch it is encrypted into
|
||||
# the system settings; all accounts use the same backend market snapshot.
|
||||
# Market-source credentials are consumed and encrypted only by xiaobai-datahub.
|
||||
# compose.yaml masks them from the xiaobai-review website process.
|
||||
TUSHARE_TOKEN=your_tushare_token_here
|
||||
|
||||
# Optional xiaobai-datahub client. All DATAHUB_READ_* / DATAHUB_SHADOW_* flags
|
||||
# default off in config/datahub.config.json, so the website keeps using Tushare.
|
||||
# Official xiaobai-datahub client. Read flags default on in config/datahub.config.json.
|
||||
# compose.yaml pins every DATAHUB_READ_* to 1. The website has no provider
|
||||
# fallback; source selection and failover happen inside xiaobai-datahub.
|
||||
# DATAHUB_SHADOW_* can still override a single dataset.
|
||||
DATAHUB_BASE_URL=http://127.0.0.1:8766
|
||||
DATAHUB_TOKEN=
|
||||
|
||||
# Optional iFinD HTTP credential. The backend exchanges it for a short-lived
|
||||
# access token and never exposes either token to browsers.
|
||||
IFIND_REFRESH_TOKEN=your_ifind_refresh_token_here
|
||||
# iFinD credentials live on xiaobai-datahub, not the website process.
|
||||
# IFIND_REFRESH_TOKEN=your_ifind_refresh_token_here
|
||||
# IFIND_ACCESS_TOKEN=
|
||||
|
||||
# Initial platform member models (OpenAI-compatible). After first launch these
|
||||
# are encrypted into system settings and used only by admins and active members.
|
||||
|
||||
+8
-3
@@ -38,10 +38,15 @@ background scheduler
|
||||
fields, and feature-specific exceptions belong to `backend/features/<feature>/routes.py`.
|
||||
- `backend/features/<feature>/` owns the mechanically moved service, repository, HTTP, agent,
|
||||
or deterministic calculation code for that product area.
|
||||
- `backend/data/` owns provider construction, source policy, provenance, units, freshness,
|
||||
coverage, display-versus-calculation eligibility, and shared numeric normalization policies.
|
||||
- `backend/data/` owns the website-side DataHub client, stable dataset contracts, provenance,
|
||||
units, freshness, coverage, display-versus-calculation eligibility, and shared numeric
|
||||
normalization policies. The website process does not construct or configure external market
|
||||
providers; provider credentials, source selection, retries, fallbacks, caching, and backfill
|
||||
belong exclusively to the `xiaobai-datahub` service.
|
||||
- `backend/data/providers/tushare_client.py` is the stable public `TushareClient` facade and
|
||||
owns only its dataclass fields and shared cache state. Tushare HTTP transport belongs to
|
||||
is retained as the dataset-contract compatibility surface and isolated test facade. Production
|
||||
website services never instantiate it directly: its query methods are served by the DataHub
|
||||
proxy. Its split modules document the stable contract: Tushare HTTP transport belongs to
|
||||
`tushare_transport.py`; market overview and realtime breadth belong to
|
||||
`tushare_dashboard.py`; indices belong to `tushare_indices.py`; Shenwan membership and
|
||||
industry snapshots belong to `tushare_industries.py`; generic sector snapshots belong to
|
||||
|
||||
+7
-4
@@ -22,9 +22,10 @@ xiaobai-review 容器 :8765
|
||||
`-- heaven_knowledge.json 优先读取;缺失时回退到上方 seed
|
||||
```
|
||||
|
||||
账号、加密后的公共数据 Token、平台模型 API Key、生辰资料、行情快照和复盘数据均在
|
||||
`data/review.db`。解密密钥来自 `.env` 中的 `APP_ENCRYPTION_KEY`。数据库与
|
||||
密钥必须成对备份,任意一个丢失都无法恢复账号内的加密资料。
|
||||
账号、平台模型 API Key、生辰资料、行情快照和复盘数据均在 `data/review.db`。外部行情源
|
||||
Token 只允许保存在 `xiaobai-datahub` 的环境或凭据库,网站进程不读取、不保存,也不向
|
||||
提供方直接发请求。解密密钥来自 `.env` 中的 `APP_ENCRYPTION_KEY`。数据库与密钥必须成对
|
||||
备份,任意一个丢失都无法恢复账号内的加密资料。
|
||||
|
||||
问天静态知识文件:
|
||||
|
||||
@@ -41,7 +42,9 @@ xiaobai-review 容器 :8765
|
||||
挂载进入容器,但被 Git 与 Docker 构建上下文排除,不会进入 Gitea 或镜像。私有 Skill
|
||||
只对管理员账号返回和开放调用,也会随本指南的 `data` 备份一起保存。
|
||||
|
||||
首个注册账号自动成为管理员。管理员在“系统管理”中配置全站共享行情、后台刷新、平台会员模型及手动会员;普通用户的“账号设置”用于个人资料、会员状态、修改密码和切换账号。后台行情更新不会主动刷新任何浏览器页面。
|
||||
首个注册账号自动成为管理员。管理员在网站“系统管理”中查看数据中枢状态并配置后台刷新、
|
||||
平台会员模型及手动会员;行情源凭据和调度策略在数据中枢后台统一管理。普通用户的“账号设置”
|
||||
用于个人资料、会员状态、修改密码和切换账号。后台行情更新不会主动刷新任何浏览器页面。
|
||||
|
||||
## 2. 服务器要求
|
||||
|
||||
|
||||
@@ -60,22 +60,6 @@ from backend.llm.service import LLMServiceMixin
|
||||
from database import ReviewDatabase
|
||||
|
||||
|
||||
LEGACY_SECRET_KEYS = {
|
||||
"TUSHARE_TOKEN",
|
||||
"IFIND_REFRESH_TOKEN",
|
||||
"IFIND_ACCESS_TOKEN",
|
||||
"LLM_API_KEY",
|
||||
"LLM_BASE_URL",
|
||||
"LLM_MODEL",
|
||||
"LLM_PRIMARY_API_KEY",
|
||||
"LLM_PRIMARY_BASE_URL",
|
||||
"LLM_PRIMARY_MODEL",
|
||||
"LLM_FALLBACK_API_KEY",
|
||||
"LLM_FALLBACK_BASE_URL",
|
||||
"LLM_FALLBACK_MODEL",
|
||||
}
|
||||
|
||||
|
||||
class DashboardService(
|
||||
SystemServiceMixin,
|
||||
AccountApplicationMixin,
|
||||
@@ -121,7 +105,6 @@ class DashboardService(
|
||||
self._system_credentials,
|
||||
MENTOR_SKILLS_DIR,
|
||||
PRIVATE_MENTOR_SKILLS_DIR,
|
||||
lambda: self.token,
|
||||
)
|
||||
self.data_gateway = self.container.data_gateway
|
||||
self.ifind = self.container.ifind
|
||||
|
||||
@@ -2,7 +2,6 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from collections.abc import Callable
|
||||
|
||||
from backend.data import DataGateway, build_data_gateway
|
||||
from backend.database.repositories import RepositoryBundle, build_repository_bundle
|
||||
@@ -13,8 +12,8 @@ from backend.features.screener.engine import ScreenerEngine
|
||||
from backend.features.screener.tracking import StrategyTrackingService
|
||||
from backend.jobs import InProcessJobRunner, JobRegistry, SQLiteJobRunRepository
|
||||
from database import ReviewDatabase
|
||||
from backend.data.providers.ifind_client import IfindHttpClient
|
||||
from backend.data.realtime import WebRealtimeAggregator
|
||||
from backend.data.datahub.ifind_proxy import HubIfindProxy
|
||||
from backend.data.datahub.realtime_proxy import HubRealtimeProxy
|
||||
from backend.features.market.charts import MarketChartClient
|
||||
|
||||
|
||||
@@ -23,13 +22,13 @@ class ApplicationContainer:
|
||||
database: ReviewDatabase
|
||||
repositories: RepositoryBundle
|
||||
data_gateway: DataGateway
|
||||
ifind: IfindHttpClient
|
||||
ifind: HubIfindProxy
|
||||
screener: ScreenerEngine
|
||||
strategy_tracking: StrategyTrackingService
|
||||
alert_service: AlertService
|
||||
trade_journal: TradeJournalService
|
||||
mentor_skills: MentorSkillRegistry
|
||||
realtime_aggregator: WebRealtimeAggregator
|
||||
realtime_aggregator: HubRealtimeProxy
|
||||
chart_data: MarketChartClient
|
||||
jobs: InProcessJobRunner
|
||||
|
||||
@@ -39,9 +38,8 @@ def build_application_container(
|
||||
credentials: dict[str, object],
|
||||
mentor_skills_dir: Path,
|
||||
private_mentor_skills_dir: Path,
|
||||
tushare_token_supplier: Callable[[], str] | None = None,
|
||||
) -> ApplicationContainer:
|
||||
data_gateway = build_data_gateway(credentials, tushare_token_supplier)
|
||||
data_gateway = build_data_gateway(credentials)
|
||||
repositories = build_repository_bundle(database)
|
||||
jobs = InProcessJobRunner(JobRegistry.load(), SQLiteJobRunRepository(database))
|
||||
return ApplicationContainer(
|
||||
|
||||
@@ -1,11 +1,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from http.server import ThreadingHTTPServer
|
||||
from typing import Any
|
||||
|
||||
|
||||
def configure_logging() -> None:
|
||||
"""让 INFO 级结构化日志(含 datahub 影子对比报告)落到容器日志。"""
|
||||
if logging.getLogger().handlers:
|
||||
return
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
)
|
||||
|
||||
|
||||
def main(handler_class: type[Any] | None = None, service: Any | None = None) -> None:
|
||||
configure_logging()
|
||||
if handler_class is None or service is None:
|
||||
from backend.application import RequestHandler, SERVICE
|
||||
|
||||
|
||||
@@ -10,11 +10,8 @@ from backend.features.accounts.security import SecretVault
|
||||
|
||||
def environment_credentials(environment: Mapping[str, str]) -> dict[str, str]:
|
||||
return {
|
||||
"tushare_token": str(environment.get("TUSHARE_TOKEN") or "").strip(),
|
||||
"datahub_token": str(environment.get("DATAHUB_TOKEN") or "").strip(),
|
||||
"datahub_base_url": str(environment.get("DATAHUB_BASE_URL") or "").strip(),
|
||||
"ifind_refresh_token": str(environment.get("IFIND_REFRESH_TOKEN") or "").strip(),
|
||||
"ifind_access_token": str(environment.get("IFIND_ACCESS_TOKEN") or "").strip(),
|
||||
"platform_llm_primary_api_key": str(
|
||||
environment.get("LLM_PRIMARY_API_KEY") or environment.get("LLM_API_KEY") or ""
|
||||
).strip(),
|
||||
|
||||
+403
-49
@@ -2,7 +2,8 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from typing import Any, Callable
|
||||
from threading import Lock
|
||||
from typing import Any, Callable, ClassVar
|
||||
|
||||
from backend.data.datahub.client import DatahubClient, DatahubResponse
|
||||
from backend.data.datahub.compare import compare_rows
|
||||
@@ -16,16 +17,47 @@ from backend.data.datahub.native import (
|
||||
yyyymmdd,
|
||||
)
|
||||
from backend.data.datahub.redact import redact_text, redact_value
|
||||
from backend.data.datahub.route_state import LEDGER
|
||||
from backend.data.datahub.settings import DatahubSettings
|
||||
from backend.data.providers.tushare_client import TushareClient
|
||||
from backend.data.providers.tushare_daily import DailyMarketMixin
|
||||
from backend.data.providers.tushare_dashboard import DashboardMixin
|
||||
from backend.data.providers.tushare_dragon_tiger import DragonTigerMixin
|
||||
from backend.data.providers.tushare_indices import IndexMixin
|
||||
from backend.data.providers.tushare_industries import ShenwanIndustryMixin
|
||||
from backend.data.providers.tushare_sectors import SectorMixin
|
||||
from backend.data.providers.tushare_stocks import StockMixin
|
||||
from backend.data.providers.tushare_transport import TushareError
|
||||
|
||||
LOGGER = logging.getLogger("xiaobai.datahub")
|
||||
ShadowSink = Callable[[dict[str, Any]], None]
|
||||
EMPTY_FAIL_DATASETS = {"stocks", "daily", "index_daily", "valuation", "moneyflow", "auction"}
|
||||
|
||||
|
||||
def _usable_intraday_points(rows: list[Any]) -> list[dict[str, Any]]:
|
||||
points: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
try:
|
||||
close = float(row.get("close") or 0)
|
||||
except (TypeError, ValueError):
|
||||
close = 0.0
|
||||
if close <= 0:
|
||||
continue
|
||||
point = dict(row)
|
||||
if "average" not in point and point.get("avg_price") is not None:
|
||||
point["average"] = point.get("avg_price")
|
||||
points.append(point)
|
||||
return points
|
||||
|
||||
|
||||
EMPTY_FAIL_DATASETS = {
|
||||
"stocks", "daily", "index_daily", "valuation", "moneyflow", "auction",
|
||||
"limit_events", "sector_daily",
|
||||
}
|
||||
|
||||
|
||||
def looks_like_heaven(module_name: str, filename: str = "") -> bool:
|
||||
"""问天调用栈识别。问天未永久冻结,只是本阶段仍走旧 Tushare 链路。"""
|
||||
"""问天调用栈识别(诊断用)。问天按数据集依赖接入,不再整栈强制旧链路。"""
|
||||
path = filename.replace("\\", "/")
|
||||
return module_name.startswith("backend.features.heaven") or "/features/heaven/" in path
|
||||
|
||||
@@ -88,53 +120,228 @@ class DatahubBridge:
|
||||
self._log_failure("status", exc)
|
||||
return None
|
||||
|
||||
def try_intraday(self, code: str) -> dict[str, Any] | None:
|
||||
flags = self.settings.flags("intraday")
|
||||
if not flags.read:
|
||||
return None
|
||||
try:
|
||||
response = self.client.intraday_points(code=code)
|
||||
data = response.data
|
||||
if not isinstance(data, dict):
|
||||
raise DatahubError("EMPTY", "datahub intraday payload invalid")
|
||||
points = _usable_intraday_points(data.get("points") or [])
|
||||
if not points:
|
||||
raise DatahubError("EMPTY", "datahub intraday empty")
|
||||
if (response.meta or {}).get("stale"):
|
||||
raise DatahubError("STALE", "datahub intraday stale")
|
||||
self._record_route("intraday", "datahub", str((response.meta or {}).get("source") or "datahub"))
|
||||
return {
|
||||
"entity_type": str(data.get("entity_type") or "stock"),
|
||||
"identifier": str(data.get("identifier") or code),
|
||||
"name": str(data.get("name") or ""),
|
||||
"code": str(data.get("code") or code),
|
||||
"trade_date": str(data.get("trade_date") or points[-1].get("date") or ""),
|
||||
"previous_close": float(data.get("previous_close") or 0),
|
||||
"points": points,
|
||||
"source": "datahub",
|
||||
}
|
||||
except Exception as exc:
|
||||
self._log_failure("intraday", exc)
|
||||
return None
|
||||
|
||||
def try_market_quotes(self, trade_date: str = "") -> list[dict[str, Any]] | None:
|
||||
return self._try_quote_rows("quotes", {}, expected_date=trade_date, minimum=200)
|
||||
|
||||
def try_quotes(self, codes: list[str]) -> list[dict[str, Any]] | None:
|
||||
cleaned = [str(item or "").strip() for item in codes if str(item or "").strip()]
|
||||
if not cleaned:
|
||||
return None
|
||||
return self._try_quote_rows("quotes", {"codes": ",".join(cleaned)}, minimum=1)
|
||||
|
||||
def try_index_quotes(self) -> list[dict[str, Any]] | None:
|
||||
flags = self.settings.flags("index_quotes")
|
||||
if not flags.read:
|
||||
return None
|
||||
try:
|
||||
response = self.client.index_quotes()
|
||||
rows = [dict(item) for item in (response.data or []) if isinstance(item, dict)]
|
||||
if len(rows) < 3:
|
||||
raise DatahubError("EMPTY", "datahub index quotes incomplete")
|
||||
if (response.meta or {}).get("stale"):
|
||||
raise DatahubError("STALE", "datahub index quotes stale")
|
||||
self._record_route(
|
||||
"index_quotes",
|
||||
"datahub",
|
||||
str((response.meta or {}).get("source") or "datahub"),
|
||||
)
|
||||
return rows
|
||||
except Exception as exc:
|
||||
self._log_failure("index_quotes", exc)
|
||||
return None
|
||||
|
||||
def try_sector_quote(self, code: str, trade_date: str = "") -> dict[str, Any] | None:
|
||||
flags = self.settings.flags("quotes")
|
||||
if not flags.read:
|
||||
return None
|
||||
try:
|
||||
response = self.client.sector_quote(code, trade_date)
|
||||
data = response.data
|
||||
if not isinstance(data, dict) or not data:
|
||||
raise DatahubError("EMPTY", "datahub sector quote empty")
|
||||
row = dict(data)
|
||||
if (response.meta or {}).get("stale"):
|
||||
row["delayed"] = True
|
||||
row["delay_seconds"] = int((response.meta or {}).get("staleness_seconds") or 0)
|
||||
row["delay_notice"] = str((response.meta or {}).get("delay_notice") or "")
|
||||
self._record_route("quotes", "datahub", str((response.meta or {}).get("source") or "datahub"))
|
||||
return row
|
||||
except Exception as exc:
|
||||
self._log_failure("quotes", exc)
|
||||
return None
|
||||
|
||||
def try_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]] | None:
|
||||
flags = self.settings.flags("limit_events")
|
||||
if not flags.read:
|
||||
return None
|
||||
try:
|
||||
response = self.client.limit_pool(trade_date)
|
||||
rows = [dict(item) for item in (response.data or []) if isinstance(item, dict)]
|
||||
if not rows:
|
||||
raise DatahubError("EMPTY", "datahub limit pool empty")
|
||||
self._record_route(
|
||||
"limit_events",
|
||||
"datahub",
|
||||
str((response.meta or {}).get("source") or "datahub"),
|
||||
)
|
||||
return rows
|
||||
except Exception as exc:
|
||||
self._log_failure("limit_events", exc)
|
||||
return None
|
||||
|
||||
def try_daily_chart(
|
||||
self,
|
||||
code: str,
|
||||
end_date: str,
|
||||
limit: int = 90,
|
||||
dataset: str = "daily",
|
||||
) -> list[dict[str, Any]] | None:
|
||||
flags = self.settings.flags(dataset)
|
||||
if not flags.read:
|
||||
return None
|
||||
compact_end = yyyymmdd(end_date)
|
||||
if not compact_end:
|
||||
return None
|
||||
try:
|
||||
start = _shift_yyyymmdd(compact_end, -max(190, int(limit) * 3))
|
||||
if dataset == "index_daily":
|
||||
response = self._paginate(
|
||||
self.client.index_bars,
|
||||
{"code": code, "from": start, "to": compact_end},
|
||||
)
|
||||
elif dataset == "sector_daily":
|
||||
response = self._paginate(
|
||||
self.client.sectors,
|
||||
{"code": code, "from": start, "to": compact_end},
|
||||
)
|
||||
else:
|
||||
response = self._paginate(
|
||||
self.client.daily_bars,
|
||||
{"code": code, "from": start, "to": compact_end, "adjust": "none"},
|
||||
)
|
||||
# Charts can use a partial history window; do not discard usable bars
|
||||
# just because the requested lookback is not fully covered.
|
||||
self._validate_usable(
|
||||
dataset,
|
||||
list(response.data or []),
|
||||
response,
|
||||
require_complete=False,
|
||||
)
|
||||
rows = _chart_bars(list(response.data or []))
|
||||
if not rows:
|
||||
raise DatahubError("EMPTY", f"{dataset} chart empty")
|
||||
self._record_route(dataset, "datahub", str((response.meta or {}).get("source") or "datahub"))
|
||||
return rows[-max(1, int(limit)):]
|
||||
except Exception as exc:
|
||||
self._log_failure(dataset, exc)
|
||||
return None
|
||||
|
||||
def record_legacy(self, dataset: str, source: str = "", error: str = "") -> None:
|
||||
self._record_route(dataset, "legacy", source, error)
|
||||
|
||||
def route_snapshot(self) -> list[dict[str, Any]]:
|
||||
return LEDGER.snapshot()
|
||||
|
||||
def _try_quote_rows(
|
||||
self,
|
||||
dataset: str,
|
||||
params: dict[str, Any],
|
||||
expected_date: str = "",
|
||||
minimum: int = 1,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
flags = self.settings.flags(dataset)
|
||||
if not flags.read:
|
||||
return None
|
||||
try:
|
||||
response = self.client.quotes_latest(**params)
|
||||
rows = [_native_quote(item) for item in (response.data or []) if isinstance(item, dict)]
|
||||
rows = [item for item in rows if item]
|
||||
want = yyyymmdd(expected_date)
|
||||
if want:
|
||||
dated = [item for item in rows if not item.get("quote_date") or item.get("quote_date") == want]
|
||||
if dated:
|
||||
rows = dated
|
||||
if len(rows) < minimum:
|
||||
raise DatahubError("EMPTY", f"datahub {dataset} empty")
|
||||
stale = bool((response.meta or {}).get("stale"))
|
||||
delay = int((response.meta or {}).get("staleness_seconds") or 0)
|
||||
notice = str((response.meta or {}).get("delay_notice") or "")
|
||||
source = str((response.meta or {}).get("source") or "datahub")
|
||||
if stale:
|
||||
for item in rows:
|
||||
item["delayed"] = True
|
||||
item["delay_seconds"] = delay
|
||||
item["delay_notice"] = notice
|
||||
item["source"] = source
|
||||
self._record_route(dataset, "datahub", source)
|
||||
return rows
|
||||
except Exception as exc:
|
||||
self._log_failure(dataset, exc)
|
||||
return None
|
||||
|
||||
def query(
|
||||
self,
|
||||
api_name: str,
|
||||
params: dict[str, Any] | None,
|
||||
fields: str,
|
||||
legacy_query: Callable[..., list[dict[str, Any]]],
|
||||
params: dict[str, Any] | None = None,
|
||||
fields: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
if api_name == "rt_sw_k":
|
||||
raise TushareError("rt_sw_k is disabled; use published sw_daily or free Shenwan realtime")
|
||||
dataset = API_TO_DATASET.get(api_name)
|
||||
# 问天允许后续纳入 datahub;首批只读接入仍保持旧链路,避免误切。
|
||||
if not dataset or self.heaven_guard():
|
||||
return legacy_query(api_name, params, fields)
|
||||
if dataset:
|
||||
flags = self.settings.flags(dataset)
|
||||
if not flags.read and not flags.shadow:
|
||||
return legacy_query(api_name, params, fields)
|
||||
|
||||
hub_rows: list[dict[str, Any]] | None = None
|
||||
hub_meta: dict[str, Any] = {}
|
||||
hub_error: str | None = None
|
||||
hub_canonical: list[dict[str, Any]] = []
|
||||
if flags.read:
|
||||
try:
|
||||
response = self._fetch_dataset(dataset, params or {})
|
||||
response = self._fetch_dataset(dataset, params or {}, api_name=api_name)
|
||||
hub_canonical = self._extract_rows(dataset, response, params or {})
|
||||
hub_rows = to_native_rows(dataset, hub_canonical)
|
||||
hub_meta = dict(response.meta)
|
||||
self._validate_usable(dataset, hub_rows, response)
|
||||
self._record_route(dataset, "datahub", str(response.meta.get("source") or "datahub"))
|
||||
return project_fields(hub_rows, fields)
|
||||
except Exception as exc:
|
||||
hub_error = self._error_text(exc)
|
||||
self._log_failure(dataset, exc)
|
||||
|
||||
if flags.shadow:
|
||||
try:
|
||||
legacy_rows = legacy_query(api_name, params, fields)
|
||||
response = self.client.query_api(api_name, params or {}, fields)
|
||||
rows = [dict(item) for item in (response.data or []) if isinstance(item, dict)]
|
||||
if dataset:
|
||||
self._record_route(dataset, "datahub", str((response.meta or {}).get("source") or "datahub"))
|
||||
else:
|
||||
self._record_route(api_name, "datahub", str((response.meta or {}).get("source") or "datahub"))
|
||||
return rows if not fields else project_fields(rows, fields)
|
||||
except Exception as exc:
|
||||
if flags.read and hub_rows is not None and hub_error is None:
|
||||
self._emit_shadow(compare_rows(dataset, [], hub_canonical, hub_meta, self._error_text(exc)))
|
||||
return project_fields(hub_rows, fields)
|
||||
raise
|
||||
self._emit_shadow(compare_rows(dataset, legacy_rows, hub_canonical, hub_meta, hub_error))
|
||||
if flags.read and hub_rows is not None and hub_error is None:
|
||||
return project_fields(hub_rows, fields)
|
||||
return legacy_rows
|
||||
self._log_failure(dataset or api_name, exc)
|
||||
raise TushareError(self._error_text(exc)) from exc
|
||||
|
||||
if flags.read and hub_rows is not None and hub_error is None:
|
||||
return project_fields(hub_rows, fields)
|
||||
return legacy_query(api_name, params, fields)
|
||||
|
||||
def _fetch_dataset(self, dataset: str, params: dict[str, Any]) -> DatahubResponse:
|
||||
def _fetch_dataset(self, dataset: str, params: dict[str, Any], api_name: str = "") -> DatahubResponse:
|
||||
date = yyyymmdd(params.get("trade_date") or params.get("date"))
|
||||
start = yyyymmdd(params.get("start_date") or params.get("from") or date)
|
||||
end = yyyymmdd(params.get("end_date") or params.get("to") or date)
|
||||
@@ -151,6 +358,10 @@ class DatahubBridge:
|
||||
"valuation": self.client.valuation,
|
||||
"moneyflow": self.client.moneyflow,
|
||||
"auction": self.client.auction,
|
||||
"limit_events": self.client.limit_events,
|
||||
"popularity": self.client.popularity,
|
||||
"dragon_tiger": self.client.dragon_tiger,
|
||||
"sector_daily": self.client.sectors,
|
||||
}
|
||||
fetcher = fetchers[dataset]
|
||||
query: dict[str, Any] = {}
|
||||
@@ -165,6 +376,23 @@ class DatahubBridge:
|
||||
query["to"] = end
|
||||
if dataset == "daily":
|
||||
query["adjust"] = "none"
|
||||
if dataset == "limit_events":
|
||||
limit_type = str(params.get("limit_type") or "").strip().upper()
|
||||
if limit_type:
|
||||
query["limit_type"] = limit_type
|
||||
if dataset == "popularity":
|
||||
if api_name == "ths_hot":
|
||||
query["source"] = "ths"
|
||||
elif api_name == "dc_hot":
|
||||
query["source"] = "dc"
|
||||
if dataset == "sector_daily":
|
||||
family = {
|
||||
"ths_daily": "ths",
|
||||
"dc_index": "dc",
|
||||
"sw_daily": "sw",
|
||||
}.get(api_name, "")
|
||||
if family:
|
||||
query["family"] = family
|
||||
return self._paginate(fetcher, query)
|
||||
|
||||
def _paginate(self, fetcher: Callable[..., DatahubResponse], params: dict[str, Any]) -> DatahubResponse:
|
||||
@@ -201,13 +429,23 @@ class DatahubBridge:
|
||||
return filter_stock_rows(rows, params)
|
||||
return rows
|
||||
|
||||
def _validate_usable(self, dataset: str, rows: list[dict[str, Any]], response: DatahubResponse) -> None:
|
||||
def _validate_usable(
|
||||
self,
|
||||
dataset: str,
|
||||
rows: list[dict[str, Any]],
|
||||
response: DatahubResponse,
|
||||
require_complete: bool = True,
|
||||
) -> None:
|
||||
meta = response.meta or {}
|
||||
stale_seconds = int(meta.get("staleness_seconds") or 0)
|
||||
if meta.get("stale") or stale_seconds > self.settings.stale_seconds_max:
|
||||
raise DatahubError("STALE", f"{dataset} data is stale")
|
||||
if dataset in EMPTY_FAIL_DATASETS and not rows:
|
||||
raise DatahubError("EMPTY", f"{dataset} returned no rows")
|
||||
coverage = meta.get("coverage") if isinstance(meta.get("coverage"), dict) else {}
|
||||
if require_complete and (meta.get("incomplete") is True or coverage.get("complete") is False):
|
||||
missing = coverage.get("missing_count")
|
||||
raise DatahubError("INCOMPLETE", f"{dataset} range is incomplete missing={missing}")
|
||||
|
||||
def _require_fresh(self, response: DatahubResponse, dataset: str) -> DatahubResponse:
|
||||
self._validate_usable(dataset, list(response.data or []) if isinstance(response.data, list) else [], response)
|
||||
@@ -220,11 +458,12 @@ class DatahubBridge:
|
||||
self.shadow_sink(report)
|
||||
|
||||
def _log_failure(self, dataset: str, exc: Exception) -> None:
|
||||
LOGGER.warning(
|
||||
"datahub fallback dataset=%s error=%s",
|
||||
dataset,
|
||||
redact_text(self._error_text(exc), self.settings.secrets()),
|
||||
)
|
||||
error = redact_text(self._error_text(exc), self.settings.secrets())
|
||||
LOGGER.warning("datahub unavailable dataset=%s error=%s", dataset, error)
|
||||
self._record_route(dataset, "datahub", "unavailable", error)
|
||||
|
||||
def _record_route(self, dataset: str, route: str, source: str = "", error: str = "") -> None:
|
||||
LEDGER.record(dataset, route, source, redact_text(error, self.settings.secrets()))
|
||||
|
||||
def _error_text(self, exc: Exception) -> str:
|
||||
if isinstance(exc, DatahubError):
|
||||
@@ -234,10 +473,110 @@ class DatahubBridge:
|
||||
return redact_text(text, self.settings.secrets())
|
||||
|
||||
|
||||
class DatahubAwareTushareClient:
|
||||
def __init__(self, legacy: TushareClient, bridge: DatahubBridge) -> None:
|
||||
self._legacy = legacy
|
||||
self._bridge = bridge
|
||||
def _native_quote(row: dict[str, Any]) -> dict[str, Any] | None:
|
||||
ts_code = str(row.get("ts_code") or "").strip()
|
||||
close = _finite(row.get("close") if row.get("close") not in (None, "") else row.get("price"))
|
||||
previous = _finite(
|
||||
row.get("pre_close") if row.get("pre_close") not in (None, "") else row.get("previous_close")
|
||||
)
|
||||
if not ts_code or close <= 0 or previous <= 0:
|
||||
return None
|
||||
volume = _finite(row.get("vol") if row.get("vol") not in (None, "") else row.get("volume"))
|
||||
payload = {
|
||||
"ts_code": ts_code,
|
||||
"name": str(row.get("name") or ts_code).strip(),
|
||||
"pre_close": previous,
|
||||
"open": _finite(row.get("open")),
|
||||
"high": _finite(row.get("high")),
|
||||
"low": _finite(row.get("low")),
|
||||
"close": close,
|
||||
"vol": volume,
|
||||
"amount": _finite(row.get("amount")),
|
||||
"num": 0,
|
||||
"quote_date": yyyymmdd(row.get("quote_date") or row.get("trade_date")),
|
||||
"source": str(row.get("source") or "datahub"),
|
||||
}
|
||||
if row.get("delayed"):
|
||||
payload["delayed"] = True
|
||||
payload["delay_seconds"] = int(row.get("delay_seconds") or 0)
|
||||
payload["delay_notice"] = str(row.get("delay_notice") or "")
|
||||
return payload
|
||||
|
||||
|
||||
def _chart_bars(rows: list[Any]) -> list[dict[str, Any]]:
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
compact = yyyymmdd(row.get("trade_date"))
|
||||
close = _finite(row.get("close"))
|
||||
if len(compact) != 8 or close <= 0:
|
||||
continue
|
||||
volume = _finite(row.get("volume") if row.get("volume") not in (None, "") else row.get("vol"))
|
||||
amount = _finite(row.get("amount"))
|
||||
if volume and volume < close * 10 and amount > 1000:
|
||||
volume = volume * 100
|
||||
trade_date = f"{compact[:4]}-{compact[4:6]}-{compact[6:8]}"
|
||||
previous = normalized[-1]["close"] if normalized else 0.0
|
||||
normalized.append(
|
||||
{
|
||||
"trade_date": trade_date,
|
||||
"open": _finite(row.get("open")),
|
||||
"high": _finite(row.get("high")),
|
||||
"low": _finite(row.get("low")),
|
||||
"close": close,
|
||||
"change": round((close / previous - 1) * 100, 4) if previous else _finite(row.get("pct_chg")),
|
||||
"volume": volume,
|
||||
"amount_billion": amount / 100_000_000,
|
||||
}
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def _shift_yyyymmdd(value: str, days: int) -> str:
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
stamp = datetime.strptime(value, "%Y%m%d")
|
||||
return (stamp + timedelta(days=days)).strftime("%Y%m%d")
|
||||
|
||||
|
||||
def _finite(value: Any) -> float:
|
||||
try:
|
||||
return float(value or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
class DatahubAwareTushareClient(
|
||||
DashboardMixin,
|
||||
IndexMixin,
|
||||
ShenwanIndustryMixin,
|
||||
SectorMixin,
|
||||
DragonTigerMixin,
|
||||
StockMixin,
|
||||
DailyMarketMixin,
|
||||
):
|
||||
"""Website market facade. Mixins call query(); query talks only to the hub."""
|
||||
|
||||
_realtime_reference_cache: ClassVar[dict[str, dict[str, Any]]] = {}
|
||||
_realtime_reference_lock: ClassVar[Lock] = Lock()
|
||||
_capital_cache: ClassVar[dict[str, dict[str, Any]]] = {}
|
||||
_latest_realtime_market: ClassVar[dict[str, dict[str, Any]]] = {}
|
||||
_stock_activity_cache: ClassVar[dict[str, dict[str, Any]]] = {}
|
||||
_stock_listing_cache: ClassVar[dict[str, Any]] = {}
|
||||
_stock_listing_lock: ClassVar[Lock] = Lock()
|
||||
_suspension_cache: ClassVar[dict[str, dict[str, str] | None]] = {}
|
||||
_suspension_lock: ClassVar[Lock] = Lock()
|
||||
_sw_member_cache: ClassVar[dict[str, Any]] = {}
|
||||
_sw_member_lock: ClassVar[Lock] = Lock()
|
||||
|
||||
def __init__(self, first: Any, second: Any | None = None) -> None:
|
||||
# Production: DatahubAwareTushareClient(bridge)
|
||||
# Older tests: DatahubAwareTushareClient(unused_legacy, bridge)
|
||||
self._bridge = second if second is not None else first
|
||||
self.token = "datahub"
|
||||
self.timeout = 30
|
||||
self.realtime_aggregator = None
|
||||
|
||||
def query(
|
||||
self,
|
||||
@@ -245,7 +584,22 @@ class DatahubAwareTushareClient:
|
||||
params: dict[str, Any] | None = None,
|
||||
fields: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
return self._bridge.query(api_name, params, fields, self._legacy.query)
|
||||
return self._bridge.query(api_name, params, fields)
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(self._legacy, name)
|
||||
def try_market_quotes(self, trade_date: str = "") -> list[dict[str, Any]] | None:
|
||||
return self._bridge.try_market_quotes(trade_date)
|
||||
|
||||
def try_quotes(self, codes: list[str]) -> list[dict[str, Any]] | None:
|
||||
return self._bridge.try_quotes(codes)
|
||||
|
||||
def try_index_quotes(self) -> list[dict[str, Any]] | None:
|
||||
return self._bridge.try_index_quotes()
|
||||
|
||||
def try_sector_quote(self, code: str, trade_date: str = "") -> dict[str, Any] | None:
|
||||
return self._bridge.try_sector_quote(code, trade_date)
|
||||
|
||||
def try_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]] | None:
|
||||
return self._bridge.try_limit_pool(trade_date)
|
||||
|
||||
def record_datahub_legacy(self, dataset: str, source: str = "", error: str = "") -> None:
|
||||
self._bridge.record_legacy(dataset, source, error)
|
||||
|
||||
@@ -60,6 +60,27 @@ class DatahubClient:
|
||||
def auction(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/auction", params)
|
||||
|
||||
def limit_events(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/limit-events", params)
|
||||
|
||||
def popularity(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/popularity", params)
|
||||
|
||||
def dragon_tiger(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/dragon-tiger", params)
|
||||
|
||||
def sectors(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/sectors", params)
|
||||
|
||||
def quotes_latest(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/quotes/latest", params)
|
||||
|
||||
def index_quotes(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/indexes/quotes", params)
|
||||
|
||||
def intraday_points(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/intraday/points", params)
|
||||
|
||||
def dataset_status(self, date: str) -> DatahubResponse:
|
||||
return self.get("/v1/datasets/status", {"date": date})
|
||||
|
||||
@@ -69,6 +90,24 @@ class DatahubClient:
|
||||
params["dataset"] = dataset
|
||||
return self.get("/v1/batches", params)
|
||||
|
||||
def query_api(self, api_name: str, params: dict[str, Any] | None = None, fields: str = "") -> DatahubResponse:
|
||||
return self.post(
|
||||
"/v1/query",
|
||||
{"api_name": api_name, "params": params or {}, "fields": fields},
|
||||
)
|
||||
|
||||
def sector_quote(self, code: str, date: str = "") -> DatahubResponse:
|
||||
payload: dict[str, Any] = {"code": code}
|
||||
if date:
|
||||
payload["date"] = date
|
||||
return self.get("/v1/sectors/quote", payload)
|
||||
|
||||
def limit_pool(self, trade_date: str = "") -> DatahubResponse:
|
||||
params: dict[str, Any] = {}
|
||||
if trade_date:
|
||||
params["date"] = trade_date
|
||||
return self.get("/v1/limit-pool", params)
|
||||
|
||||
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
|
||||
if not self.settings.token:
|
||||
raise DatahubError("NOT_CONFIGURED", "DATAHUB_TOKEN is not configured")
|
||||
@@ -97,15 +136,41 @@ class DatahubClient:
|
||||
)
|
||||
raise last_error or DatahubError("INTERNAL", "datahub request failed")
|
||||
|
||||
def _request(self, url: str) -> DatahubResponse:
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
def post(self, path: str, body: dict[str, Any] | None = None) -> DatahubResponse:
|
||||
if not self.settings.token:
|
||||
raise DatahubError("NOT_CONFIGURED", "DATAHUB_TOKEN is not configured")
|
||||
url = self.settings.base_url + path
|
||||
attempts = 1 + max(0, self.settings.retries)
|
||||
last_error: DatahubError | None = None
|
||||
payload = json.dumps(body or {}, ensure_ascii=False).encode("utf-8")
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
return self._request(url, method="POST", data=payload)
|
||||
except DatahubError as exc:
|
||||
last_error = exc
|
||||
if exc.code not in {"TIMEOUT", "UNAVAILABLE"} or attempt + 1 >= attempts:
|
||||
raise
|
||||
LOGGER.warning(
|
||||
"datahub retry %s/%s %s",
|
||||
attempt + 1,
|
||||
attempts,
|
||||
redact_text(str(exc), self.settings.secrets()),
|
||||
)
|
||||
raise last_error or DatahubError("INTERNAL", "datahub request failed")
|
||||
|
||||
def _request(self, url: str, method: str = "GET", data: bytes | None = None) -> DatahubResponse:
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"X-Datahub-Token": self.settings.token,
|
||||
"User-Agent": "XiaobaiReviewDatahub/1.0",
|
||||
},
|
||||
method="GET",
|
||||
}
|
||||
if data is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
headers=headers,
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with self._urlopen(request, timeout=self.settings.timeout_seconds) as response:
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import Any
|
||||
from backend.data.datahub.native import SCALE_FIELDS, row_key, to_canonical_row, yyyymmdd
|
||||
|
||||
NUMERIC_TOLERANCE = 1e-4
|
||||
CANONICAL_ALIASES = {"volume": "vol"}
|
||||
|
||||
|
||||
def compare_rows(
|
||||
@@ -13,8 +14,10 @@ def compare_rows(
|
||||
hub_rows: list[dict[str, Any]] | None,
|
||||
hub_meta: dict[str, Any] | None = None,
|
||||
hub_error: str | None = None,
|
||||
fields: str = "",
|
||||
) -> dict[str, Any]:
|
||||
hub = hub_rows or []
|
||||
requested = _requested_fields(fields)
|
||||
legacy_map = {row_key(dataset, row): row for row in legacy_rows}
|
||||
hub_map = {row_key(dataset, _align_hub_row(row)): row for row in hub}
|
||||
missing_hub = sorted(key for key in legacy_map if key not in hub_map)
|
||||
@@ -26,7 +29,7 @@ def compare_rows(
|
||||
hub_row = hub_map.get(key)
|
||||
if hub_row is None:
|
||||
continue
|
||||
field_report = _compare_fields(dataset, legacy, hub_row)
|
||||
field_report = _compare_fields(dataset, legacy, hub_row, requested)
|
||||
if field_report["unit_conversion"]:
|
||||
unit_conversion.append({"key": list(key), "fields": field_report["unit_conversion"]})
|
||||
if field_report["value_diff"]:
|
||||
@@ -53,6 +56,7 @@ def compare_rows(
|
||||
"published_at": (hub_meta or {}).get("published_at"),
|
||||
"trade_date": yyyymmdd((hub_meta or {}).get("trade_date")),
|
||||
"hub_error": hub_error,
|
||||
"fields_compared": sorted(requested) if requested is not None else None,
|
||||
"equal": (
|
||||
not hub_error
|
||||
and not missing_hub
|
||||
@@ -71,13 +75,36 @@ def _align_hub_row(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return aligned
|
||||
|
||||
|
||||
def _compare_fields(dataset: str, legacy: dict[str, Any], hub: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
|
||||
def _requested_fields(fields: str) -> list[str] | None:
|
||||
"""Fields the website actually asked for; None means "no projection"."""
|
||||
keys = [item.strip() for item in str(fields or "").split(",") if item.strip()]
|
||||
if not keys:
|
||||
return None
|
||||
seen: list[str] = []
|
||||
for key in keys:
|
||||
canonical = CANONICAL_ALIASES.get(key, key)
|
||||
if canonical not in seen:
|
||||
seen.append(canonical)
|
||||
return seen
|
||||
|
||||
|
||||
def _compare_fields(
|
||||
dataset: str,
|
||||
legacy: dict[str, Any],
|
||||
hub: dict[str, Any],
|
||||
requested: list[str] | None = None,
|
||||
) -> dict[str, list[dict[str, Any]]]:
|
||||
canonical_legacy = to_canonical_row(dataset, legacy)
|
||||
hub_canonical = _hub_canonical(dataset, hub)
|
||||
native_hub = _align_hub_row(hub)
|
||||
value_diff: list[dict[str, Any]] = []
|
||||
unit_conversion: list[dict[str, Any]] = []
|
||||
keys = (set(canonical_legacy) | set(hub_canonical)) - {"batch_id", "updated_at", "volume"}
|
||||
if requested is not None:
|
||||
# Compare only what the website asked for. Extra hub columns are
|
||||
# transport detail, not business differences; a requested field still
|
||||
# alarms when it is missing or holds a different value.
|
||||
keys = set(requested) - {"batch_id", "updated_at", "volume"}
|
||||
scales = SCALE_FIELDS.get(dataset) or {}
|
||||
for field in sorted(keys):
|
||||
left = canonical_legacy.get(field)
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from backend.data.datahub.bridge import DatahubBridge
|
||||
from backend.data.datahub.errors import DatahubError
|
||||
from backend.data.providers.ifind_client import IfindError
|
||||
|
||||
class HubIfindProxy:
|
||||
"""Website-facing iFinD facade. Talks only to xiaobai-datahub."""
|
||||
|
||||
def __init__(self, datahub: DatahubBridge) -> None:
|
||||
self._datahub = datahub
|
||||
self._status: dict[str, Any] | None = None
|
||||
self._status_at = 0.0
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self.status().get("configured"))
|
||||
|
||||
def status(self) -> dict[str, Any]:
|
||||
now = time.monotonic()
|
||||
if self._status is not None and now - self._status_at < 30:
|
||||
return dict(self._status)
|
||||
fallback = {"configured": False, "access_ready": False, "access_expires_at": ""}
|
||||
if not self._datahub.settings.token:
|
||||
self._status = fallback
|
||||
self._status_at = now
|
||||
return dict(fallback)
|
||||
try:
|
||||
rows = self._rows("ifind_status", {})
|
||||
except IfindError:
|
||||
self._status = fallback
|
||||
self._status_at = now
|
||||
return dict(fallback)
|
||||
row = rows[0] if rows else {}
|
||||
status = {
|
||||
"configured": bool(row.get("configured")),
|
||||
"access_ready": bool(row.get("access_ready")),
|
||||
"access_expires_at": str(row.get("access_expires_at") or ""),
|
||||
}
|
||||
self._status = status
|
||||
self._status_at = now
|
||||
return dict(status)
|
||||
|
||||
def wencai(self, query: str, search_type: str = "stock", cache_ttl: int = 300) -> list[dict[str, Any]]:
|
||||
return self._rows(
|
||||
"ifind_wencai",
|
||||
{"query": query, "search_type": search_type, "cache_ttl": cache_ttl},
|
||||
)
|
||||
|
||||
def snapshots(
|
||||
self,
|
||||
codes: str | list[str],
|
||||
indicators: list[str],
|
||||
start_time: str,
|
||||
end_time: str,
|
||||
cache_ttl: int = 8,
|
||||
) -> list[dict[str, Any]]:
|
||||
return self._rows(
|
||||
"ifind_snapshots",
|
||||
{
|
||||
"codes": codes,
|
||||
"indicators": indicators,
|
||||
"start_time": start_time,
|
||||
"end_time": end_time,
|
||||
"cache_ttl": cache_ttl,
|
||||
},
|
||||
)
|
||||
|
||||
def history(
|
||||
self,
|
||||
codes: str | list[str],
|
||||
indicators: list[str],
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
cache_ttl: int = 300,
|
||||
) -> list[dict[str, Any]]:
|
||||
return self._rows(
|
||||
"ifind_history",
|
||||
{
|
||||
"codes": codes,
|
||||
"indicators": indicators,
|
||||
"start_date": start_date,
|
||||
"end_date": end_date,
|
||||
"cache_ttl": cache_ttl,
|
||||
},
|
||||
)
|
||||
|
||||
def real_time(
|
||||
self,
|
||||
codes: str | list[str],
|
||||
indicators: list[str],
|
||||
cache_ttl: int = 10,
|
||||
) -> list[dict[str, Any]]:
|
||||
return self._rows(
|
||||
"ifind_realtime",
|
||||
{"codes": codes, "indicators": indicators, "cache_ttl": cache_ttl},
|
||||
)
|
||||
|
||||
def intraday(
|
||||
self,
|
||||
code: str,
|
||||
start_time: str,
|
||||
end_time: str,
|
||||
cache_ttl: int = 20,
|
||||
) -> list[dict[str, Any]]:
|
||||
return self._rows(
|
||||
"ifind_intraday",
|
||||
{
|
||||
"code": code,
|
||||
"start_time": start_time,
|
||||
"end_time": end_time,
|
||||
"cache_ttl": cache_ttl,
|
||||
},
|
||||
)
|
||||
|
||||
def test_connection(self) -> dict[str, Any]:
|
||||
payload = self.real_time(
|
||||
"000001.SH",
|
||||
["open", "high", "low", "latest", "preClose"],
|
||||
cache_ttl=0,
|
||||
)
|
||||
return {
|
||||
"ok": bool(payload),
|
||||
"sample_time": str(payload[0].get("time") or "") if payload else "",
|
||||
}
|
||||
|
||||
def _rows(self, api_name: str, params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
try:
|
||||
response = self._datahub.client.query_api(api_name, params)
|
||||
except DatahubError as exc:
|
||||
raise IfindError(str(exc) or "iFinD 数据中枢暂不可用") from exc
|
||||
data = response.data
|
||||
if isinstance(data, list):
|
||||
return [dict(item) for item in data if isinstance(item, dict)]
|
||||
if isinstance(data, dict):
|
||||
return [dict(data)]
|
||||
return []
|
||||
@@ -17,6 +17,13 @@ API_TO_DATASET = {
|
||||
"index_daily": "index_daily",
|
||||
"moneyflow": "moneyflow",
|
||||
"stk_auction": "auction",
|
||||
"limit_list_d": "limit_events",
|
||||
"ths_hot": "popularity",
|
||||
"dc_hot": "popularity",
|
||||
"hm_detail": "dragon_tiger",
|
||||
"ths_daily": "sector_daily",
|
||||
"dc_index": "sector_daily",
|
||||
"sw_daily": "sector_daily",
|
||||
}
|
||||
|
||||
SCALE_FIELDS = {
|
||||
@@ -35,6 +42,16 @@ SCALE_FIELDS = {
|
||||
"net_mf_amount": AMOUNT_WAN_YUAN,
|
||||
},
|
||||
"auction": {"vol": VOLUME_LOT, "float_share": AMOUNT_WAN_YUAN},
|
||||
"limit_events": {
|
||||
"limit_amount": AMOUNT_WAN_YUAN,
|
||||
"float_mv": AMOUNT_WAN_YUAN,
|
||||
"total_mv": AMOUNT_WAN_YUAN,
|
||||
},
|
||||
"dragon_tiger": {
|
||||
"buy_amount": AMOUNT_WAN_YUAN,
|
||||
"sell_amount": AMOUNT_WAN_YUAN,
|
||||
"net_amount": AMOUNT_WAN_YUAN,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -67,6 +84,16 @@ def to_native_row(dataset: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
converted[field] = _unscale(converted.get(field), factor)
|
||||
if dataset == "stocks":
|
||||
converted.pop("updated_at", None)
|
||||
if dataset == "popularity":
|
||||
# keep hub source; callers filter ths/dc themselves when needed
|
||||
if converted.get("ts_name") and not converted.get("name"):
|
||||
converted["name"] = converted.get("ts_name")
|
||||
if dataset == "dragon_tiger":
|
||||
if converted.get("ts_name") and not converted.get("name"):
|
||||
converted["name"] = converted.get("ts_name")
|
||||
if dataset == "sector_daily":
|
||||
if converted.get("pct_change") is not None and converted.get("pct_chg") is None:
|
||||
converted["pct_chg"] = converted.get("pct_change")
|
||||
return converted
|
||||
|
||||
|
||||
@@ -96,6 +123,30 @@ def row_key(dataset: str, row: dict[str, Any]) -> tuple[str, ...]:
|
||||
return (str(row.get("ts_code") or "").upper(),)
|
||||
if dataset == "status":
|
||||
return (str(row.get("dataset") or ""), yyyymmdd(row.get("trade_date")))
|
||||
if dataset == "limit_events":
|
||||
return (
|
||||
str(row.get("ts_code") or "").upper(),
|
||||
yyyymmdd(row.get("trade_date")),
|
||||
str(row.get("limit_type") or ""),
|
||||
)
|
||||
if dataset == "popularity":
|
||||
return (
|
||||
str(row.get("ts_code") or "").upper(),
|
||||
yyyymmdd(row.get("trade_date")),
|
||||
str(row.get("source") or ""),
|
||||
)
|
||||
if dataset == "dragon_tiger":
|
||||
return (
|
||||
str(row.get("ts_code") or "").upper(),
|
||||
yyyymmdd(row.get("trade_date")),
|
||||
str(row.get("hm_name") or ""),
|
||||
)
|
||||
if dataset == "sector_daily":
|
||||
return (
|
||||
str(row.get("ts_code") or "").upper(),
|
||||
yyyymmdd(row.get("trade_date")),
|
||||
str(row.get("family") or ""),
|
||||
)
|
||||
return (str(row.get("ts_code") or "").upper(), yyyymmdd(row.get("trade_date")))
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from backend.data.datahub.bridge import DatahubBridge
|
||||
from backend.data.realtime import RealtimeAggregateError
|
||||
|
||||
|
||||
class HubRealtimeProxy:
|
||||
"""Realtime observation facade. Talks only to xiaobai-datahub."""
|
||||
|
||||
def __init__(self, datahub: DatahubBridge) -> None:
|
||||
self._datahub = datahub
|
||||
|
||||
def health_snapshot(self, sector: str = "") -> dict[str, Any]:
|
||||
started = datetime.now().astimezone()
|
||||
indices: list[dict[str, Any]] = []
|
||||
error = ""
|
||||
try:
|
||||
indices = self.tencent_indices()
|
||||
except RealtimeAggregateError as exc:
|
||||
error = str(exc)
|
||||
epochs = [int(item.get("quote_time_epoch") or 0) for item in indices]
|
||||
max_skew = 120
|
||||
index_consistent = bool(epochs) and max(epochs) - min(epochs) <= max_skew
|
||||
ready = len(indices) == 3 and index_consistent
|
||||
return {
|
||||
"ready": ready,
|
||||
"isolated": True,
|
||||
"generated_at": started.isoformat(timespec="seconds"),
|
||||
"elapsed_ms": 0,
|
||||
"indices": indices,
|
||||
"index_consistent": index_consistent,
|
||||
"sector": None,
|
||||
"sources": {
|
||||
"datahub_indices": {
|
||||
"ok": ready,
|
||||
"error": error,
|
||||
"source": "datahub",
|
||||
}
|
||||
},
|
||||
"observations": {},
|
||||
"policy": {
|
||||
"integration": "datahub_exclusive",
|
||||
"max_index_time_skew_seconds": max_skew,
|
||||
"notice": "实时观察只走数据中枢,主网站不再直连东财/腾讯。",
|
||||
},
|
||||
}
|
||||
|
||||
def tencent_indices(self) -> list[dict[str, Any]]:
|
||||
rows = self._datahub.try_index_quotes() or []
|
||||
result = [_as_index(item) for item in rows if _as_index(item)]
|
||||
wanted = {"000001", "399001", "399006"}
|
||||
result = [item for item in result if item.get("code") in wanted]
|
||||
result.sort(key=lambda item: str(item.get("code") or ""))
|
||||
if len(result) != 3:
|
||||
raise RealtimeAggregateError(f"datahub returned {len(result)}/3 indices")
|
||||
return result
|
||||
|
||||
def eastmoney_indices(self) -> list[dict[str, Any]]:
|
||||
return self.tencent_indices()
|
||||
|
||||
def tencent_stock_quote(self, code: str, expected_date: str = "") -> dict[str, Any]:
|
||||
return self._stock_quote(code, expected_date)
|
||||
|
||||
def eastmoney_stock_quote(self, code: str, expected_date: str = "") -> dict[str, Any]:
|
||||
return self._stock_quote(code, expected_date)
|
||||
|
||||
def tencent_stock_quotes(
|
||||
self,
|
||||
codes: list[str],
|
||||
expected_date: str = "",
|
||||
minimum: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
return self._stock_quotes(codes, expected_date, minimum)
|
||||
|
||||
def eastmoney_stock_quotes(
|
||||
self,
|
||||
codes: list[str],
|
||||
expected_date: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
return self._stock_quotes(codes, expected_date, None)
|
||||
|
||||
def eastmoney_shenwan_quote(self, ts_code: str, expected_date: str = "") -> dict[str, Any]:
|
||||
quote = self._datahub.try_sector_quote(ts_code, expected_date)
|
||||
if not quote:
|
||||
raise RealtimeAggregateError(f"datahub shenwan quote unavailable for {ts_code}")
|
||||
return quote
|
||||
|
||||
def _stock_quote(self, code: str, expected_date: str) -> dict[str, Any]:
|
||||
rows = self._stock_quotes([code], expected_date, 1)
|
||||
if not rows:
|
||||
raise RealtimeAggregateError(f"datahub stock quote unavailable for {code}")
|
||||
return rows[0]
|
||||
|
||||
def _stock_quotes(
|
||||
self,
|
||||
codes: list[str],
|
||||
expected_date: str,
|
||||
minimum: int | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
cleaned = [str(item or "").strip() for item in codes if str(item or "").strip()]
|
||||
rows = self._datahub.try_quotes(cleaned) if cleaned else (self._datahub.try_market_quotes(expected_date) or [])
|
||||
quotes = [_as_stock(item) for item in (rows or []) if _as_stock(item)]
|
||||
if expected_date:
|
||||
compact = str(expected_date).replace("-", "")
|
||||
quotes = [
|
||||
item
|
||||
for item in quotes
|
||||
if not item.get("quote_date") or str(item.get("quote_date") or "").replace("-", "") == compact
|
||||
]
|
||||
if minimum is not None and len(quotes) < minimum:
|
||||
raise RealtimeAggregateError(f"datahub returned {len(quotes)} quotes, need {minimum}")
|
||||
return quotes
|
||||
|
||||
|
||||
def _as_index(row: dict[str, Any]) -> dict[str, Any] | None:
|
||||
code = str(row.get("code") or str(row.get("ts_code") or "").split(".")[0] or "")
|
||||
price = _number(row.get("price") if row.get("price") not in (None, "") else row.get("close"))
|
||||
if not code or price <= 0:
|
||||
return None
|
||||
epoch = int(_number(row.get("quote_time_epoch")))
|
||||
amount = _number(row.get("amount_billion"))
|
||||
if amount <= 0:
|
||||
amount = round(_number(row.get("amount")) / 100_000_000, 2)
|
||||
return {
|
||||
"code": code,
|
||||
"name": row.get("name") or code,
|
||||
"price": price,
|
||||
"change": _number(row.get("change") if row.get("change") not in (None, "") else row.get("pct_chg")),
|
||||
"change_amount": _number(row.get("change_amount")),
|
||||
"open": _number(row.get("open")),
|
||||
"high": _number(row.get("high")),
|
||||
"low": _number(row.get("low")),
|
||||
"previous_close": _number(
|
||||
row.get("previous_close") if row.get("previous_close") not in (None, "") else row.get("pre_close")
|
||||
),
|
||||
"amount_billion": amount,
|
||||
"quote_time_epoch": epoch,
|
||||
"quote_time": str(row.get("quote_time") or ""),
|
||||
"source": str(row.get("source") or "datahub"),
|
||||
"cache_age_seconds": 0,
|
||||
}
|
||||
|
||||
|
||||
def _as_stock(row: dict[str, Any]) -> dict[str, Any] | None:
|
||||
close = _number(row.get("close") if row.get("close") not in (None, "") else row.get("price"))
|
||||
if close <= 0:
|
||||
return None
|
||||
ts_code = str(row.get("ts_code") or "")
|
||||
code = str(row.get("code") or ts_code.split(".")[0] or "")
|
||||
return {
|
||||
"ts_code": ts_code or code,
|
||||
"code": code,
|
||||
"name": row.get("name") or "",
|
||||
"close": close,
|
||||
"pre_close": _number(
|
||||
row.get("pre_close") if row.get("pre_close") not in (None, "") else row.get("previous_close")
|
||||
),
|
||||
"open": _number(row.get("open")),
|
||||
"high": _number(row.get("high")),
|
||||
"low": _number(row.get("low")),
|
||||
"volume": _number(row.get("volume") if row.get("volume") not in (None, "") else row.get("vol")),
|
||||
"vol": _number(row.get("vol") if row.get("vol") not in (None, "") else row.get("volume")),
|
||||
"amount": _number(row.get("amount")),
|
||||
"quote_time_epoch": int(_number(row.get("quote_time_epoch"))),
|
||||
"quote_time": str(row.get("quote_time") or ""),
|
||||
"quote_date": str(row.get("quote_date") or ""),
|
||||
"source": str(row.get("source") or "datahub"),
|
||||
"delayed": bool(row.get("delayed")),
|
||||
}
|
||||
|
||||
|
||||
def _number(value: Any) -> float:
|
||||
try:
|
||||
return float(value or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
@@ -0,0 +1,57 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from threading import Lock
|
||||
from typing import Any
|
||||
|
||||
from backend.data.datahub.settings import DATASETS
|
||||
|
||||
DATASET_LABELS = {
|
||||
"calendar": "交易日历",
|
||||
"stocks": "股票主档",
|
||||
"daily": "个股日K",
|
||||
"index_daily": "指数日K",
|
||||
"valuation": "估值",
|
||||
"moneyflow": "资金流",
|
||||
"auction": "竞价",
|
||||
"limit_events": "涨停池",
|
||||
"popularity": "人气榜",
|
||||
"dragon_tiger": "龙虎榜",
|
||||
"sector_daily": "题材板块",
|
||||
"quotes": "全市场实时行情",
|
||||
"index_quotes": "指数实时行情",
|
||||
"intraday": "分时",
|
||||
"status": "数据集状态",
|
||||
}
|
||||
|
||||
|
||||
class DatahubRouteLedger:
|
||||
def __init__(self) -> None:
|
||||
self._lock = Lock()
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def record(self, dataset: str, route: str, source: str = "", error: str = "") -> None:
|
||||
name = str(dataset or "").strip() or "unknown"
|
||||
with self._lock:
|
||||
self._rows[name] = {
|
||||
"dataset": name,
|
||||
"label": DATASET_LABELS.get(name, name),
|
||||
"route": "legacy" if route == "legacy" else "datahub",
|
||||
"source": str(source or "").strip(),
|
||||
"error": str(error or "").strip(),
|
||||
"at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
}
|
||||
|
||||
def snapshot(self) -> list[dict[str, Any]]:
|
||||
with self._lock:
|
||||
rows = [dict(item) for item in self._rows.values()]
|
||||
order = {name: index for index, name in enumerate(DATASETS)}
|
||||
rows.sort(key=lambda item: (order.get(str(item.get("dataset")), 99), str(item.get("dataset"))))
|
||||
return rows
|
||||
|
||||
def clear(self) -> None:
|
||||
with self._lock:
|
||||
self._rows.clear()
|
||||
|
||||
|
||||
LEDGER = DatahubRouteLedger()
|
||||
@@ -17,6 +17,13 @@ DATASETS = (
|
||||
"valuation",
|
||||
"moneyflow",
|
||||
"auction",
|
||||
"limit_events",
|
||||
"popularity",
|
||||
"dragon_tiger",
|
||||
"sector_daily",
|
||||
"quotes",
|
||||
"index_quotes",
|
||||
"intraday",
|
||||
"status",
|
||||
)
|
||||
|
||||
@@ -28,6 +35,13 @@ ENV_DATASET = {
|
||||
"valuation": "VALUATION",
|
||||
"moneyflow": "MONEYFLOW",
|
||||
"auction": "AUCTION",
|
||||
"limit_events": "LIMIT_EVENTS",
|
||||
"popularity": "POPULARITY",
|
||||
"dragon_tiger": "DRAGON_TIGER",
|
||||
"sector_daily": "SECTOR_DAILY",
|
||||
"quotes": "QUOTES",
|
||||
"index_quotes": "INDEX_QUOTES",
|
||||
"intraday": "INTRADAY",
|
||||
"status": "STATUS",
|
||||
}
|
||||
|
||||
|
||||
+38
-23
@@ -1,43 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from backend.data.contracts import DataUsage
|
||||
from backend.data.datahub import DatahubAwareTushareClient, DatahubBridge, DatahubClient, DatahubSettings
|
||||
from backend.data.datahub.ifind_proxy import HubIfindProxy
|
||||
from backend.data.datahub.realtime_proxy import HubRealtimeProxy
|
||||
from backend.data.policy import DataSourcePolicy
|
||||
from backend.data.providers import IfindProvider, TushareProvider
|
||||
from backend.data.providers import IfindProvider
|
||||
from backend.data.quality import DataQualityGate, QualityEvidence, QualityReport
|
||||
from backend.data.providers.ifind_client import IfindHttpClient
|
||||
from backend.data.providers.tushare_client import TushareClient
|
||||
from backend.data.realtime import WebRealtimeAggregator
|
||||
from backend.features.market.charts import EastmoneyChartClient, MarketChartClient
|
||||
from backend.features.market.charts import MarketChartClient
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DataGateway:
|
||||
policy: DataSourcePolicy
|
||||
quality: DataQualityGate
|
||||
tushare_provider: TushareProvider
|
||||
ifind_provider: IfindProvider
|
||||
chart_data: MarketChartClient
|
||||
realtime_observer: WebRealtimeAggregator
|
||||
realtime_observer: HubRealtimeProxy
|
||||
datahub: DatahubBridge
|
||||
|
||||
@property
|
||||
def ifind(self) -> IfindHttpClient:
|
||||
def ifind(self) -> HubIfindProxy:
|
||||
return self.ifind_provider.client
|
||||
|
||||
def tushare(
|
||||
self,
|
||||
dataset_id: str = "",
|
||||
usage: DataUsage = "calculation",
|
||||
) -> TushareClient:
|
||||
) -> DatahubAwareTushareClient:
|
||||
if dataset_id:
|
||||
self.policy.assert_allowed(dataset_id, "tushare", usage)
|
||||
return DatahubAwareTushareClient(self.tushare_provider.client(), self.datahub)
|
||||
return DatahubAwareTushareClient(self.datahub)
|
||||
|
||||
def dataset_status(self, trade_date: str) -> list[dict[str, Any]] | None:
|
||||
return self.datahub.dataset_status(trade_date)
|
||||
@@ -45,6 +42,31 @@ class DataGateway:
|
||||
def batches(self, trade_date: str, dataset: str = "") -> list[dict[str, Any]] | None:
|
||||
return self.datahub.batches(trade_date, dataset)
|
||||
|
||||
def datahub_status(self) -> dict[str, Any]:
|
||||
from backend.data.datahub.route_state import DATASET_LABELS, LEDGER
|
||||
from backend.data.datahub.settings import DATASETS
|
||||
|
||||
settings = self.datahub.settings
|
||||
flags = []
|
||||
enabled = 0
|
||||
for name in DATASETS:
|
||||
read = bool(settings.flags(name).read)
|
||||
if read:
|
||||
enabled += 1
|
||||
flags.append({"dataset": name, "label": DATASET_LABELS.get(name, name), "read": read})
|
||||
routes = LEDGER.snapshot()
|
||||
fallbacks = [item for item in routes if item.get("route") == "legacy"]
|
||||
return {
|
||||
"configured": bool(settings.token and settings.base_url),
|
||||
"base_url": settings.base_url,
|
||||
"enabled_reads": enabled,
|
||||
"total_reads": len(DATASETS),
|
||||
"flags": flags,
|
||||
"routes": routes,
|
||||
"fallback_count": len(fallbacks),
|
||||
"fallback_labels": [str(item.get("label") or item.get("dataset")) for item in fallbacks],
|
||||
}
|
||||
|
||||
def assert_source(self, dataset_id: str, provider_id: str, usage: DataUsage) -> None:
|
||||
self.policy.assert_allowed(dataset_id, provider_id, usage)
|
||||
|
||||
@@ -72,25 +94,18 @@ class DataGateway:
|
||||
|
||||
def build_data_gateway(
|
||||
credentials: dict[str, object],
|
||||
tushare_token_supplier: Callable[[], str] | None = None,
|
||||
datahub_settings: DatahubSettings | None = None,
|
||||
) -> DataGateway:
|
||||
ifind = IfindHttpClient(
|
||||
str(credentials.get("ifind_refresh_token") or ""),
|
||||
str(credentials.get("ifind_access_token") or ""),
|
||||
)
|
||||
token_supplier = tushare_token_supplier or (
|
||||
lambda: str(credentials.get("tushare_token") or "")
|
||||
)
|
||||
policy = DataSourcePolicy.load()
|
||||
settings = datahub_settings or DatahubSettings.load(credentials=credentials)
|
||||
datahub_client = DatahubClient(settings)
|
||||
datahub = DatahubBridge(settings, datahub_client)
|
||||
ifind = HubIfindProxy(datahub)
|
||||
return DataGateway(
|
||||
policy=policy,
|
||||
quality=DataQualityGate.load(policy),
|
||||
tushare_provider=TushareProvider(token_supplier),
|
||||
ifind_provider=IfindProvider(ifind),
|
||||
chart_data=MarketChartClient(ifind, EastmoneyChartClient()),
|
||||
realtime_observer=WebRealtimeAggregator(),
|
||||
datahub=DatahubBridge(settings, datahub_client),
|
||||
chart_data=MarketChartClient(datahub),
|
||||
realtime_observer=HubRealtimeProxy(datahub),
|
||||
datahub=datahub,
|
||||
)
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from backend.data.providers.ifind_client import IfindHttpClient
|
||||
from typing import Any
|
||||
|
||||
|
||||
class IfindProvider:
|
||||
def __init__(self, client: IfindHttpClient) -> None:
|
||||
def __init__(self, client: Any) -> None:
|
||||
self.client = client
|
||||
|
||||
def set_credentials(self, refresh_token: str, access_token: str = "") -> None:
|
||||
self.client.set_credentials(refresh_token, access_token)
|
||||
setter = getattr(self.client, "set_credentials", None)
|
||||
if callable(setter):
|
||||
setter(refresh_token, access_token)
|
||||
|
||||
@@ -66,3 +66,5 @@ class TushareClient(
|
||||
_stock_listing_lock: ClassVar[Lock] = Lock()
|
||||
_suspension_cache: ClassVar[dict[str, dict[str, str] | None]] = {}
|
||||
_suspension_lock: ClassVar[Lock] = Lock()
|
||||
_sw_member_cache: ClassVar[dict[str, Any]] = {}
|
||||
_sw_member_lock: ClassVar[Lock] = Lock()
|
||||
|
||||
@@ -3,7 +3,12 @@ from __future__ import annotations
|
||||
from typing import Any
|
||||
|
||||
from backend.data.numbers import finite_number as _number
|
||||
from backend.data.providers.tushare_helpers import _display_time, _prices_equal
|
||||
from backend.data.providers.tushare_helpers import (
|
||||
_display_time,
|
||||
_optional_number,
|
||||
_prices_equal,
|
||||
calendar_is_open,
|
||||
)
|
||||
|
||||
|
||||
class DailyMarketMixin:
|
||||
@@ -17,7 +22,11 @@ class DailyMarketMixin:
|
||||
trade_date = requested
|
||||
else:
|
||||
row = requested_rows[0]
|
||||
trade_date = row["cal_date"] if row.get("is_open") == 1 else row.get("pretrade_date", requested)
|
||||
trade_date = (
|
||||
row["cal_date"]
|
||||
if calendar_is_open(row.get("is_open"))
|
||||
else row.get("pretrade_date", requested)
|
||||
)
|
||||
|
||||
resolved_rows = self.query(
|
||||
"trade_cal",
|
||||
@@ -129,7 +138,75 @@ class DailyMarketMixin:
|
||||
)
|
||||
item["capital_trade_date"] = str(capital.get("trade_date") or "")
|
||||
result.append(item)
|
||||
return result
|
||||
return self._overlay_board_fields(result, trade_date)
|
||||
|
||||
def _overlay_board_fields(
|
||||
self,
|
||||
rows: list[dict[str, Any]],
|
||||
trade_date: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
if not rows:
|
||||
return rows
|
||||
official = self._official_board_map(trade_date)
|
||||
free = self._free_board_map(trade_date) if not official else {}
|
||||
merged: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
code = str(row.get("ts_code") or "")
|
||||
extra = official.get(code) or free.get(code) or {}
|
||||
if not extra:
|
||||
merged.append(row)
|
||||
continue
|
||||
item = dict(row)
|
||||
for key in (
|
||||
"first_time",
|
||||
"last_time",
|
||||
"fd_amount",
|
||||
"open_times",
|
||||
"limit_times",
|
||||
"turnover_ratio",
|
||||
):
|
||||
incoming = extra.get(key)
|
||||
current = item.get(key)
|
||||
if incoming in (None, "", "--"):
|
||||
continue
|
||||
# The daily fallback and the board pool can arrive at different
|
||||
# times. Keep the stronger streak instead of freezing the
|
||||
# provisional default at one or letting a stale pool lower it.
|
||||
if key == "limit_times":
|
||||
item[key] = max(
|
||||
1,
|
||||
int(_number(current, 1)),
|
||||
int(_number(incoming, 1)),
|
||||
)
|
||||
elif current in (None, "", "--", 0, 0.0):
|
||||
item[key] = incoming
|
||||
merged.append(item)
|
||||
return merged
|
||||
|
||||
def _official_board_map(self, trade_date: str) -> dict[str, dict[str, Any]]:
|
||||
mapped: dict[str, dict[str, Any]] = {}
|
||||
try:
|
||||
for row in self._load_limit_lists(trade_date):
|
||||
code = str(row.get("ts_code") or "")
|
||||
if code:
|
||||
mapped[code] = row
|
||||
except Exception:
|
||||
return {}
|
||||
return mapped
|
||||
|
||||
def _free_board_map(self, trade_date: str) -> dict[str, dict[str, Any]]:
|
||||
loader = getattr(self, "try_limit_pool", None)
|
||||
if not callable(loader):
|
||||
return {}
|
||||
try:
|
||||
rows = loader(trade_date) or []
|
||||
except Exception:
|
||||
return {}
|
||||
return {
|
||||
str(row.get("ts_code") or ""): row
|
||||
for row in rows
|
||||
if row.get("ts_code")
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_limit(row: dict[str, Any], status: str) -> dict[str, Any]:
|
||||
@@ -154,7 +231,11 @@ class DailyMarketMixin:
|
||||
"turnover_source": row.get("turnover_source") or "provider",
|
||||
"capital_trade_date": row.get("capital_trade_date") or "",
|
||||
"amount_billion": round(amount_billion, 2),
|
||||
"seal_amount_million": round(_number(row.get("fd_amount")) / 10000, 0),
|
||||
"seal_amount_million": (
|
||||
round(fd / 10000, 0)
|
||||
if (fd := _optional_number(row.get("fd_amount"))) is not None
|
||||
else None
|
||||
),
|
||||
"float_mv_billion": round(_number(row.get("float_mv")) / 100000000, 1),
|
||||
"status": status,
|
||||
}
|
||||
|
||||
@@ -16,6 +16,12 @@ from backend.data.providers.tushare_transport import TushareError
|
||||
|
||||
|
||||
class DashboardMixin:
|
||||
def _now(self) -> datetime:
|
||||
clock = getattr(self, "clock", None)
|
||||
if callable(clock):
|
||||
return clock()
|
||||
return datetime.now().astimezone()
|
||||
|
||||
def dashboard(self, requested_date: str) -> dict[str, Any]:
|
||||
trade_date, previous_trade_date = self.resolve_trade_context(requested_date)
|
||||
if self.should_use_realtime(requested_date, trade_date):
|
||||
@@ -26,11 +32,12 @@ class DashboardMixin:
|
||||
)
|
||||
|
||||
daily = self._load_daily(trade_date)
|
||||
now = self._now()
|
||||
if (
|
||||
not daily
|
||||
and requested_date == datetime.now().astimezone().strftime("%Y%m%d")
|
||||
and requested_date == now.strftime("%Y%m%d")
|
||||
and trade_date == requested_date
|
||||
and datetime.now().astimezone().time().replace(tzinfo=None) >= dt_time(9, 15)
|
||||
and now.time().replace(tzinfo=None) >= dt_time(9, 15)
|
||||
):
|
||||
return self._realtime_dashboard(
|
||||
requested_date,
|
||||
@@ -48,16 +55,24 @@ class DashboardMixin:
|
||||
if not limit_rows:
|
||||
limit_data_source = "derived"
|
||||
notices.append("涨跌停高级接口当日数据尚未更新,已使用日线数据推算。")
|
||||
limit_rows = self._derive_limits(trade_date, daily)
|
||||
limit_rows = self._derive_limits(
|
||||
trade_date,
|
||||
daily,
|
||||
previous_limit_rows=previous_limit_rows,
|
||||
)
|
||||
except TushareError as exc:
|
||||
limit_data_source = "derived"
|
||||
notices.append(f"涨跌停高级接口不可用,已使用日线数据推算:{exc}")
|
||||
limit_rows = self._derive_limits(trade_date, daily)
|
||||
previous_daily = self._load_daily(previous_trade_date)
|
||||
previous_limit_rows = [
|
||||
row for row in self._derive_limits(previous_trade_date, previous_daily)
|
||||
if row.get("limit_type") == "U"
|
||||
]
|
||||
limit_rows = self._derive_limits(
|
||||
trade_date,
|
||||
daily,
|
||||
previous_limit_rows=previous_limit_rows,
|
||||
)
|
||||
|
||||
up_rows = [row for row in limit_rows if row.get("limit_type") == "U"]
|
||||
down_rows = [row for row in limit_rows if row.get("limit_type") == "D"]
|
||||
@@ -98,15 +113,14 @@ class DashboardMixin:
|
||||
}
|
||||
return apply_sentiment_to_dashboard(dashboard)
|
||||
|
||||
@staticmethod
|
||||
def should_use_realtime(requested_date: str, trade_date: str) -> bool:
|
||||
"""Use rt_k for today's open market until end-of-day datasets settle."""
|
||||
now = datetime.now().astimezone()
|
||||
def should_use_realtime(self, requested_date: str, trade_date: str) -> bool:
|
||||
"""Use live quotes for today's open session until official daily settles."""
|
||||
now = self._now()
|
||||
today = now.strftime("%Y%m%d")
|
||||
return (
|
||||
requested_date == today
|
||||
and trade_date == today
|
||||
and dt_time(9, 15) <= now.time().replace(tzinfo=None) < dt_time(16, 30)
|
||||
and dt_time(9, 15) <= now.time().replace(tzinfo=None) < dt_time(15, 5)
|
||||
)
|
||||
|
||||
def _realtime_dashboard(
|
||||
@@ -122,7 +136,7 @@ class DashboardMixin:
|
||||
)
|
||||
if not codes:
|
||||
raise TushareError("No active stock codes available for rt_k")
|
||||
quotes = self.query("rt_k", {"ts_code": codes})
|
||||
quotes, quote_source = self._load_realtime_quotes(codes, trade_date)
|
||||
if not quotes:
|
||||
raise TushareError(f"No realtime data returned for {trade_date}")
|
||||
|
||||
@@ -178,14 +192,35 @@ class DashboardMixin:
|
||||
)
|
||||
sectors = _build_sectors(limits)
|
||||
previous_sectors = _build_sectors(previous_limits)
|
||||
now = datetime.now().astimezone()
|
||||
now = self._now()
|
||||
market_status = _realtime_market_status(now.time().replace(tzinfo=None))
|
||||
if quote_source == "datahub":
|
||||
notice = (
|
||||
"盘中行情由数据中枢统一提供;涨停原因、封板时间和开板次数以盘后榜单校正为准。"
|
||||
)
|
||||
source_name = "datahub"
|
||||
elif quote_source == "eastmoney_clist":
|
||||
notice = (
|
||||
"盘中行情由东财免费实时快照计算;涨停原因、封板时间和开板次数以盘后榜单校正为准。"
|
||||
)
|
||||
source_name = "eastmoney"
|
||||
elif quote_source == "tencent_qt":
|
||||
notice = (
|
||||
"盘中行情由腾讯免费实时行情计算;涨停原因、封板时间和开板次数以盘后榜单校正为准。"
|
||||
)
|
||||
source_name = "tencent"
|
||||
else:
|
||||
notice = (
|
||||
"盘中行情由 Tushare rt_k 实时计算;涨停原因、封板时间和开板次数以盘后榜单校正为准。"
|
||||
)
|
||||
source_name = "tushare"
|
||||
dashboard = {
|
||||
"meta": {
|
||||
"requested_date": _display_date(requested_date),
|
||||
"trade_date": _display_date(trade_date),
|
||||
"previous_trade_date": _display_date(previous_trade_date),
|
||||
"source": "tushare",
|
||||
"source": source_name,
|
||||
"quote_source": quote_source,
|
||||
"mode": "realtime",
|
||||
"realtime": True,
|
||||
"market_status": market_status,
|
||||
@@ -193,7 +228,8 @@ class DashboardMixin:
|
||||
"auto_refresh": False,
|
||||
"quote_count": len(daily),
|
||||
"updated_at": now.isoformat(timespec="seconds"),
|
||||
"notice": "盘中行情由 Tushare rt_k 实时计算;涨停原因、封板时间和开板次数以盘后榜单校正为准。",
|
||||
"notice": notice,
|
||||
"indices": self._free_realtime_indices() if quote_source != "tushare_rt_k" else [],
|
||||
},
|
||||
"overview": _build_overview(daily, up_rows, down_rows, broken_rows),
|
||||
"limits": limits,
|
||||
@@ -207,6 +243,62 @@ class DashboardMixin:
|
||||
}
|
||||
return apply_sentiment_to_dashboard(dashboard)
|
||||
|
||||
def _realtime_aggregator(self):
|
||||
aggregator = getattr(self, "realtime_aggregator", None)
|
||||
if aggregator is None:
|
||||
raise TushareError("免费实时源未配置")
|
||||
return aggregator
|
||||
|
||||
def _load_realtime_quotes(
|
||||
self,
|
||||
codes: str,
|
||||
trade_date: str,
|
||||
) -> tuple[list[dict[str, Any]], str]:
|
||||
hub = getattr(self, "try_market_quotes", None)
|
||||
if callable(hub):
|
||||
quotes = hub(trade_date)
|
||||
if quotes:
|
||||
return list(quotes), "datahub"
|
||||
named = getattr(self, "try_quotes", None)
|
||||
code_list = [item for item in str(codes or "").split(",") if item]
|
||||
if callable(named) and code_list:
|
||||
collected: list[dict[str, Any]] = []
|
||||
for index in range(0, len(code_list), 60):
|
||||
collected.extend(named(code_list[index:index + 60]) or [])
|
||||
if collected:
|
||||
delayed = any(item.get("delayed") for item in collected)
|
||||
return collected, "datahub_delayed" if delayed else "datahub"
|
||||
try:
|
||||
quotes = self.query("rt_k", {"ts_code": codes})
|
||||
if quotes:
|
||||
delayed = any(item.get("delayed") for item in quotes)
|
||||
return list(quotes), "datahub_delayed" if delayed else "datahub"
|
||||
except TushareError as exc:
|
||||
raise TushareError(f"当天盘中实时行情不可用:{exc}") from exc
|
||||
raise TushareError("当天盘中实时行情不可用:数据中枢未返回可用行情")
|
||||
|
||||
def _mark_quote_legacy(self, source: str, error: str = "") -> None:
|
||||
marker = getattr(self, "record_datahub_legacy", None)
|
||||
if callable(marker):
|
||||
marker("quotes", source, error)
|
||||
|
||||
def _free_realtime_quotes(
|
||||
self,
|
||||
trade_date: str,
|
||||
codes: str = "",
|
||||
) -> tuple[list[dict[str, Any]], str]:
|
||||
del trade_date, codes
|
||||
raise TushareError("主网站不再直连免费行情源,请走数据中枢")
|
||||
|
||||
def _free_realtime_indices(self) -> list[dict[str, Any]]:
|
||||
hub = getattr(self, "try_index_quotes", None)
|
||||
if callable(hub):
|
||||
rows = hub()
|
||||
converted = [item for item in (_hub_index_quote(row) for row in rows or []) if item]
|
||||
if converted:
|
||||
return converted
|
||||
return []
|
||||
|
||||
def _load_realtime_reference(
|
||||
self,
|
||||
trade_date: str,
|
||||
@@ -234,7 +326,7 @@ class DashboardMixin:
|
||||
{"trade_date": previous_trade_date},
|
||||
"ts_code,trade_date,total_share,float_share,free_share,total_mv,circ_mv",
|
||||
)
|
||||
if not basic_rows or not price_limits:
|
||||
if not basic_rows:
|
||||
raise TushareError(f"Realtime reference data is incomplete for {trade_date}")
|
||||
result = {
|
||||
"basic_rows": basic_rows,
|
||||
@@ -254,10 +346,9 @@ class DashboardMixin:
|
||||
ts_code: str,
|
||||
reference_date: str = "",
|
||||
) -> dict[str, Any]:
|
||||
rows = self.query("rt_k", {"ts_code": ts_code})
|
||||
if not rows:
|
||||
row = self._realtime_quote_row(ts_code, reference_date)
|
||||
if not row:
|
||||
raise TushareError(f"No realtime quote returned for {ts_code}")
|
||||
row = rows[0]
|
||||
close = _number(row.get("close"))
|
||||
previous_close = _number(row.get("pre_close"))
|
||||
if close <= 0 or previous_close <= 0:
|
||||
@@ -285,6 +376,11 @@ class DashboardMixin:
|
||||
basic = basics[0] if basics else {}
|
||||
capital = self._latest_capital(ts_code, reference_date)
|
||||
float_share = _number(capital.get("float_share"))
|
||||
if not float_share and previous_close:
|
||||
# The hub's canonical valuation table stores market values but not
|
||||
# share counts. circ_mv (10k CNY) / price (CNY) is float shares in
|
||||
# 10k shares, the same unit used by the turnover formula below.
|
||||
float_share = _number(capital.get("circ_mv")) / previous_close
|
||||
# rt_k volume is shares; daily_basic float_share is reported in 10k shares.
|
||||
turnover_rate = _number(row.get("vol")) / float_share / 100 if float_share else 0
|
||||
market_date = reference_date or datetime.now().astimezone().strftime("%Y%m%d")
|
||||
@@ -306,6 +402,13 @@ class DashboardMixin:
|
||||
for item in market_rows:
|
||||
item_capital = capital_map.get(str(item.get("ts_code") or ""), {})
|
||||
item_float_share = _number(item_capital.get("float_share"))
|
||||
item_previous = _number(
|
||||
item.get("pre_close")
|
||||
if item.get("pre_close") not in (None, "")
|
||||
else item.get("previous_close")
|
||||
)
|
||||
if not item_float_share and item_previous:
|
||||
item_float_share = _number(item_capital.get("circ_mv")) / item_previous
|
||||
if item_float_share:
|
||||
market_turnovers.append(_number(item.get("vol")) / item_float_share / 100)
|
||||
market_turnover = (
|
||||
@@ -317,6 +420,10 @@ class DashboardMixin:
|
||||
market_date,
|
||||
_number(row.get("vol")) / 100,
|
||||
)
|
||||
trade_time = str(row.get("trade_time") or row.get("quote_time") or "")
|
||||
trade_date = str(row.get("quote_date") or "").replace("-", "")[:8]
|
||||
if not trade_date and trade_time:
|
||||
trade_date = trade_time[:10].replace("-", "")
|
||||
return {
|
||||
"code": ts_code.split(".")[0],
|
||||
"ts_code": ts_code,
|
||||
@@ -341,10 +448,26 @@ class DashboardMixin:
|
||||
"float_share_10k": float_share,
|
||||
"capital_trade_date": str(capital.get("trade_date") or ""),
|
||||
"turnover_source": "rt_volume/latest_float_share" if float_share else "unavailable",
|
||||
"data_source": "tushare",
|
||||
"data_source": str(row.get("source") or "tushare"),
|
||||
"trade_date": trade_date,
|
||||
"trade_time": trade_time,
|
||||
"realtime": True,
|
||||
}
|
||||
|
||||
def _realtime_quote_row(self, ts_code: str, reference_date: str = "") -> dict[str, Any]:
|
||||
hub = getattr(self, "try_quotes", None)
|
||||
if callable(hub):
|
||||
rows = hub([ts_code]) or []
|
||||
if rows:
|
||||
return dict(rows[0])
|
||||
try:
|
||||
rows = self.query("rt_k", {"ts_code": ts_code})
|
||||
if rows:
|
||||
return dict(rows[0])
|
||||
except TushareError:
|
||||
pass
|
||||
return {}
|
||||
|
||||
def _stock_activity_metrics(
|
||||
self,
|
||||
ts_code: str,
|
||||
@@ -364,7 +487,10 @@ class DashboardMixin:
|
||||
{
|
||||
"ts_code": ts_code,
|
||||
"start_date": (end - timedelta(days=30)).strftime("%Y%m%d"),
|
||||
"end_date": reference_date,
|
||||
# Intraday bars are not official daily history yet. Asking
|
||||
# the hub for today's daily row makes a complete historical
|
||||
# range look incomplete and discards otherwise valid data.
|
||||
"end_date": (end - timedelta(days=1)).strftime("%Y%m%d"),
|
||||
},
|
||||
"ts_code,trade_date,vol,amount",
|
||||
)
|
||||
@@ -462,7 +588,7 @@ class DashboardMixin:
|
||||
for row in reference.get("basic_rows") or []
|
||||
if row.get("ts_code")
|
||||
]
|
||||
quotes = self.query("rt_k", {"ts_code": ",".join(codes)}, "")
|
||||
quotes, quote_source = self._load_realtime_quotes(",".join(codes), trade_date)
|
||||
rows = [
|
||||
row for row in quotes
|
||||
if _number(row.get("close")) > 0 and _number(row.get("pre_close")) > 0
|
||||
@@ -489,7 +615,13 @@ class DashboardMixin:
|
||||
start_date = (end - timedelta(days=20)).strftime("%Y%m%d")
|
||||
rows = self.query(
|
||||
"daily_basic",
|
||||
{"ts_code": ts_code, "start_date": start_date, "end_date": end_date},
|
||||
{
|
||||
"ts_code": ts_code,
|
||||
"start_date": start_date,
|
||||
# Same rule as price history: today's official valuation is
|
||||
# unavailable during the session, so use the latest prior row.
|
||||
"end_date": (end - timedelta(days=1)).strftime("%Y%m%d"),
|
||||
},
|
||||
"ts_code,trade_date,turnover_rate,volume_ratio,total_share,float_share,"
|
||||
"free_share,total_mv,circ_mv",
|
||||
)
|
||||
@@ -608,6 +740,31 @@ def _build_yesterday_performance(
|
||||
return result
|
||||
|
||||
|
||||
def _hub_index_quote(row: dict[str, Any]) -> dict[str, Any] | None:
|
||||
ts_code = str(row.get("ts_code") or "")
|
||||
code = str(row.get("code") or ts_code.split(".")[0])
|
||||
close = _number(row.get("price") if row.get("price") not in (None, "") else row.get("close"))
|
||||
previous = _number(
|
||||
row.get("previous_close") if row.get("previous_close") not in (None, "") else row.get("pre_close")
|
||||
)
|
||||
if close <= 0 or previous <= 0:
|
||||
return None
|
||||
amount = _number(row.get("amount"))
|
||||
amount_billion = _number(row.get("amount_billion"))
|
||||
if not amount_billion and amount:
|
||||
amount_billion = round(amount / 100_000_000, 2)
|
||||
return {
|
||||
"code": code,
|
||||
"name": str(row.get("name") or code),
|
||||
"price": close,
|
||||
"change": _number(row.get("pct_chg") if row.get("pct_chg") not in (None, "") else row.get("change")),
|
||||
"previous_close": previous,
|
||||
"amount_billion": amount_billion,
|
||||
"quote_time": str(row.get("quote_time") or ""),
|
||||
"source": "datahub",
|
||||
}
|
||||
|
||||
|
||||
def _build_limit_performance(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for level in sorted({int(row.get("prior_streak") or 1) for row in rows}, reverse=True):
|
||||
|
||||
@@ -6,12 +6,72 @@ from typing import Any
|
||||
from backend.data.numbers import finite_number as _number
|
||||
|
||||
|
||||
def calendar_is_open(value: Any) -> bool:
|
||||
if value in (True, 1, "1", "Y", "y"):
|
||||
return True
|
||||
if value in (False, 0, "0", "N", "n", None, ""):
|
||||
return False
|
||||
try:
|
||||
return int(value) == 1
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return "、".join(str(item).strip() for item in value if str(item).strip())
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
def _optional_number(value: Any) -> float | None:
|
||||
if value in (None, "", "-"):
|
||||
return None
|
||||
number = _number(value, default=float("nan"))
|
||||
if number != number:
|
||||
return None
|
||||
return number
|
||||
|
||||
|
||||
def _moneyflow_payload(flow: dict[str, Any] | None) -> dict[str, Any]:
|
||||
if not flow:
|
||||
return {
|
||||
"available": False,
|
||||
"net_million": None,
|
||||
"large_million": None,
|
||||
"medium_million": None,
|
||||
"small_million": None,
|
||||
}
|
||||
net = _optional_number(flow.get("net_mf_amount"))
|
||||
buy_lg = _optional_number(flow.get("buy_lg_amount"))
|
||||
sell_lg = _optional_number(flow.get("sell_lg_amount"))
|
||||
buy_elg = _optional_number(flow.get("buy_elg_amount"))
|
||||
sell_elg = _optional_number(flow.get("sell_elg_amount"))
|
||||
buy_md = _optional_number(flow.get("buy_md_amount"))
|
||||
sell_md = _optional_number(flow.get("sell_md_amount"))
|
||||
buy_sm = _optional_number(flow.get("buy_sm_amount"))
|
||||
sell_sm = _optional_number(flow.get("sell_sm_amount"))
|
||||
large = None
|
||||
if None not in (buy_lg, sell_lg, buy_elg, sell_elg):
|
||||
large = (buy_lg + buy_elg - sell_lg - sell_elg)
|
||||
elif _optional_number(flow.get("large_amount")) is not None:
|
||||
large = _optional_number(flow.get("large_amount"))
|
||||
medium = None if None in (buy_md, sell_md) else (buy_md - sell_md)
|
||||
if medium is None:
|
||||
medium = _optional_number(flow.get("medium_amount"))
|
||||
small = None if None in (buy_sm, sell_sm) else (buy_sm - sell_sm)
|
||||
if small is None:
|
||||
small = _optional_number(flow.get("small_amount"))
|
||||
if net is None and large is None and medium is None and small is None:
|
||||
return _moneyflow_payload(None)
|
||||
return {
|
||||
"available": True,
|
||||
"net_million": None if net is None else round(net / 100, 2),
|
||||
"large_million": None if large is None else round(large / 100, 2),
|
||||
"medium_million": None if medium is None else round(medium / 100, 2),
|
||||
"small_million": None if small is None else round(small / 100, 2),
|
||||
}
|
||||
|
||||
|
||||
def _prices_equal(left: Any, right: Any) -> bool:
|
||||
if left is None or right is None:
|
||||
return False
|
||||
|
||||
@@ -59,6 +59,73 @@ class IndexMixin:
|
||||
}
|
||||
|
||||
def realtime_market_indices(self, requested_date: str) -> dict[str, Any]:
|
||||
hub = getattr(self, "try_index_quotes", None)
|
||||
if callable(hub):
|
||||
rows = hub()
|
||||
if rows:
|
||||
return self._hub_realtime_market_indices(requested_date, rows)
|
||||
raise TushareError("Realtime index quotes are incomplete")
|
||||
|
||||
def _hub_realtime_market_indices(
|
||||
self,
|
||||
requested_date: str,
|
||||
rows: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
trade_date, _ = self.resolve_trade_context(requested_date)
|
||||
index_names = {
|
||||
"000001.SH": "上证指数",
|
||||
"399001.SZ": "深证成指",
|
||||
"399006.SZ": "创业板指",
|
||||
}
|
||||
by_code = {str(row.get("ts_code") or ""): row for row in rows}
|
||||
by_symbol = {str(row.get("code") or ""): row for row in rows}
|
||||
indices = []
|
||||
for ts_code, name in index_names.items():
|
||||
row = by_code.get(ts_code) or by_symbol.get(ts_code.split(".")[0])
|
||||
if not row:
|
||||
continue
|
||||
close = _number(row.get("price") if row.get("price") not in (None, "") else row.get("close"))
|
||||
previous_close = _number(
|
||||
row.get("previous_close") if row.get("previous_close") not in (None, "") else row.get("pre_close")
|
||||
)
|
||||
if close <= 0 or previous_close <= 0:
|
||||
continue
|
||||
amount = _number(row.get("amount"))
|
||||
amount_billion = _number(row.get("amount_billion"))
|
||||
if not amount_billion and amount:
|
||||
amount_billion = round(amount / 100_000_000, 2)
|
||||
indices.append(
|
||||
{
|
||||
"ts_code": ts_code,
|
||||
"name": str(row.get("name") or name).strip(),
|
||||
"trade_date": trade_date,
|
||||
"close": close,
|
||||
"pct_chg": round(
|
||||
_number(row.get("pct_chg")) or (close / previous_close - 1) * 100,
|
||||
3,
|
||||
),
|
||||
"return_5d": 0,
|
||||
"amount_billion": amount_billion,
|
||||
"quote_time": str(row.get("quote_time") or ""),
|
||||
"source": "datahub",
|
||||
}
|
||||
)
|
||||
if len(indices) != 3:
|
||||
raise TushareError("Realtime index quotes are incomplete")
|
||||
return {
|
||||
"trade_date": trade_date,
|
||||
"source": "datahub",
|
||||
"realtime": True,
|
||||
"precise": True,
|
||||
"indices": indices,
|
||||
"aggregate": {
|
||||
"average_pct_chg": round(sum(item["pct_chg"] for item in indices) / len(indices), 3),
|
||||
"average_return_5d": 0,
|
||||
"average_return_20d": 0,
|
||||
},
|
||||
}
|
||||
|
||||
def _tushare_realtime_market_indices(self, requested_date: str) -> dict[str, Any]:
|
||||
trade_date, _ = self.resolve_trade_context(requested_date)
|
||||
index_names = {
|
||||
"000001.SH": "上证指数",
|
||||
@@ -116,3 +183,7 @@ class IndexMixin:
|
||||
"average_return_20d": 0,
|
||||
},
|
||||
}
|
||||
|
||||
def _free_realtime_market_indices(self, requested_date: str) -> dict[str, Any]:
|
||||
del requested_date
|
||||
raise TushareError("主网站不再直连免费行情源,请走数据中枢")
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.data.numbers import finite_number as _number
|
||||
from backend.data.providers.tushare_transport import TushareError
|
||||
|
||||
_SW_MEMBER_TTL = timedelta(hours=24)
|
||||
_SW_MEMBER_DIR = Path(__file__).resolve().parents[3] / "data" / "cache" / "sw_members"
|
||||
|
||||
|
||||
class ShenwanIndustryMixin:
|
||||
def sw_stock_industry(self, ts_code: str, trade_date: str) -> dict[str, Any]:
|
||||
@@ -132,21 +137,73 @@ class ShenwanIndustryMixin:
|
||||
actual_trade_date = str(daily.get("trade_date") or "")
|
||||
outer_precise = actual_trade_date == trade_date
|
||||
outer_error = "" if outer_precise else (
|
||||
f"No Shenwan daily returned for {sector_code} on {trade_date}"
|
||||
f"申万行业 {sector_code} 当日盘后正式数据尚未入库"
|
||||
)
|
||||
outer_source = "tushare_sw_daily" if outer_precise else "unavailable"
|
||||
if not outer_precise and allow_realtime_close:
|
||||
inner_ok = bool(member_rows) and not coverage_issue
|
||||
if inner_ok:
|
||||
sw_row, rt_source, rt_error = self._sw_outer_realtime(
|
||||
sector_code,
|
||||
str(industry.get("l2_name") or ""),
|
||||
trade_date,
|
||||
finalized=True,
|
||||
)
|
||||
if sw_row:
|
||||
daily = sw_row
|
||||
actual_trade_date = str(
|
||||
sw_row.get("quote_date") or sw_row.get("trade_date") or ""
|
||||
)
|
||||
trade_time = str(sw_row.get("trade_time") or sw_row.get("quote_time") or "")
|
||||
quote_clock = (
|
||||
trade_time[11:19]
|
||||
if len(trade_time) >= 19
|
||||
else str(sw_row.get("quote_clock") or "")
|
||||
)
|
||||
outer_precise = actual_trade_date == trade_date
|
||||
if quote_clock and quote_clock < "15:00:00":
|
||||
outer_precise = False
|
||||
outer_source = rt_source or "eastmoney_sw"
|
||||
outer_error = "" if outer_precise else (
|
||||
rt_error or f"申万行业 {sector_code} 免费实时尚未形成收盘快照"
|
||||
)
|
||||
else:
|
||||
outer_error = rt_error or outer_error
|
||||
if not outer_precise:
|
||||
# The exact SW L2 member set and every member's final daily bar
|
||||
# are already available. Use their equal-weight move as an
|
||||
# explicitly labelled provisional outer value until Tushare's
|
||||
# official sw_daily row is published.
|
||||
daily = {"change": equal_change}
|
||||
actual_trade_date = trade_date
|
||||
outer_precise = True
|
||||
outer_source = "sw_member_equal_daily"
|
||||
outer_error = ""
|
||||
else:
|
||||
try:
|
||||
return self._sw_realtime_sector_snapshot(
|
||||
snapshot = self._sw_realtime_sector_snapshot(
|
||||
industry,
|
||||
members,
|
||||
trade_date,
|
||||
previous_trade_date,
|
||||
finalized=True,
|
||||
)
|
||||
except TushareError as exc:
|
||||
outer_error = f"{outer_error}; realtime close fallback failed: {exc}"
|
||||
snapshot.update({
|
||||
"raw_member_count": raw_member_count,
|
||||
"excluded_member_count": len(excluded_members),
|
||||
"excluded_members": excluded_members,
|
||||
})
|
||||
return snapshot
|
||||
except TushareError:
|
||||
outer_error = f"{outer_error}; 免费实时成分暂不可用"
|
||||
|
||||
official_change = _number(daily.get("pct_change")) if outer_precise else None
|
||||
official_change = None
|
||||
if outer_precise:
|
||||
official_change = _number(
|
||||
daily.get("pct_change")
|
||||
if daily.get("pct_change") not in (None, "")
|
||||
else daily.get("change")
|
||||
)
|
||||
return {
|
||||
"code": sector_code,
|
||||
"name": industry.get("l2_name") or daily.get("name") or sector_code,
|
||||
@@ -173,9 +230,9 @@ class ShenwanIndustryMixin:
|
||||
"amount_billion": round(amount_billion, 2),
|
||||
"count": 0,
|
||||
"max_streak": 0,
|
||||
"source": "tushare_sw_daily+member_daily" if outer_precise else "tushare_member_daily",
|
||||
"source": f"{outer_source}+tushare_member_daily" if outer_precise else "tushare_member_daily",
|
||||
"inner_source": "tushare_member_daily",
|
||||
"outer_source": "tushare_sw_daily" if outer_precise else "unavailable",
|
||||
"outer_source": outer_source,
|
||||
"taxonomy": "sw_l2",
|
||||
"industry": industry,
|
||||
"trade_date": trade_date,
|
||||
@@ -189,7 +246,7 @@ class ShenwanIndustryMixin:
|
||||
"inner_error": inner_error,
|
||||
"outer_error": outer_error,
|
||||
"schema_version": 6,
|
||||
"methodology": "外显使用申万二级行业官方日线;内核独立使用当日成分日线宽度与等权涨跌聚合",
|
||||
"methodology": "外显使用已发布 sw_daily 或免费申万实时;内核优先使用当日成分日线,不调用 rt_sw_k",
|
||||
}
|
||||
|
||||
def _sw_sector_members(
|
||||
@@ -197,7 +254,11 @@ class ShenwanIndustryMixin:
|
||||
sector_code: str,
|
||||
trade_date: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = []
|
||||
cached_rows = self._read_local_sw_members(sector_code)
|
||||
if cached_rows is not None:
|
||||
return _active_members(cached_rows, trade_date)
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
for is_new in ("Y", "N"):
|
||||
rows.extend(
|
||||
self.query(
|
||||
@@ -206,14 +267,87 @@ class ShenwanIndustryMixin:
|
||||
"l2_code,l2_name,ts_code,name,in_date,out_date,is_new",
|
||||
)
|
||||
)
|
||||
deduped: dict[str, dict[str, Any]] = {}
|
||||
for row in _reconcile_membership_rows(rows):
|
||||
code = str(row.get("ts_code") or "")
|
||||
if code and _membership_active_on(row, trade_date):
|
||||
current = deduped.get(code)
|
||||
if current is None or str(row.get("in_date") or "") > str(current.get("in_date") or ""):
|
||||
deduped[code] = row
|
||||
return list(deduped.values())
|
||||
except TushareError:
|
||||
stale = self._read_local_sw_members(sector_code, allow_stale=True) or []
|
||||
if stale:
|
||||
return _active_members(stale, trade_date)
|
||||
raise
|
||||
reconciled = _reconcile_membership_rows(rows)
|
||||
self._write_local_sw_members(sector_code, reconciled)
|
||||
return _active_members(reconciled, trade_date)
|
||||
|
||||
def _read_local_sw_members(
|
||||
self,
|
||||
sector_code: str,
|
||||
allow_stale: bool = False,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
now = datetime.now().astimezone()
|
||||
cache = getattr(self, "_sw_member_cache", None)
|
||||
lock = getattr(self, "_sw_member_lock", None)
|
||||
if isinstance(cache, dict) and lock is not None:
|
||||
with lock:
|
||||
packed = cache.get(sector_code)
|
||||
if isinstance(packed, dict):
|
||||
loaded_at = packed.get("loaded_at")
|
||||
rows = packed.get("rows")
|
||||
fresh = (
|
||||
isinstance(loaded_at, datetime)
|
||||
and now - loaded_at < _SW_MEMBER_TTL
|
||||
)
|
||||
if isinstance(rows, list) and (fresh or allow_stale):
|
||||
return [dict(item) for item in rows]
|
||||
path = _sw_member_path(sector_code)
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
rows = list(payload.get("rows") or [])
|
||||
updated = str(payload.get("updated_at") or "")
|
||||
fresh = False
|
||||
try:
|
||||
stamped = datetime.fromisoformat(updated)
|
||||
if stamped.tzinfo is None:
|
||||
stamped = stamped.replace(tzinfo=now.tzinfo)
|
||||
fresh = now - stamped.astimezone(now.tzinfo) < _SW_MEMBER_TTL
|
||||
except ValueError:
|
||||
fresh = False
|
||||
if rows and (fresh or allow_stale):
|
||||
self._remember_sw_members(sector_code, rows)
|
||||
return rows
|
||||
return None
|
||||
|
||||
def _write_local_sw_members(self, sector_code: str, rows: list[dict[str, Any]]) -> None:
|
||||
packed = [dict(item) for item in rows]
|
||||
self._remember_sw_members(sector_code, packed)
|
||||
path = _sw_member_path(sector_code)
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"sector_code": sector_code,
|
||||
"updated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
"rows": packed,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _remember_sw_members(self, sector_code: str, rows: list[dict[str, Any]]) -> None:
|
||||
cache = getattr(self, "_sw_member_cache", None)
|
||||
lock = getattr(self, "_sw_member_lock", None)
|
||||
if not isinstance(cache, dict) or lock is None:
|
||||
return
|
||||
with lock:
|
||||
cache[sector_code] = {
|
||||
"loaded_at": datetime.now().astimezone(),
|
||||
"rows": [dict(item) for item in rows],
|
||||
}
|
||||
|
||||
def sw_sector_members(self, sector_code: str, trade_date: str) -> list[dict[str, Any]]:
|
||||
"""Return constituents active in a Shenwan L2 industry on the target date."""
|
||||
@@ -311,37 +445,37 @@ class ShenwanIndustryMixin:
|
||||
finalized: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
sector_code = str(industry.get("l2_code") or "")
|
||||
sw_rows = self.query(
|
||||
"rt_sw_k",
|
||||
{"ts_code": sector_code},
|
||||
"ts_code,name,trade_time,close,pre_close,high,open,low,vol,amount,pct_change",
|
||||
sw_row, outer_source, outer_error = self._sw_outer_realtime(
|
||||
sector_code,
|
||||
str(industry.get("l2_name") or ""),
|
||||
trade_date,
|
||||
finalized=finalized,
|
||||
)
|
||||
sw_row = sw_rows[0] if sw_rows else {}
|
||||
trade_time = str(sw_row.get("trade_time") or "")
|
||||
quote_date = trade_time[:10].replace("-", "")
|
||||
quote_clock = trade_time[11:19] if len(trade_time) >= 19 else ""
|
||||
trade_time = str(sw_row.get("trade_time") or sw_row.get("quote_time") or "")
|
||||
quote_date = str(sw_row.get("quote_date") or trade_time[:10].replace("-", ""))
|
||||
quote_clock = trade_time[11:19] if len(trade_time) >= 19 else str(sw_row.get("quote_clock") or "")
|
||||
outer_precise = bool(sw_row and quote_date == trade_date)
|
||||
if finalized and (not quote_clock or quote_clock < "15:00:00"):
|
||||
if finalized and quote_clock and quote_clock < "15:00:00":
|
||||
outer_precise = False
|
||||
official_change = _number(sw_row.get("pct_change"))
|
||||
official_change = _number(sw_row.get("pct_change") if sw_row.get("pct_change") not in (None, "") else sw_row.get("change"))
|
||||
if not official_change:
|
||||
close = _number(sw_row.get("close"))
|
||||
pre_close = _number(sw_row.get("pre_close"))
|
||||
close = _number(sw_row.get("close") if sw_row.get("close") not in (None, "") else sw_row.get("price"))
|
||||
pre_close = _number(sw_row.get("pre_close") if sw_row.get("pre_close") not in (None, "") else sw_row.get("previous_close"))
|
||||
official_change = (close / pre_close - 1) * 100 if close and pre_close else 0
|
||||
if not outer_precise:
|
||||
official_change = None
|
||||
outer_error = ""
|
||||
if not sw_row:
|
||||
outer_error = f"No Shenwan realtime index returned for {sector_code}"
|
||||
elif quote_date != trade_date:
|
||||
outer_error = f"Shenwan realtime index date is {quote_date or 'unknown'}, expected {trade_date}"
|
||||
elif finalized and (not quote_clock or quote_clock < "15:00:00"):
|
||||
outer_error = f"Shenwan realtime index is not a close snapshot ({trade_time})"
|
||||
if not sw_row and not outer_error:
|
||||
outer_error = f"申万行业 {sector_code} 当日外显待盘后正式数据或免费实时源"
|
||||
elif quote_date and quote_date != trade_date:
|
||||
outer_error = f"申万实时行业日期是 {quote_date},期望 {trade_date}"
|
||||
elif finalized and quote_clock and quote_clock < "15:00:00":
|
||||
outer_error = f"申万行业尚未形成收盘快照({trade_time})"
|
||||
|
||||
valid: list[dict[str, Any]] = []
|
||||
codes: list[str] = []
|
||||
reference: dict[str, Any] = {}
|
||||
inner_error = ""
|
||||
inner_source = "unavailable"
|
||||
try:
|
||||
reference = self._load_realtime_reference(trade_date, previous_trade_date)
|
||||
active_codes = {
|
||||
@@ -352,20 +486,27 @@ class ShenwanIndustryMixin:
|
||||
codes = [
|
||||
str(row.get("ts_code") or "")
|
||||
for row in members
|
||||
if str(row.get("ts_code") or "") in active_codes
|
||||
if str(row.get("ts_code") or "")
|
||||
]
|
||||
if codes:
|
||||
quotes = self.query("rt_k", {"ts_code": ",".join(codes)}, "")
|
||||
if active_codes:
|
||||
listed = [code for code in codes if code in active_codes]
|
||||
if listed:
|
||||
codes = listed
|
||||
quotes, inner_source = self._load_member_realtime_quotes(codes, trade_date)
|
||||
for row in quotes:
|
||||
close = _number(row.get("close"))
|
||||
previous_close = _number(row.get("pre_close"))
|
||||
if close <= 0 or previous_close <= 0:
|
||||
continue
|
||||
valid.append({**row, "change": (close / previous_close - 1) * 100})
|
||||
else:
|
||||
if not codes:
|
||||
inner_error = f"No active Shenwan members returned for {sector_code}"
|
||||
elif not quotes:
|
||||
inner_error = f"申万成分实时行情暂不可用:{sector_code}"
|
||||
except TushareError as exc:
|
||||
inner_error = str(exc)
|
||||
if "rt_k" in inner_error or "权限" in inner_error:
|
||||
inner_error = "申万成分实时行情暂不可用,已避开无权限接口"
|
||||
|
||||
coverage = len(valid) / max(len(codes), 1) * 100
|
||||
valid_codes = {str(item.get("ts_code") or "") for item in valid}
|
||||
@@ -389,17 +530,25 @@ class ShenwanIndustryMixin:
|
||||
for item in members
|
||||
}
|
||||
equal_change = sum(item["change"] for item in valid) / len(valid) if valid else 0
|
||||
if not outer_precise and inner_precise:
|
||||
official_change = equal_change
|
||||
outer_precise = True
|
||||
outer_source = "sw_member_equal_realtime"
|
||||
outer_error = ""
|
||||
quote_date = trade_date
|
||||
amount_billion = sum(_number(item.get("amount")) for item in valid) / 100000000
|
||||
market_rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
self._ensure_realtime_market_cache(trade_date)
|
||||
with self._realtime_reference_lock:
|
||||
market_rows = list(
|
||||
(self._latest_realtime_market.get(trade_date) or {}).get("rows") or []
|
||||
)
|
||||
market_rows = self._ensure_realtime_market_cache(trade_date)
|
||||
except TushareError as exc:
|
||||
market_rows = []
|
||||
message = str(exc)
|
||||
if "rt_k" in message or "权限" in message:
|
||||
market_error = "全市场实时行情暂不可用,已避开无权限接口"
|
||||
else:
|
||||
market_error = message
|
||||
if not valid:
|
||||
inner_precise = False
|
||||
inner_error = inner_error or str(exc)
|
||||
inner_error = inner_error or market_error
|
||||
capital_map = {
|
||||
str(item.get("ts_code") or ""): item
|
||||
for item in reference.get("capital_rows") or []
|
||||
@@ -408,20 +557,42 @@ class ShenwanIndustryMixin:
|
||||
for item in valid:
|
||||
capital = capital_map.get(str(item.get("ts_code") or ""), {})
|
||||
float_share = _number(capital.get("float_share"))
|
||||
if float_share:
|
||||
sector_turnovers.append(_number(item.get("vol")) / float_share / 100)
|
||||
previous = _number(
|
||||
item.get("pre_close")
|
||||
if item.get("pre_close") not in (None, "")
|
||||
else item.get("previous_close")
|
||||
)
|
||||
if not float_share and previous:
|
||||
float_share = _number(capital.get("circ_mv")) / previous
|
||||
volume = _number(item.get("vol"))
|
||||
if float_share and volume:
|
||||
# 免费源成交量为股;daily_basic.float_share 为万股。
|
||||
sector_turnovers.append(volume / float_share / 100)
|
||||
market_turnovers = []
|
||||
for item in market_rows:
|
||||
capital = capital_map.get(str(item.get("ts_code") or ""), {})
|
||||
float_share = _number(capital.get("float_share"))
|
||||
if float_share:
|
||||
market_turnovers.append(_number(item.get("vol")) / float_share / 100)
|
||||
previous = _number(
|
||||
item.get("pre_close")
|
||||
if item.get("pre_close") not in (None, "")
|
||||
else item.get("previous_close")
|
||||
)
|
||||
if not float_share and previous:
|
||||
float_share = _number(capital.get("circ_mv")) / previous
|
||||
volume = _number(item.get("vol"))
|
||||
if float_share and volume:
|
||||
market_turnovers.append(volume / float_share / 100)
|
||||
average_turnover = sum(sector_turnovers) / len(sector_turnovers) if sector_turnovers else 0
|
||||
market_turnover = sum(market_turnovers) / len(market_turnovers) if market_turnovers else 0
|
||||
relative_turnover = average_turnover / market_turnover if market_turnover else 0
|
||||
if not relative_turnover:
|
||||
inner_precise = False
|
||||
inner_error = inner_error or "Shenwan member relative turnover is unavailable"
|
||||
delayed = "delayed" in str(inner_source) or any(item.get("delayed") for item in valid)
|
||||
delay_seconds = max((int(item.get("delay_seconds") or 0) for item in valid), default=0)
|
||||
delay_notice = ""
|
||||
if delayed:
|
||||
delay_notice = next(
|
||||
(str(item.get("delay_notice") or "") for item in valid if item.get("delay_notice")),
|
||||
"",
|
||||
) or f"主备免费行情均暂不可用,显示最近一次真实快照(延迟 {delay_seconds} 秒)"
|
||||
return {
|
||||
"code": sector_code,
|
||||
"name": str(industry.get("l2_name") or sw_row.get("name") or ""),
|
||||
@@ -447,9 +618,9 @@ class ShenwanIndustryMixin:
|
||||
"amount_billion": round(amount_billion, 2),
|
||||
"count": sum(item["change"] >= 9.5 for item in valid),
|
||||
"max_streak": 0,
|
||||
"source": "tushare_rt_sw_k+sw_members_rt_k",
|
||||
"inner_source": "tushare_sw_members+rt_k",
|
||||
"outer_source": "tushare_rt_sw_k",
|
||||
"source": f"{outer_source or 'unavailable'}+{inner_source}",
|
||||
"inner_source": inner_source,
|
||||
"outer_source": outer_source or "unavailable",
|
||||
"taxonomy": "sw_l2",
|
||||
"industry": industry,
|
||||
"trade_date": trade_date,
|
||||
@@ -463,10 +634,101 @@ class ShenwanIndustryMixin:
|
||||
"precise": inner_precise and outer_precise,
|
||||
"inner_error": inner_error,
|
||||
"outer_error": outer_error,
|
||||
"delayed": delayed,
|
||||
"delay_seconds": delay_seconds,
|
||||
"delay_notice": delay_notice,
|
||||
"schema_version": 6,
|
||||
"methodology": "外显使用申万官方 rt_sw_k;内核独立使用申万成分 rt_k 宽度与相对换手聚合",
|
||||
"methodology": "外显使用已发布 sw_daily 或免费申万实时;内核使用数据中枢/免费实时成分,不调用 rt_sw_k",
|
||||
}
|
||||
|
||||
def _sw_outer_realtime(
|
||||
self,
|
||||
sector_code: str,
|
||||
sector_name: str,
|
||||
trade_date: str,
|
||||
finalized: bool = False,
|
||||
) -> tuple[dict[str, Any], str, str]:
|
||||
hub = getattr(self, "try_sector_quote", None)
|
||||
if callable(hub):
|
||||
try:
|
||||
row = hub(sector_code, "" if finalized else trade_date)
|
||||
except Exception as exc:
|
||||
message = str(exc)
|
||||
if finalized:
|
||||
return {}, "", f"申万行业 {sector_code} 盘后正式数据待入库"
|
||||
return {}, "", f"数据中枢申万实时暂不可用:{message[:180]}"
|
||||
if row:
|
||||
actual_name = str(row.get("name") or "").strip()
|
||||
if sector_name and actual_name != str(sector_name).strip():
|
||||
return {}, "", (
|
||||
f"数据中枢返回的行业名称不匹配:期望 {sector_name},实际 {actual_name or '--'}"
|
||||
)
|
||||
return dict(row), str(row.get("source") or "datahub"), ""
|
||||
if finalized:
|
||||
return {}, "", f"申万行业 {sector_code} 当日盘后正式数据尚未入库"
|
||||
return {}, "", f"申万行业 {sector_code} 当日外显待补充"
|
||||
|
||||
def _load_member_realtime_quotes(
|
||||
self,
|
||||
codes: list[str],
|
||||
trade_date: str,
|
||||
) -> tuple[list[dict[str, Any]], str]:
|
||||
wanted = [str(code).strip() for code in codes if str(code or "").strip()]
|
||||
if not wanted:
|
||||
return [], "unavailable"
|
||||
best_rows: list[dict[str, Any]] = []
|
||||
best_source = "unavailable"
|
||||
|
||||
def consider(rows: list[dict[str, Any]] | None, source: str) -> list[dict[str, Any]]:
|
||||
nonlocal best_rows, best_source
|
||||
filtered = _filter_quotes_for_codes(rows, wanted)
|
||||
if len(filtered) > len(best_rows):
|
||||
best_rows = filtered
|
||||
best_source = source
|
||||
return filtered
|
||||
|
||||
hub_market = getattr(self, "try_market_quotes", None)
|
||||
if callable(hub_market):
|
||||
filtered = consider(hub_market(trade_date) or [], "datahub")
|
||||
if len(filtered) >= max(1, int(len(wanted) * 0.9)):
|
||||
delayed = any(item.get("delayed") for item in filtered)
|
||||
return filtered, "datahub_delayed" if delayed else "datahub"
|
||||
|
||||
hub = getattr(self, "try_quotes", None)
|
||||
if callable(hub):
|
||||
collected: list[dict[str, Any]] = []
|
||||
for index in range(0, len(wanted), _QUOTE_BATCH):
|
||||
collected.extend(hub(wanted[index:index + _QUOTE_BATCH]) or [])
|
||||
filtered = consider(collected, "datahub")
|
||||
if len(filtered) >= max(1, int(len(wanted) * 0.9)):
|
||||
delayed = any(item.get("delayed") for item in filtered)
|
||||
return filtered, "datahub_delayed" if delayed else "datahub"
|
||||
|
||||
if best_rows:
|
||||
delayed = any(item.get("delayed") for item in best_rows)
|
||||
if delayed and not str(best_source).endswith("_delayed"):
|
||||
return best_rows, f"{best_source}_delayed"
|
||||
return best_rows, best_source
|
||||
return [], "unavailable"
|
||||
|
||||
|
||||
_QUOTE_BATCH = 60
|
||||
|
||||
|
||||
def _filter_quotes_for_codes(
|
||||
rows: list[dict[str, Any]] | None,
|
||||
codes: list[str],
|
||||
) -> list[dict[str, Any]]:
|
||||
wanted = {str(code) for code in codes if code}
|
||||
filtered: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for row in rows or []:
|
||||
ts_code = str(row.get("ts_code") or "")
|
||||
if ts_code in wanted and ts_code not in seen:
|
||||
seen.add(ts_code)
|
||||
filtered.append(row)
|
||||
return filtered
|
||||
|
||||
|
||||
def _filter_members_by_listing(
|
||||
members: list[dict[str, Any]],
|
||||
@@ -568,6 +830,22 @@ def _reconcile_membership_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any
|
||||
return list(reconciled.values())
|
||||
|
||||
|
||||
def _sw_member_path(sector_code: str) -> Path:
|
||||
safe = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in str(sector_code or ""))
|
||||
return _SW_MEMBER_DIR / f"{safe or 'unknown'}.json"
|
||||
|
||||
|
||||
def _active_members(rows: list[dict[str, Any]], trade_date: str) -> list[dict[str, Any]]:
|
||||
deduped: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
code = str(row.get("ts_code") or "")
|
||||
if code and _membership_active_on(row, trade_date):
|
||||
current = deduped.get(code)
|
||||
if current is None or str(row.get("in_date") or "") > str(current.get("in_date") or ""):
|
||||
deduped[code] = dict(row)
|
||||
return list(deduped.values())
|
||||
|
||||
|
||||
def _match_sector_row(rows: list[dict[str, Any]], identifier: str) -> dict[str, Any] | None:
|
||||
if not rows:
|
||||
return None
|
||||
|
||||
@@ -5,13 +5,14 @@ from typing import Any
|
||||
|
||||
from backend.bootstrap.config import display_compact_date as _display_date
|
||||
from backend.data.numbers import finite_number as _number
|
||||
from backend.data.providers.tushare_helpers import _moneyflow_payload
|
||||
|
||||
|
||||
class StockMixin:
|
||||
def stock_detail(self, ts_code: str, requested_date: str) -> dict[str, Any]:
|
||||
trade_date, _ = self.resolve_trade_context(requested_date)
|
||||
end = datetime.strptime(trade_date, "%Y%m%d")
|
||||
start_date = (end - timedelta(days=190)).strftime("%Y%m%d")
|
||||
start_date = (end - timedelta(days=400)).strftime("%Y%m%d")
|
||||
daily = self.query(
|
||||
"daily",
|
||||
{"ts_code": ts_code, "start_date": start_date, "end_date": trade_date},
|
||||
@@ -41,7 +42,7 @@ class StockMixin:
|
||||
factor_map = {row["trade_date"]: _number(row.get("adj_factor"), 1) for row in factors}
|
||||
latest_factor = max(factor_map.values(), default=1) or 1
|
||||
prices = []
|
||||
for row in sorted(daily, key=lambda item: item.get("trade_date", ""))[-90:]:
|
||||
for row in sorted(daily, key=lambda item: item.get("trade_date", ""))[-250:]:
|
||||
factor = factor_map.get(row.get("trade_date"), latest_factor)
|
||||
ratio = factor / latest_factor
|
||||
prices.append(
|
||||
@@ -56,7 +57,7 @@ class StockMixin:
|
||||
"amount_billion": round(_number(row.get("amount")) / 100000, 2),
|
||||
}
|
||||
)
|
||||
flow = moneyflow[0] if moneyflow else {}
|
||||
flow = moneyflow[0] if moneyflow else None
|
||||
basic = basics[0] if basics else {}
|
||||
daily_basic = daily_basics[0] if daily_basics else {}
|
||||
latest = prices[-1] if prices else {}
|
||||
@@ -87,22 +88,7 @@ class StockMixin:
|
||||
"amount_billion": latest.get("amount_billion", 0),
|
||||
},
|
||||
"prices": prices,
|
||||
"moneyflow": {
|
||||
"net_million": round(_number(flow.get("net_mf_amount")) / 100, 2),
|
||||
"large_million": round(
|
||||
(_number(flow.get("buy_lg_amount")) + _number(flow.get("buy_elg_amount"))
|
||||
- _number(flow.get("sell_lg_amount")) - _number(flow.get("sell_elg_amount"))) / 100,
|
||||
2,
|
||||
),
|
||||
"medium_million": round(
|
||||
(_number(flow.get("buy_md_amount")) - _number(flow.get("sell_md_amount"))) / 100,
|
||||
2,
|
||||
),
|
||||
"small_million": round(
|
||||
(_number(flow.get("buy_sm_amount")) - _number(flow.get("sell_sm_amount"))) / 100,
|
||||
2,
|
||||
),
|
||||
},
|
||||
"moneyflow": _moneyflow_payload(flow),
|
||||
}
|
||||
|
||||
def stock_intraday(self, ts_code: str, requested_date: str) -> dict[str, Any]:
|
||||
|
||||
@@ -20,6 +20,8 @@ class TushareTransportMixin:
|
||||
params: dict[str, Any] | None = None,
|
||||
fields: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
if api_name == "rt_sw_k":
|
||||
raise TushareError("rt_sw_k is disabled; use published sw_daily or free Shenwan realtime")
|
||||
payload = json.dumps(
|
||||
{
|
||||
"api_name": api_name,
|
||||
|
||||
+521
-4
@@ -19,8 +19,22 @@ class RealtimeAggregateError(RuntimeError):
|
||||
|
||||
|
||||
EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get"
|
||||
EASTMONEY_STOCK_URL = "https://push2.eastmoney.com/api/qt/stock/get"
|
||||
EASTMONEY_STOCK_FIELDS = "f43,f44,f45,f46,f47,f48,f57,f58,f60,f86,f168,f62,f66,f72,f78,f84"
|
||||
EASTMONEY_SECTOR_URL = "https://push2.eastmoney.com/api/qt/clist/get"
|
||||
EASTMONEY_ZT_POOL_URL = "https://push2ex.eastmoney.com/getTopicZTPool"
|
||||
EASTMONEY_ZB_POOL_URL = "https://push2ex.eastmoney.com/getTopicZBPool"
|
||||
EASTMONEY_A_SHARE_BOARDS = (
|
||||
"m:0+t:6",
|
||||
"m:0+t:80",
|
||||
"m:1+t:2",
|
||||
"m:1+t:23",
|
||||
"m:0+t:81",
|
||||
)
|
||||
EASTMONEY_QUOTE_FIELDS = "f12,f13,f14,f2,f3,f4,f5,f6,f15,f16,f17,f18,f8,f124"
|
||||
EASTMONEY_MARKET_PAGE_SIZE = 100
|
||||
TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006"
|
||||
TENCENT_QUOTE_URL = "https://qt.gtimg.cn/q="
|
||||
THS_LIMIT_URL = "https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool"
|
||||
XGB_POOL_URL = "https://flash-api.xuangubao.cn/api/pool/detail"
|
||||
BROWSER_USER_AGENT = (
|
||||
@@ -134,6 +148,312 @@ class WebRealtimeAggregator:
|
||||
raise RealtimeAggregateError(f"Eastmoney returned {len(result)}/3 indices")
|
||||
return result
|
||||
|
||||
def eastmoney_market_quotes(self, expected_date: str = "") -> list[dict[str, Any]]:
|
||||
"""Full A-share snapshot via Eastmoney clist, used when Tushare rt_k is unavailable."""
|
||||
now = time.time()
|
||||
cache_key = "assembled:eastmoney_market"
|
||||
with self._response_cache_lock:
|
||||
cached = self._response_cache.get(cache_key)
|
||||
cache_age = now - float((cached or {}).get("created_at") or 0)
|
||||
if cached and cache_age <= min(20, self.response_cache_ttl_seconds):
|
||||
quotes = list(cached.get("payload") or [])
|
||||
return self._filter_quotes_by_date(quotes, expected_date)
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
board_errors: list[str] = []
|
||||
for board in EASTMONEY_A_SHARE_BOARDS:
|
||||
try:
|
||||
rows.extend(self._eastmoney_board_quotes(board))
|
||||
except Exception as exc:
|
||||
board_errors.append(f"{board}:{exc}")
|
||||
quotes = []
|
||||
seen: set[str] = set()
|
||||
for row in rows:
|
||||
quote = _normalize_eastmoney_quote(row)
|
||||
ts_code = str((quote or {}).get("ts_code") or "")
|
||||
if not quote or ts_code in seen:
|
||||
continue
|
||||
seen.add(ts_code)
|
||||
quotes.append(quote)
|
||||
if len(quotes) < 200:
|
||||
detail = f";{'; '.join(board_errors)}" if board_errors else ""
|
||||
raise RealtimeAggregateError(
|
||||
f"Eastmoney market snapshot too small: {len(quotes)}{detail}"
|
||||
)
|
||||
quotes = self._filter_quotes_by_date(quotes, expected_date)
|
||||
with self._response_cache_lock:
|
||||
self._response_cache[cache_key] = {"created_at": now, "payload": quotes}
|
||||
return quotes
|
||||
|
||||
def _eastmoney_board_quotes(self, board: str) -> list[dict[str, Any]]:
|
||||
first = self._eastmoney_market_page(board, 1)
|
||||
data = first.get("data") or {}
|
||||
rows = _diff_rows(data)
|
||||
total = int(_number(data.get("total")))
|
||||
page_count = 1
|
||||
if total > 0:
|
||||
page_count = max(1, (total + EASTMONEY_MARKET_PAGE_SIZE - 1) // EASTMONEY_MARKET_PAGE_SIZE)
|
||||
for page in range(2, min(page_count, 40) + 1):
|
||||
payload = self._eastmoney_market_page(board, page)
|
||||
rows.extend(_diff_rows(payload.get("data") or {}))
|
||||
return rows
|
||||
|
||||
def _eastmoney_market_page(self, board: str, page: int) -> dict[str, Any]:
|
||||
return self._get_json(
|
||||
EASTMONEY_SECTOR_URL,
|
||||
{
|
||||
"pn": str(page),
|
||||
"pz": str(EASTMONEY_MARKET_PAGE_SIZE),
|
||||
"po": "1",
|
||||
"np": "1",
|
||||
"fltt": "2",
|
||||
"invt": "2",
|
||||
"fid": "f12",
|
||||
"fs": board,
|
||||
"fields": EASTMONEY_QUOTE_FIELDS,
|
||||
},
|
||||
referer="https://quote.eastmoney.com/center/gridlist.html",
|
||||
)
|
||||
|
||||
def _filter_quotes_by_date(
|
||||
self,
|
||||
quotes: list[dict[str, Any]],
|
||||
expected_date: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
want = str(expected_date or "").replace("-", "")
|
||||
if not want or not quotes:
|
||||
return quotes
|
||||
dated = [item for item in quotes if str(item.get("quote_date") or "") == want]
|
||||
if dated and len(dated) >= max(100, int(len(quotes) * 0.2)):
|
||||
return dated
|
||||
if dated:
|
||||
return dated
|
||||
if all(not item.get("quote_date") for item in quotes):
|
||||
return quotes
|
||||
raise RealtimeAggregateError(f"Eastmoney quotes are not for {want}")
|
||||
|
||||
def tencent_market_quotes(
|
||||
self,
|
||||
codes: list[str],
|
||||
expected_date: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
quotes = self.tencent_stock_quotes(codes, expected_date="", minimum=200)
|
||||
return self._filter_quotes_by_date(quotes, expected_date)
|
||||
|
||||
def tencent_stock_quotes(
|
||||
self,
|
||||
codes: list[str],
|
||||
expected_date: str = "",
|
||||
minimum: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
symbols: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in codes:
|
||||
ts = str(raw or "").strip().upper()
|
||||
if not ts:
|
||||
continue
|
||||
symbol = ts.split(".")[0]
|
||||
if not symbol.isdigit() or len(symbol) != 6 or symbol in seen:
|
||||
continue
|
||||
seen.add(symbol)
|
||||
if ts.endswith(".SH") or symbol.startswith(("5", "6", "9")):
|
||||
symbols.append(f"sh{symbol}")
|
||||
elif ts.endswith(".BJ") or symbol.startswith(("4", "8")):
|
||||
symbols.append(f"bj{symbol}")
|
||||
else:
|
||||
symbols.append(f"sz{symbol}")
|
||||
if not symbols:
|
||||
raise RealtimeAggregateError("No stock codes available for Tencent quotes")
|
||||
|
||||
quotes: list[dict[str, Any]] = []
|
||||
batch_size = 80
|
||||
|
||||
def load_batch(batch: list[str]) -> list[dict[str, Any]]:
|
||||
raw, _cache_age = self._get_text(
|
||||
f"{TENCENT_QUOTE_URL}{','.join(batch)}",
|
||||
referer="https://gu.qq.com/",
|
||||
encoding="gb18030",
|
||||
)
|
||||
return [
|
||||
quote
|
||||
for line in raw.splitlines()
|
||||
if (quote := _parse_tencent_stock_quote(line))
|
||||
]
|
||||
|
||||
batches = [symbols[index:index + batch_size] for index in range(0, len(symbols), batch_size)]
|
||||
errors: list[str] = []
|
||||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||||
for result in executor.map(self._capture, [lambda batch=batch: load_batch(batch) for batch in batches]):
|
||||
rows, status = result
|
||||
if status.get("ok") and rows:
|
||||
quotes.extend(rows)
|
||||
elif not status.get("ok"):
|
||||
errors.append(str(status.get("error") or "batch failed"))
|
||||
floor = minimum if minimum is not None else max(1, int(len(symbols) * 0.5))
|
||||
if len(quotes) < floor:
|
||||
detail = f";{'; '.join(errors[:3])}" if errors else ""
|
||||
raise RealtimeAggregateError(
|
||||
f"Tencent quotes too small: {len(quotes)}/{len(symbols)}{detail}"
|
||||
)
|
||||
return self._filter_quotes_by_date(quotes, expected_date)
|
||||
|
||||
def tencent_stock_quote(self, code: str, expected_date: str = "") -> dict[str, Any]:
|
||||
symbol, _secid, ts_code = _a_share_identity(code)
|
||||
raw, _cache_age = self._get_text(
|
||||
f"{TENCENT_QUOTE_URL}{symbol}",
|
||||
referer="https://gu.qq.com/",
|
||||
encoding="gb18030",
|
||||
)
|
||||
quote = next(
|
||||
(
|
||||
item
|
||||
for line in raw.splitlines()
|
||||
if (item := _parse_tencent_stock_quote(line))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not quote:
|
||||
raise RealtimeAggregateError(f"Tencent stock quote unavailable for {ts_code}")
|
||||
return _require_quote_date(quote, expected_date)
|
||||
|
||||
def eastmoney_stock_quote(self, code: str, expected_date: str = "") -> dict[str, Any]:
|
||||
_symbol, secid, ts_code = _a_share_identity(code)
|
||||
payload = self._get_json(
|
||||
EASTMONEY_STOCK_URL,
|
||||
{
|
||||
"secid": secid,
|
||||
"invt": "2",
|
||||
"fltt": "2",
|
||||
"fields": EASTMONEY_STOCK_FIELDS,
|
||||
},
|
||||
referer="https://quote.eastmoney.com/",
|
||||
)
|
||||
quote = _normalize_eastmoney_stock_quote(payload.get("data") or {}, ts_code)
|
||||
if not quote:
|
||||
raise RealtimeAggregateError(f"Eastmoney stock quote unavailable for {ts_code}")
|
||||
return _require_quote_date(quote, expected_date)
|
||||
|
||||
def eastmoney_stock_quotes(
|
||||
self,
|
||||
codes: list[str],
|
||||
expected_date: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
secids = []
|
||||
for code in codes:
|
||||
try:
|
||||
_symbol, secid, _ts = _a_share_identity(code)
|
||||
except RealtimeAggregateError:
|
||||
continue
|
||||
secids.append(secid)
|
||||
quotes: list[dict[str, Any]] = []
|
||||
for index in range(0, len(secids), 60):
|
||||
payload = self._get_json(
|
||||
EASTMONEY_INDEX_URL,
|
||||
{
|
||||
"secids": ",".join(secids[index:index + 60]),
|
||||
"fltt": "2",
|
||||
"invt": "2",
|
||||
"fields": EASTMONEY_QUOTE_FIELDS,
|
||||
},
|
||||
referer="https://quote.eastmoney.com/",
|
||||
)
|
||||
for row in _diff_rows(payload.get("data") or {}):
|
||||
quote = _normalize_eastmoney_quote(row)
|
||||
if quote:
|
||||
quotes.append(quote)
|
||||
return self._filter_quotes_by_date(quotes, expected_date)
|
||||
|
||||
def eastmoney_shenwan_quote(
|
||||
self,
|
||||
ts_code: str,
|
||||
expected_date: str = "",
|
||||
) -> dict[str, Any]:
|
||||
code = str(ts_code or "").split(".")[0]
|
||||
if not code:
|
||||
raise RealtimeAggregateError("Invalid Shenwan code")
|
||||
payload = self._get_json(
|
||||
EASTMONEY_INDEX_URL,
|
||||
{
|
||||
"secids": f"90.{code}",
|
||||
"fltt": "2",
|
||||
"invt": "2",
|
||||
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f8,f104,f105,f128,f136,f140,f124",
|
||||
},
|
||||
referer="https://quote.eastmoney.com/",
|
||||
)
|
||||
row = next((item for item in _diff_rows(payload.get("data") or {}) if item), None)
|
||||
if not row:
|
||||
raise RealtimeAggregateError(f"Eastmoney Shenwan quote missing for {code}")
|
||||
epoch = int(_number(row.get("f124")))
|
||||
quote_time = (
|
||||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||
if epoch
|
||||
else ""
|
||||
)
|
||||
close = _number(row.get("f2"))
|
||||
previous = _number(row.get("f18"))
|
||||
if close <= 0 or previous <= 0:
|
||||
raise RealtimeAggregateError(f"Eastmoney Shenwan quote empty for {code}")
|
||||
result = {
|
||||
"ts_code": f"{code}.SI",
|
||||
"code": f"{code}.SI",
|
||||
"name": row.get("f14") or code,
|
||||
"price": close,
|
||||
"close": close,
|
||||
"pre_close": previous,
|
||||
"previous_close": previous,
|
||||
"open": _number(row.get("f17")),
|
||||
"high": _number(row.get("f15")),
|
||||
"low": _number(row.get("f16")),
|
||||
"change": _number(row.get("f3")),
|
||||
"pct_change": _number(row.get("f3")),
|
||||
"amount": _number(row.get("f6")),
|
||||
"leader": row.get("f128") or "--",
|
||||
"leader_code": row.get("f140") or "",
|
||||
"leading_pct": _number(row.get("f136")),
|
||||
"up_count": int(_number(row.get("f104"))),
|
||||
"down_count": int(_number(row.get("f105"))),
|
||||
"quote_time": quote_time,
|
||||
"trade_time": quote_time,
|
||||
"quote_date": datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d") if epoch else "",
|
||||
"quote_time_epoch": epoch,
|
||||
"source": "eastmoney_sw",
|
||||
}
|
||||
return _require_quote_date(result, expected_date) if expected_date else result
|
||||
|
||||
def eastmoney_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]]:
|
||||
day = str(trade_date or "").replace("-", "")
|
||||
rows: list[dict[str, Any]] = []
|
||||
for url, limit_type in (
|
||||
(EASTMONEY_ZT_POOL_URL, "U"),
|
||||
(EASTMONEY_ZB_POOL_URL, "Z"),
|
||||
):
|
||||
try:
|
||||
payload = self._get_json(
|
||||
url,
|
||||
{
|
||||
"ut": "7eea3edcaed734bea9cbfc24409ed989",
|
||||
"dpt": "wz.ztzt",
|
||||
"PageIndex": "0",
|
||||
"PageSize": "200",
|
||||
"sort": "fbt:asc",
|
||||
"date": day,
|
||||
},
|
||||
referer="https://quote.eastmoney.com/ztb/detail",
|
||||
)
|
||||
except RealtimeAggregateError:
|
||||
continue
|
||||
pool = (payload.get("data") or {}).get("pool") or []
|
||||
if isinstance(pool, dict):
|
||||
pool = list(pool.values())
|
||||
for item in pool:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
parsed = _normalize_eastmoney_limit_row(item, limit_type)
|
||||
if parsed:
|
||||
rows.append(parsed)
|
||||
return rows
|
||||
|
||||
def tencent_indices(self) -> list[dict[str, Any]]:
|
||||
raw, cache_age = self._get_text(
|
||||
TENCENT_INDEX_URL,
|
||||
@@ -185,11 +505,17 @@ class WebRealtimeAggregator:
|
||||
if not matched:
|
||||
raise RealtimeAggregateError(f"Eastmoney sector not found: {query}")
|
||||
epoch = int(_number(matched.get("f124")))
|
||||
quote_time = (
|
||||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||
if epoch else ""
|
||||
)
|
||||
return {
|
||||
"code": matched.get("f12") or "",
|
||||
"name": matched.get("f14") or query,
|
||||
"price": _number(matched.get("f2")),
|
||||
"close": _number(matched.get("f2")),
|
||||
"change": _number(matched.get("f3")),
|
||||
"pct_change": _number(matched.get("f3")),
|
||||
"change_amount": _number(matched.get("f4")),
|
||||
"turnover_rate": _number(matched.get("f8")),
|
||||
"up_count": int(_number(matched.get("f104"))),
|
||||
@@ -198,10 +524,9 @@ class WebRealtimeAggregator:
|
||||
"leader_code": matched.get("f140") or "",
|
||||
"leading_pct": _number(matched.get("f136")),
|
||||
"quote_time_epoch": epoch,
|
||||
"quote_time": (
|
||||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||
if epoch else ""
|
||||
),
|
||||
"quote_time": quote_time,
|
||||
"trade_time": quote_time,
|
||||
"quote_date": datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d") if epoch else "",
|
||||
"source": "eastmoney_push2",
|
||||
"match_query": query,
|
||||
}
|
||||
@@ -397,6 +722,198 @@ class WebRealtimeAggregator:
|
||||
) from last_error
|
||||
|
||||
|
||||
def _diff_rows(data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
diff = data.get("diff") or []
|
||||
if isinstance(diff, dict):
|
||||
return [row for row in diff.values() if isinstance(row, dict)]
|
||||
return [row for row in diff if isinstance(row, dict)]
|
||||
|
||||
|
||||
def _a_share_identity(code: str) -> tuple[str, str, str]:
|
||||
raw = str(code or "").strip().upper()
|
||||
symbol = raw.split(".")[0]
|
||||
if not symbol.isdigit() or len(symbol) != 6:
|
||||
raise RealtimeAggregateError("Invalid stock code")
|
||||
if raw.endswith(".SH") or symbol.startswith(("5", "6", "9")):
|
||||
return f"sh{symbol}", f"1.{symbol}", f"{symbol}.SH"
|
||||
if raw.endswith(".BJ") or symbol.startswith(("4", "8")):
|
||||
return f"bj{symbol}", f"0.{symbol}", f"{symbol}.BJ"
|
||||
return f"sz{symbol}", f"0.{symbol}", f"{symbol}.SZ"
|
||||
|
||||
|
||||
def _require_quote_date(quote: dict[str, Any], expected_date: str) -> dict[str, Any]:
|
||||
want = str(expected_date or "").replace("-", "")
|
||||
got = str(quote.get("quote_date") or "")
|
||||
if want and got != want:
|
||||
raise RealtimeAggregateError(f"quote date {got or 'empty'} is not {want}")
|
||||
return quote
|
||||
|
||||
|
||||
def _normalize_eastmoney_stock_quote(
|
||||
row: dict[str, Any], ts_code: str
|
||||
) -> dict[str, Any] | None:
|
||||
close = _number(row.get("f43"))
|
||||
previous_close = _number(row.get("f60"))
|
||||
if close <= 0 or previous_close <= 0:
|
||||
return None
|
||||
epoch = int(_number(row.get("f86")))
|
||||
quote_date = ""
|
||||
if epoch > 0:
|
||||
quote_date = datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d")
|
||||
return {
|
||||
"ts_code": ts_code,
|
||||
"name": row.get("f58") or ts_code.split(".")[0],
|
||||
"pre_close": previous_close,
|
||||
"open": _number(row.get("f46")),
|
||||
"high": _number(row.get("f44")),
|
||||
"low": _number(row.get("f45")),
|
||||
"close": close,
|
||||
"vol": _number(row.get("f47")) * 100,
|
||||
"amount": _number(row.get("f48")),
|
||||
"num": 0,
|
||||
"quote_date": quote_date,
|
||||
"quote_time_epoch": epoch,
|
||||
"turnover_rate": _number(row.get("f168")),
|
||||
"net_mf_amount": _eastmoney_flow_wan(row.get("f62")),
|
||||
"large_amount": _eastmoney_flow_wan(row.get("f62")),
|
||||
"medium_amount": _eastmoney_flow_wan(row.get("f78")),
|
||||
"small_amount": _eastmoney_flow_wan(row.get("f84")),
|
||||
"source": "eastmoney_stock",
|
||||
}
|
||||
|
||||
|
||||
def _parse_tencent_stock_quote(line: str) -> dict[str, Any] | None:
|
||||
if '="' not in line:
|
||||
return None
|
||||
prefix, payload = line.split('="', 1)
|
||||
fields = payload.rsplit('";', 1)[0].split("~")
|
||||
if len(fields) < 38:
|
||||
return None
|
||||
symbol = fields[2]
|
||||
if not symbol.isdigit() or len(symbol) != 6:
|
||||
return None
|
||||
close = _number(fields[3])
|
||||
previous_close = _number(fields[4])
|
||||
if close <= 0 or previous_close <= 0:
|
||||
return None
|
||||
marker = prefix.lower()
|
||||
if "sh" in marker:
|
||||
ts_code = f"{symbol}.SH"
|
||||
elif "bj" in marker:
|
||||
ts_code = f"{symbol}.BJ"
|
||||
else:
|
||||
ts_code = f"{symbol}.SZ"
|
||||
try:
|
||||
quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S")
|
||||
quote_date = quote_time.strftime("%Y%m%d")
|
||||
epoch = int(quote_time.timestamp())
|
||||
except ValueError:
|
||||
quote_date = ""
|
||||
epoch = 0
|
||||
return {
|
||||
"ts_code": ts_code,
|
||||
"name": fields[1] or symbol,
|
||||
"pre_close": previous_close,
|
||||
"open": _number(fields[5]),
|
||||
"high": _number(fields[33]),
|
||||
"low": _number(fields[34]),
|
||||
"close": close,
|
||||
"vol": _number(fields[6]) * 100,
|
||||
"amount": _number(fields[37]) * 10000,
|
||||
"num": 0,
|
||||
"quote_date": quote_date,
|
||||
"quote_time_epoch": epoch,
|
||||
"source": "tencent_qt",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_eastmoney_quote(row: dict[str, Any]) -> dict[str, Any] | None:
|
||||
symbol = str(row.get("f12") or "").strip()
|
||||
if not symbol.isdigit() or len(symbol) != 6:
|
||||
return None
|
||||
close = _number(row.get("f2"))
|
||||
previous_close = _number(row.get("f18"))
|
||||
if close <= 0 or previous_close <= 0:
|
||||
return None
|
||||
market = int(_number(row.get("f13")))
|
||||
if market == 1 or symbol.startswith(("5", "6", "9")):
|
||||
ts_code = f"{symbol}.SH"
|
||||
elif symbol.startswith(("4", "8")):
|
||||
ts_code = f"{symbol}.BJ"
|
||||
else:
|
||||
ts_code = f"{symbol}.SZ"
|
||||
epoch = int(_number(row.get("f124")))
|
||||
quote_date = ""
|
||||
if epoch > 0:
|
||||
quote_date = datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d")
|
||||
return {
|
||||
"ts_code": ts_code,
|
||||
"name": row.get("f14") or symbol,
|
||||
"pre_close": previous_close,
|
||||
"open": _number(row.get("f17")),
|
||||
"high": _number(row.get("f15")),
|
||||
"low": _number(row.get("f16")),
|
||||
"close": close,
|
||||
"vol": _number(row.get("f5")) * 100,
|
||||
"amount": _number(row.get("f6")),
|
||||
"num": 0,
|
||||
"quote_date": quote_date,
|
||||
"quote_time_epoch": epoch,
|
||||
"source": "eastmoney_clist",
|
||||
}
|
||||
|
||||
|
||||
def _eastmoney_flow_wan(value: Any) -> float | None:
|
||||
if value in (None, "", "-"):
|
||||
return None
|
||||
amount = _number(value, default=float("nan"))
|
||||
if amount != amount:
|
||||
return None
|
||||
return amount / 10000
|
||||
|
||||
|
||||
def _board_clock(value: Any) -> str:
|
||||
digits = "".join(character for character in str(value or "") if character.isdigit())
|
||||
if len(digits) >= 6:
|
||||
return f"{digits[:2]}:{digits[2:4]}:{digits[4:6]}"
|
||||
if len(digits) == 5:
|
||||
digits = digits.zfill(6)
|
||||
return f"{digits[:2]}:{digits[2:4]}:{digits[4:6]}"
|
||||
if len(digits) == 4:
|
||||
return f"{digits[:2]}:{digits[2:]}:00"
|
||||
return ""
|
||||
|
||||
|
||||
def _normalize_eastmoney_limit_row(row: dict[str, Any], limit_type: str) -> dict[str, Any] | None:
|
||||
symbol = str(row.get("c") or row.get("code") or "").strip()
|
||||
if not symbol.isdigit() or len(symbol) != 6:
|
||||
return None
|
||||
market = int(_number(row.get("m") if row.get("m") not in (None, "") else row.get("market")))
|
||||
if market == 1 or symbol.startswith(("5", "6", "9")):
|
||||
ts_code = f"{symbol}.SH"
|
||||
elif symbol.startswith(("4", "8")):
|
||||
ts_code = f"{symbol}.BJ"
|
||||
else:
|
||||
ts_code = f"{symbol}.SZ"
|
||||
first_time = _board_clock(row.get("fbt") if row.get("fbt") not in (None, "") else row.get("first_time"))
|
||||
last_time = _board_clock(row.get("lbt") if row.get("lbt") not in (None, "") else row.get("last_time"))
|
||||
fund = row.get("fund")
|
||||
if fund in (None, ""):
|
||||
fund = row.get("fd_amount")
|
||||
return {
|
||||
"ts_code": ts_code,
|
||||
"name": row.get("n") or row.get("name") or symbol,
|
||||
"limit_type": limit_type,
|
||||
"first_time": first_time or None,
|
||||
"last_time": last_time or None,
|
||||
"open_times": int(_number(row.get("zbc") if row.get("zbc") not in (None, "") else row.get("open_times"))),
|
||||
"limit_times": max(1, int(_number(row.get("lbc") if row.get("lbc") not in (None, "") else 1))),
|
||||
"turnover_ratio": _number(row.get("hs") if row.get("hs") not in (None, "") else row.get("turnover_ratio")),
|
||||
"fd_amount": _number(fund) if fund not in (None, "", "-") else None,
|
||||
"source": "eastmoney_zt_pool",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_sector(value: Any) -> str:
|
||||
text = str(value or "").strip().replace(" ", "")
|
||||
for suffix in ("板块", "概念", "行业", "Ⅱ", "Ⅲ", "(A股)", "(A股)"):
|
||||
|
||||
@@ -802,7 +802,11 @@ def _market_line_scores(
|
||||
{},
|
||||
)
|
||||
sector_quantitative_mode = str(sector.get("_quantitative_mode") or "")
|
||||
actual_sector_source = str(sector.get("source") or "").startswith("tushare_")
|
||||
sector_source = str(sector.get("source") or "")
|
||||
actual_sector_source = (
|
||||
sector_source.startswith("tushare_")
|
||||
or "sw_member_equal_" in sector_source
|
||||
)
|
||||
if (sector.get("realtime") and actual_sector_source) or sector_quantitative_mode == "intraday":
|
||||
sector_change = float(sector.get("change") or 0)
|
||||
sector_change_score = _clamp(sector_change / 5, -1, 1)
|
||||
@@ -831,8 +835,13 @@ def _market_line_scores(
|
||||
f"成分上涨 {int(sector_up)} 家、下跌 {int(sector_down)} 家",
|
||||
f"平均换手 {float(sector.get('turnover_rate') or 0):.2f}%,相对市场 {float(sector.get('relative_turnover') or 0):.2f} 倍",
|
||||
]
|
||||
sector_change_label = (
|
||||
"申万二级行业成分等权涨跌"
|
||||
if "sw_member_equal_" in sector_source
|
||||
else "申万二级行业官方涨跌"
|
||||
)
|
||||
sector_outer_evidence = [
|
||||
f"申万二级行业官方涨跌 {sector_change:+.2f}%",
|
||||
f"{sector_change_label} {sector_change:+.2f}%",
|
||||
f"领涨 {sector.get('leader') or '--'} {float(sector.get('leading_pct') or 0):+.2f}%",
|
||||
]
|
||||
elif actual_sector_source or sector_quantitative_mode == "historical":
|
||||
|
||||
@@ -28,7 +28,7 @@ class HeavenManualMixin:
|
||||
"sector_coverage": {"line": 3, "label": "成分行情覆盖率", "unit": "%", "min": 0, "max": 100},
|
||||
"sector_relative_turnover": {"line": 3, "label": "行业相对市场换手", "unit": "倍", "min": 0, "max": 20},
|
||||
"sector_member_equal_change": {"line": 3, "label": "成分等权涨跌幅", "unit": "%", "min": -100, "max": 100},
|
||||
"sector_change": {"line": 4, "label": "申万官方涨跌幅", "unit": "%", "min": -100, "max": 100},
|
||||
"sector_change": {"line": 4, "label": "申万行业涨跌幅", "unit": "%", "min": -100, "max": 100},
|
||||
"sector_leading_pct": {"line": [3, 4], "label": "行业领涨股涨跌幅", "unit": "%", "min": -100, "max": 100},
|
||||
"market_sentiment_score": {"line": 5, "label": "市场情绪温度", "unit": "分", "min": 0, "max": 100},
|
||||
"market_seal_rate": {"line": 5, "label": "封板率", "unit": "%", "min": 0, "max": 100},
|
||||
|
||||
@@ -202,8 +202,7 @@ class HeavenMarketContextMixin:
|
||||
}
|
||||
if len(quotes) != 3 or quote_dates != {trade_date}:
|
||||
raise ValueError("腾讯三大指数日期与目标交易日不一致")
|
||||
now = datetime.now().astimezone()
|
||||
max_skew = 120 if now.hour >= 15 else 15
|
||||
max_skew = 120
|
||||
if max(epochs) - min(epochs) > max_skew:
|
||||
raise ValueError(f"腾讯三大指数时间差超过{max_skew}秒")
|
||||
|
||||
@@ -217,11 +216,14 @@ class HeavenMarketContextMixin:
|
||||
start_date = (
|
||||
datetime.strptime(trade_date, "%Y%m%d") - timedelta(days=20)
|
||||
).strftime("%Y%m%d")
|
||||
history_end = (
|
||||
datetime.strptime(trade_date, "%Y%m%d") - timedelta(days=1)
|
||||
).strftime("%Y%m%d")
|
||||
for quote in quotes:
|
||||
ts_code = code_map[str(quote.get("code") or "")]
|
||||
history = client.query(
|
||||
"index_daily",
|
||||
{"ts_code": ts_code, "start_date": start_date, "end_date": trade_date},
|
||||
{"ts_code": ts_code, "start_date": start_date, "end_date": history_end},
|
||||
"ts_code,trade_date,close,pct_chg",
|
||||
)
|
||||
history.sort(key=lambda item: str(item.get("trade_date") or ""))
|
||||
@@ -283,9 +285,9 @@ class HeavenMarketContextMixin:
|
||||
) -> dict[str, Any] | None:
|
||||
"""Return the Shenwan L2 sector context for heaven trend.
|
||||
|
||||
观势行业层只使用申万二级行业。外显盘中使用 rt_sw_k、历史使用
|
||||
sw_daily;内核独立使用目标日期成分股行情聚合。收盘过渡期在
|
||||
sw_daily 入库前接受同日15:00后的 rt_sw_k 收盘快照。
|
||||
观势行业层只使用申万二级行业。外显优先使用已发布的 sw_daily,
|
||||
盘中及收盘过渡期使用免费申万实时行情;内核使用数据中枢或免费
|
||||
实时成分行情。不再调用无权限的 rt_sw_k / rt_k。
|
||||
"""
|
||||
cache_key = f"{trade_date}:{identifier.strip().lower()}"
|
||||
cached = self.database.get_data_snapshot("heaven_sector", cache_key)
|
||||
@@ -299,6 +301,14 @@ class HeavenMarketContextMixin:
|
||||
and not cached.get("realtime")
|
||||
and int(cached.get("schema_version") or 0) >= 6
|
||||
)
|
||||
cached_quotes = int((cached or {}).get("quote_count") or 0)
|
||||
cached_lkg = bool(
|
||||
cached
|
||||
and cached_date == trade_date
|
||||
and cached.get("taxonomy") == "sw_l2"
|
||||
and cached_quotes > 0
|
||||
and int(cached.get("schema_version") or 0) >= 6
|
||||
)
|
||||
if market_mode != "intraday" and cached_valid:
|
||||
return cached
|
||||
if not self.configured:
|
||||
@@ -311,8 +321,12 @@ class HeavenMarketContextMixin:
|
||||
allow_realtime_close=market_mode == "closed",
|
||||
)
|
||||
except TushareError as exc:
|
||||
if cached_valid:
|
||||
return cached
|
||||
if cached_lkg:
|
||||
delayed = dict(cached)
|
||||
delayed["delayed"] = True
|
||||
delayed["delay_notice"] = "主备免费行情均暂不可用,显示最近一次真实快照"
|
||||
delayed["realtime"] = market_mode == "intraday"
|
||||
return delayed
|
||||
return {
|
||||
"name": "",
|
||||
"code": "",
|
||||
@@ -323,12 +337,16 @@ class HeavenMarketContextMixin:
|
||||
"precise": False,
|
||||
"inner_precise": False,
|
||||
"outer_precise": False,
|
||||
"coverage": 0,
|
||||
"member_count": 0,
|
||||
"quote_count": 0,
|
||||
"error": f"申万二级行业数据获取失败:{exc}",
|
||||
}
|
||||
if not payload.get("realtime") and payload.get("precise"):
|
||||
if int(payload.get("quote_count") or 0) > 0:
|
||||
self.database.save_data_snapshot(
|
||||
"heaven_sector",
|
||||
cache_key,
|
||||
str(payload.get("source") or "tushare"),
|
||||
payload,
|
||||
)
|
||||
elif not payload.get("realtime") and payload.get("precise"):
|
||||
self.database.save_data_snapshot(
|
||||
"heaven_sector",
|
||||
cache_key,
|
||||
|
||||
@@ -243,6 +243,7 @@ class HeavenTrendMixin:
|
||||
"detail": (
|
||||
f"申万二级 {sector.get('name') or '--'} {sector.get('code') or '--'} "
|
||||
f"成分覆盖 {int(sector.get('quote_count') or 0)}/{int(sector.get('member_count') or 0)}"
|
||||
+ (";延迟快照" if sector.get("delayed") or sector.get("delay_notice") else "")
|
||||
),
|
||||
},
|
||||
{
|
||||
@@ -341,7 +342,9 @@ class HeavenTrendMixin:
|
||||
issues.append("行业外显缺少申万官方行情")
|
||||
if sector and sector_coverage_issue:
|
||||
issues.append(sector_coverage_issue)
|
||||
if sector.get("realtime") and not sector.get("relative_turnover"):
|
||||
if sector.get("delay_notice"):
|
||||
issues.append(str(sector.get("delay_notice")))
|
||||
if sector.get("realtime") and not sector.get("relative_turnover") and not sector.get("delayed"):
|
||||
issues.append("行业内核缺少相对全市场换手活跃度")
|
||||
|
||||
stock = stock or {}
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -13,14 +14,20 @@ from threading import Lock
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from backend.bootstrap.config import tushare_code as _stock_market_code
|
||||
from backend.data.providers.ifind_client import IfindError, IfindHttpClient
|
||||
from backend.data.providers.ifind_client import IfindError
|
||||
|
||||
LOGGER = logging.getLogger("xiaobai.charts")
|
||||
|
||||
|
||||
class ChartDataError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
DAILY_CHART_LIMIT = 45
|
||||
|
||||
|
||||
TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get"
|
||||
HIS_TRENDS_URL = "https://push2his.eastmoney.com/api/qt/stock/trends2/get"
|
||||
BOARD_LIST_URL = "https://push2delay.eastmoney.com/api/qt/clist/get"
|
||||
BROWSER_USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
@@ -35,55 +42,186 @@ INDEX_SECIDS = {
|
||||
|
||||
|
||||
class MarketChartClient:
|
||||
"""Prefer iFinD for display charts and retain Eastmoney as a last resort."""
|
||||
"""Display charts are served by the data hub only."""
|
||||
|
||||
def __init__(self, ifind: IfindHttpClient, fallback: "EastmoneyChartClient") -> None:
|
||||
self.ifind = ifind
|
||||
self.fallback = fallback
|
||||
def __init__(self, datahub: Any = None) -> None:
|
||||
self.datahub = datahub
|
||||
self.ifind = None
|
||||
self.fallback = None
|
||||
|
||||
def stock_intraday(self, code: str) -> dict[str, Any]:
|
||||
normalized = str(code or "").strip()
|
||||
if not re.fullmatch(r"\d{6}", normalized):
|
||||
raise ChartDataError("Invalid stock code")
|
||||
ifind_code = _stock_market_code(normalized)
|
||||
try:
|
||||
return self._ifind_intraday(ifind_code, "stock", normalized)
|
||||
except (IfindError, ChartDataError):
|
||||
return self.fallback.stock_intraday(normalized)
|
||||
hub_chart = self._datahub_intraday(normalized)
|
||||
if hub_chart is not None:
|
||||
return hub_chart
|
||||
raise ChartDataError("分时图数据中枢暂不可用")
|
||||
|
||||
def stock_daily(self, code: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
|
||||
def stock_daily(self, code: str, end_date: str, limit: int = DAILY_CHART_LIMIT) -> list[dict[str, Any]]:
|
||||
normalized = str(code or "").strip()
|
||||
if not re.fullmatch(r"\d{6}", normalized):
|
||||
raise ChartDataError("Invalid stock code")
|
||||
return self._ifind_daily(_stock_market_code(normalized), end_date, limit)
|
||||
hub_rows = self._datahub_daily(normalized, end_date, limit, "daily")
|
||||
if hub_rows:
|
||||
return hub_rows
|
||||
raise ChartDataError("日K数据中枢暂不可用")
|
||||
|
||||
def index_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
|
||||
def index_daily(self, identifier: str, end_date: str, limit: int = DAILY_CHART_LIMIT) -> list[dict[str, Any]]:
|
||||
normalized = str(identifier or "").strip().upper()
|
||||
if normalized not in INDEX_SECIDS:
|
||||
raise ChartDataError("Unsupported index")
|
||||
return self._ifind_daily(normalized, end_date, limit)
|
||||
hub_rows = self._datahub_daily(normalized, end_date, limit, "index_daily")
|
||||
if hub_rows:
|
||||
return hub_rows
|
||||
raise ChartDataError("指数日K数据中枢暂不可用")
|
||||
|
||||
def board_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
|
||||
normalized = str(identifier or "").strip().upper()
|
||||
if not normalized:
|
||||
raise ChartDataError("Invalid board code")
|
||||
return self._ifind_daily(normalized, end_date, limit)
|
||||
hub_rows = self._datahub_daily(normalized, end_date, limit, "sector_daily")
|
||||
if hub_rows:
|
||||
return hub_rows
|
||||
raise ChartDataError("板块日K数据中枢暂不可用")
|
||||
|
||||
def index_intraday(self, identifier: str) -> dict[str, Any]:
|
||||
normalized = str(identifier or "").strip().upper()
|
||||
if normalized not in INDEX_SECIDS:
|
||||
raise ChartDataError("Unsupported index")
|
||||
hub_chart = self._datahub_intraday(normalized)
|
||||
if hub_chart is not None:
|
||||
return hub_chart
|
||||
raise ChartDataError("指数分时数据中枢暂不可用")
|
||||
|
||||
def _datahub_intraday(self, code: str) -> dict[str, Any] | None:
|
||||
if self.datahub is None:
|
||||
return None
|
||||
try:
|
||||
return self._ifind_intraday(normalized, "index", normalized)
|
||||
except (IfindError, ChartDataError):
|
||||
return self.fallback.index_intraday(normalized)
|
||||
chart = self.datahub.try_intraday(code)
|
||||
except Exception as exc:
|
||||
LOGGER.warning("datahub intraday unexpected error: %s", exc)
|
||||
return None
|
||||
if not chart:
|
||||
return None
|
||||
points = list(chart.get("points") or [])
|
||||
if not points:
|
||||
return None
|
||||
return chart
|
||||
|
||||
def _datahub_daily(
|
||||
self,
|
||||
code: str,
|
||||
end_date: str,
|
||||
limit: int,
|
||||
dataset: str,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
if self.datahub is None or not hasattr(self.datahub, "try_daily_chart"):
|
||||
return None
|
||||
try:
|
||||
rows = self.datahub.try_daily_chart(code, end_date, limit, dataset)
|
||||
except Exception as exc:
|
||||
LOGGER.warning("datahub daily unexpected error: %s", exc)
|
||||
rows = None
|
||||
if not rows:
|
||||
return None
|
||||
compact_end = str(end_date or "").replace("-", "")
|
||||
market_now = datetime.now().astimezone()
|
||||
today = market_now.strftime("%Y%m%d")
|
||||
market_open = (
|
||||
market_now.weekday() < 5
|
||||
and market_now.time().replace(tzinfo=None) >= dt_time(9, 30)
|
||||
)
|
||||
if compact_end == today and market_open:
|
||||
overlay = self._datahub_today_bar(code, dataset, rows)
|
||||
if overlay:
|
||||
if rows and rows[-1]["trade_date"] == overlay["trade_date"]:
|
||||
rows[-1] = overlay
|
||||
else:
|
||||
rows.append(overlay)
|
||||
return rows
|
||||
|
||||
def _datahub_today_bar(
|
||||
self,
|
||||
code: str,
|
||||
dataset: str,
|
||||
history: list[dict[str, Any]],
|
||||
) -> dict[str, Any] | None:
|
||||
today_display = datetime.now().astimezone().date().isoformat()
|
||||
previous = history[-1]["close"] if history and history[-1]["trade_date"] != today_display else (
|
||||
history[-2]["close"] if len(history) >= 2 else 0.0
|
||||
)
|
||||
quote = None
|
||||
if dataset == "index_daily" and hasattr(self.datahub, "try_index_quotes"):
|
||||
quotes = self.datahub.try_index_quotes() or []
|
||||
quote = next(
|
||||
(
|
||||
item for item in quotes
|
||||
if str(item.get("ts_code") or "") == code or str(item.get("code") or "") == code.split(".")[0]
|
||||
),
|
||||
None,
|
||||
)
|
||||
elif hasattr(self.datahub, "try_quotes"):
|
||||
quotes = self.datahub.try_quotes([code]) or []
|
||||
quote = quotes[0] if quotes else None
|
||||
if quote:
|
||||
close = _number(quote.get("close") if quote.get("close") not in (None, "") else quote.get("price"))
|
||||
open_price = _number(quote.get("open"))
|
||||
high = _number(quote.get("high"))
|
||||
low = _number(quote.get("low"))
|
||||
previous_close = _number(
|
||||
quote.get("pre_close") if quote.get("pre_close") not in (None, "") else quote.get("previous_close")
|
||||
) or previous
|
||||
volume = _number(quote.get("vol") if quote.get("vol") not in (None, "") else quote.get("volume"))
|
||||
amount = _number(quote.get("amount"))
|
||||
if close > 0 and open_price > 0:
|
||||
return {
|
||||
"trade_date": today_display,
|
||||
"open": open_price,
|
||||
"high": high or close,
|
||||
"low": low or close,
|
||||
"close": close,
|
||||
"change": round((close / previous_close - 1) * 100, 4) if previous_close else 0.0,
|
||||
"volume": volume,
|
||||
"amount_billion": amount / 100_000_000,
|
||||
"realtime": True,
|
||||
}
|
||||
chart = self._datahub_intraday(code)
|
||||
points = list((chart or {}).get("points") or [])
|
||||
if not points:
|
||||
return None
|
||||
closes = [_number(point.get("close")) for point in points if _number(point.get("close")) > 0]
|
||||
if not closes:
|
||||
return None
|
||||
opens = [_number(point.get("open")) for point in points if _number(point.get("open")) > 0]
|
||||
highs = [_number(point.get("high")) for point in points if _number(point.get("high")) > 0]
|
||||
lows = [_number(point.get("low")) for point in points if _number(point.get("low")) > 0]
|
||||
volume = sum(_number(point.get("volume")) for point in points)
|
||||
amount = sum(_number(point.get("amount")) for point in points)
|
||||
previous_close = _number((chart or {}).get("previous_close")) or previous
|
||||
close = closes[-1]
|
||||
open_price = opens[0] if opens else closes[0]
|
||||
return {
|
||||
"trade_date": today_display,
|
||||
"open": open_price,
|
||||
"high": max(highs or closes),
|
||||
"low": min(lows or closes),
|
||||
"close": close,
|
||||
"change": round((close / previous_close - 1) * 100, 4) if previous_close else 0.0,
|
||||
"volume": volume,
|
||||
"amount_billion": amount / 100_000_000,
|
||||
"realtime": True,
|
||||
}
|
||||
|
||||
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
|
||||
normalized = str(identifier or "").strip().upper()
|
||||
try:
|
||||
return self._ifind_intraday(normalized, "board", normalized, name)
|
||||
except (IfindError, ChartDataError):
|
||||
return self.fallback.board_intraday(normalized, name)
|
||||
hub_chart = self._datahub_intraday(normalized)
|
||||
if hub_chart is not None:
|
||||
if name:
|
||||
hub_chart = dict(hub_chart)
|
||||
hub_chart["name"] = name
|
||||
return hub_chart
|
||||
raise ChartDataError("板块分时数据中枢暂不可用")
|
||||
|
||||
def _ifind_intraday(
|
||||
self,
|
||||
@@ -92,7 +230,7 @@ class MarketChartClient:
|
||||
identifier: str,
|
||||
name: str = "",
|
||||
) -> dict[str, Any]:
|
||||
if not self.ifind.configured:
|
||||
if not self.ifind or not self.ifind.configured:
|
||||
raise ChartDataError("iFinD is not configured")
|
||||
now = datetime.now().astimezone()
|
||||
rows: list[dict[str, Any]] = []
|
||||
@@ -129,7 +267,7 @@ class MarketChartClient:
|
||||
def _ifind_daily(
|
||||
self, ifind_code: str, end_date: str, limit: int
|
||||
) -> list[dict[str, Any]]:
|
||||
if not self.ifind.configured:
|
||||
if not self.ifind or not self.ifind.configured:
|
||||
raise ChartDataError("iFinD is not configured")
|
||||
compact_end = str(end_date or "").replace("-", "")
|
||||
if not re.fullmatch(r"\d{8}", compact_end):
|
||||
@@ -231,9 +369,11 @@ class MarketChartClient:
|
||||
pass
|
||||
if not normalized:
|
||||
raise ChartDataError("No iFinD daily chart data returned")
|
||||
return normalized[-max(20, min(180, int(limit))):]
|
||||
return normalized[-max(1, int(limit)):]
|
||||
|
||||
def _previous_close(self, code: str, trade_date: str, fallback: float) -> float:
|
||||
if not self.ifind:
|
||||
return fallback
|
||||
today = datetime.now().astimezone().date().isoformat()
|
||||
if trade_date == today:
|
||||
try:
|
||||
@@ -305,21 +445,29 @@ class EastmoneyChartClient:
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
payload = self._request_json(
|
||||
TRENDS_URL,
|
||||
{
|
||||
params = {
|
||||
"secid": secid,
|
||||
"fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13",
|
||||
"fields2": "f51,f52,f53,f54,f55,f56,f57,f58",
|
||||
"iscr": "0",
|
||||
"ndays": "1",
|
||||
},
|
||||
"https://quote.eastmoney.com/",
|
||||
)
|
||||
}
|
||||
last_error: Exception | None = None
|
||||
data: dict[str, Any] = {}
|
||||
points: list[dict[str, Any]] = []
|
||||
for url, ndays in ((TRENDS_URL, "1"), (TRENDS_URL, "5"), (HIS_TRENDS_URL, "5")):
|
||||
request_params = {**params, "ndays": ndays}
|
||||
try:
|
||||
payload = self._request_json(url, request_params, "https://quote.eastmoney.com/")
|
||||
except ChartDataError as exc:
|
||||
last_error = exc
|
||||
continue
|
||||
data = payload.get("data") or {}
|
||||
points = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))]
|
||||
parsed = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))]
|
||||
points = _latest_session(parsed)
|
||||
if points:
|
||||
break
|
||||
if not points:
|
||||
raise ChartDataError("No intraday chart data returned")
|
||||
raise ChartDataError("No intraday chart data returned") from last_error
|
||||
|
||||
result = {
|
||||
"entity_type": entity_type,
|
||||
@@ -433,6 +581,15 @@ class EastmoneyChartClient:
|
||||
raise ChartDataError("Intraday chart request failed") from last_error
|
||||
|
||||
|
||||
def _latest_session(points: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
if not points:
|
||||
return []
|
||||
latest = max(str(point.get("date") or "") for point in points)
|
||||
if not latest:
|
||||
return points
|
||||
return [point for point in points if str(point.get("date") or "") == latest]
|
||||
|
||||
|
||||
def _parse_trend(raw: Any) -> dict[str, Any] | None:
|
||||
fields = str(raw or "").split(",")
|
||||
if len(fields) < 8 or " " not in fields[0]:
|
||||
|
||||
@@ -14,7 +14,9 @@ from backend.bootstrap.config import (
|
||||
validate_text,
|
||||
)
|
||||
from backend.data.providers.ifind_client import IfindError
|
||||
from backend.data.providers.tushare_client import TushareClient, TushareError
|
||||
from backend.data.providers.tushare_client import TushareError
|
||||
from backend.data.providers.tushare_helpers import _moneyflow_payload, _optional_number
|
||||
from backend.data.realtime import RealtimeAggregateError
|
||||
from backend.features.market.backfill_history import (
|
||||
DEFAULT_RECENT_TRADING_DAYS,
|
||||
MAX_RANGE_TRADING_DAYS,
|
||||
@@ -26,7 +28,7 @@ from backend.features.market.backfill_history import (
|
||||
select_open_trade_dates,
|
||||
select_open_trade_dates_in_range,
|
||||
)
|
||||
from backend.features.market.charts import ChartDataError
|
||||
from backend.features.market.charts import ChartDataError, DAILY_CHART_LIMIT
|
||||
from backend.features.market.insights import MarketInsightsService
|
||||
from backend.features.sentiment.engine import SENTIMENT_ENGINE_VERSION
|
||||
|
||||
@@ -42,6 +44,7 @@ SEARCH_TYPE_LABELS = {
|
||||
"theme": "题材",
|
||||
"index": "指数",
|
||||
}
|
||||
TODAY_DAILY_UNAVAILABLE_NOTICE = "今日日K暂不可用,仍显示最近收盘K线。"
|
||||
THS_SEARCH_TYPES = {
|
||||
"I": ("sector", "行业板块"),
|
||||
"R": ("sector", "地域板块"),
|
||||
@@ -58,16 +61,40 @@ class MarketServiceMixin:
|
||||
self._tushare_client(),
|
||||
ifind=self.ifind,
|
||||
)
|
||||
def _tushare_client(self) -> TushareClient:
|
||||
def _tushare_client(self) -> Any:
|
||||
override = getattr(self, "_market_client_override", None)
|
||||
if override is not None:
|
||||
return override
|
||||
gateway = getattr(self, "data_gateway", None)
|
||||
if gateway is not None:
|
||||
if gateway is None:
|
||||
raise RuntimeError("数据中枢尚未装配。")
|
||||
return gateway.tushare()
|
||||
# Compatibility for isolated legacy unit-test service stubs.
|
||||
return TushareClient(self.token)
|
||||
|
||||
def _now(self) -> datetime:
|
||||
clock = getattr(self, "clock", None)
|
||||
if callable(clock):
|
||||
return clock()
|
||||
return datetime.now().astimezone()
|
||||
|
||||
def _is_requested_open_session(self, requested_date: str) -> bool:
|
||||
now = self._now()
|
||||
if requested_date != now.strftime("%Y%m%d"):
|
||||
return False
|
||||
if now.time().replace(tzinfo=None) < dt_time(9, 15):
|
||||
return False
|
||||
client = self._tushare_client() if self.configured else None
|
||||
resolve = getattr(client, "resolve_trade_context", None) if client else None
|
||||
if resolve is None:
|
||||
return now.weekday() < 5
|
||||
try:
|
||||
trade_date, _ = resolve(requested_date)
|
||||
except Exception:
|
||||
return now.weekday() < 5
|
||||
return str(trade_date or "") == requested_date
|
||||
|
||||
def get_dashboard(self, trade_date: str, force: bool = False) -> dict[str, Any]:
|
||||
normalized_date = normalize_date(trade_date)
|
||||
now = datetime.now().astimezone()
|
||||
now = self._now()
|
||||
if (
|
||||
normalized_date == now.strftime("%Y%m%d")
|
||||
and now.time().replace(tzinfo=None) < datetime.strptime("09:15", "%H:%M").time()
|
||||
@@ -149,11 +176,10 @@ class MarketServiceMixin:
|
||||
return "最近可用交易日"
|
||||
return f"{int(compact[4:6])} 月 {int(compact[6:8])} 日"
|
||||
|
||||
@classmethod
|
||||
def _preparing_display_notice(cls, actual_date: str, requested_date: str) -> str:
|
||||
shown = cls._chinese_month_day(actual_date)
|
||||
def _preparing_display_notice(self, actual_date: str, requested_date: str) -> str:
|
||||
shown = self._chinese_month_day(actual_date)
|
||||
requested = str(requested_date or "").replace("-", "")
|
||||
if requested == date.today().strftime("%Y%m%d"):
|
||||
if requested == self._now().strftime("%Y%m%d"):
|
||||
return f"今日数据正在准备,当前展示 {shown}"
|
||||
return f"所选日期数据尚未到齐,当前展示 {shown}"
|
||||
|
||||
@@ -174,14 +200,14 @@ class MarketServiceMixin:
|
||||
def _should_retry_incomplete_snapshot(
|
||||
self, snapshot: dict[str, Any], requested_date: str
|
||||
) -> bool:
|
||||
if requested_date != date.today().strftime("%Y%m%d"):
|
||||
if requested_date != self._now().strftime("%Y%m%d"):
|
||||
return False
|
||||
meta = snapshot.get("meta") or {}
|
||||
incomplete = (
|
||||
meta.get("limit_data_source") == "derived"
|
||||
or bool(meta.get("carried_forward"))
|
||||
or str(meta.get("trade_date") or "").replace("-", "") != requested_date
|
||||
)
|
||||
actual = str(meta.get("trade_date") or "").replace("-", "")
|
||||
stale_carry = bool(meta.get("carried_forward") or actual != requested_date)
|
||||
if stale_carry and self._is_requested_open_session(requested_date):
|
||||
return True
|
||||
incomplete = meta.get("limit_data_source") == "derived" or stale_carry
|
||||
return incomplete and self._snapshot_age_seconds(meta) >= 60
|
||||
|
||||
def _annotate_data_status(self, dashboard: dict[str, Any]) -> dict[str, Any]:
|
||||
@@ -199,6 +225,9 @@ class MarketServiceMixin:
|
||||
else:
|
||||
meta["data_status"] = "preparing"
|
||||
meta["display_notice"] = self._preparing_display_notice(actual, requested)
|
||||
elif meta.get("realtime"):
|
||||
meta["data_status"] = "intraday"
|
||||
meta.setdefault("display_notice", "")
|
||||
else:
|
||||
meta["data_status"] = "official"
|
||||
meta.setdefault("display_notice", "")
|
||||
@@ -225,9 +254,9 @@ class MarketServiceMixin:
|
||||
normalized_date: str,
|
||||
snapshot: dict[str, Any],
|
||||
) -> bool:
|
||||
if not self.configured or normalized_date != date.today().strftime("%Y%m%d"):
|
||||
if not self.configured or normalized_date != self._now().strftime("%Y%m%d"):
|
||||
return False
|
||||
now = datetime.now().astimezone()
|
||||
now = self._now()
|
||||
local_time = now.time().replace(tzinfo=None)
|
||||
realtime_start = datetime.strptime("09:15", "%H:%M").time()
|
||||
morning_end = datetime.strptime("11:35", "%H:%M").time()
|
||||
@@ -264,7 +293,10 @@ class MarketServiceMixin:
|
||||
raise TushareError("公共行情尚未配置")
|
||||
dashboard = self._tushare_client().dashboard(normalized_date)
|
||||
meta = dashboard.setdefault("meta", {})
|
||||
quote_source = str(meta.get("quote_source") or "")
|
||||
meta["source"] = source
|
||||
if quote_source:
|
||||
meta["quote_source"] = quote_source
|
||||
meta["requested_date"] = self._display_compact_date(normalized_date)
|
||||
if meta.get("limit_data_source") == "derived":
|
||||
meta.setdefault(
|
||||
@@ -276,6 +308,12 @@ class MarketServiceMixin:
|
||||
actual_date = normalize_date(
|
||||
str(dashboard.get("meta", {}).get("trade_date") or normalized_date)
|
||||
)
|
||||
if actual_date != normalized_date and self._is_requested_open_session(
|
||||
normalized_date
|
||||
):
|
||||
raise TushareError(
|
||||
f"Intraday dashboard resolved {actual_date} instead of {normalized_date}"
|
||||
)
|
||||
self.database.save_snapshot(actual_date, source, dashboard)
|
||||
if actual_date != normalized_date:
|
||||
dashboard.setdefault("meta", {}).update(
|
||||
@@ -297,6 +335,30 @@ class MarketServiceMixin:
|
||||
)
|
||||
return self._apply_reason_overrides(self._with_storage(dashboard, cached=False))
|
||||
except TushareError as exc:
|
||||
if self._is_requested_open_session(normalized_date):
|
||||
existing = self.database.get_snapshot(normalized_date)
|
||||
existing_date = str(
|
||||
((existing or {}).get("meta") or {}).get("trade_date") or ""
|
||||
).replace("-", "")
|
||||
if existing and existing_date == normalized_date:
|
||||
kept = copy.deepcopy(existing)
|
||||
kept.setdefault("meta", {}).update(
|
||||
{
|
||||
"requested_date": self._display_compact_date(normalized_date),
|
||||
}
|
||||
)
|
||||
self.database.finish_sync(
|
||||
sync_id,
|
||||
"fallback",
|
||||
self._record_count(kept),
|
||||
str(exc),
|
||||
"tushare",
|
||||
)
|
||||
return self._apply_reason_overrides(
|
||||
self._with_storage(kept, cached=True)
|
||||
)
|
||||
self.database.finish_sync(sync_id, "failed", message=str(exc))
|
||||
raise ValueError("当天盘中行情暂时不可用,请稍后重试。") from exc
|
||||
fallback = self.database.get_latest_real_snapshot(normalized_date)
|
||||
if fallback:
|
||||
actual = str((fallback.get("meta") or {}).get("trade_date") or "")
|
||||
@@ -613,7 +675,7 @@ class MarketServiceMixin:
|
||||
"index_daily",
|
||||
{
|
||||
"ts_code": basic["id"],
|
||||
"start_date": (end - timedelta(days=190)).strftime("%Y%m%d"),
|
||||
"start_date": (end - timedelta(days=400)).strftime("%Y%m%d"),
|
||||
"end_date": resolved_date,
|
||||
},
|
||||
"ts_code,trade_date,open,high,low,close,pct_chg,vol,amount",
|
||||
@@ -629,10 +691,10 @@ class MarketServiceMixin:
|
||||
"change": float(row.get("pct_chg") or 0),
|
||||
"volume": float(row.get("vol") or 0),
|
||||
}
|
||||
for row in rows[-90:]
|
||||
for row in rows[-DAILY_CHART_LIMIT:]
|
||||
]
|
||||
try:
|
||||
chart_series = self.chart_data.index_daily(str(basic["id"]), resolved_date, 90)
|
||||
chart_series = self.chart_data.index_daily(str(basic["id"]), resolved_date, DAILY_CHART_LIMIT)
|
||||
if chart_series:
|
||||
series = chart_series
|
||||
except (AttributeError, ChartDataError):
|
||||
@@ -740,7 +802,7 @@ class MarketServiceMixin:
|
||||
result = copy.deepcopy(payload)
|
||||
now = datetime.now().astimezone()
|
||||
try:
|
||||
result["prices"] = self.chart_data.stock_daily(code, requested_date, 90)
|
||||
result["prices"] = self.chart_data.stock_daily(code, requested_date, DAILY_CHART_LIMIT)
|
||||
result["meta"] = {**(result.get("meta") or {}), "chart_source": "market_chart"}
|
||||
except (AttributeError, ChartDataError):
|
||||
pass
|
||||
@@ -752,27 +814,28 @@ class MarketServiceMixin:
|
||||
"trade_date": f"{actual_date[:4]}-{actual_date[4:6]}-{actual_date[6:]}",
|
||||
}
|
||||
today = now.strftime("%Y%m%d")
|
||||
latest_bar = (result.get("prices") or [{}])[-1] if result.get("prices") else {}
|
||||
official_today = (
|
||||
actual_date == today and not bool(latest_bar.get("realtime"))
|
||||
)
|
||||
after_close = now.time().replace(tzinfo=None) >= dt_time(15, 0)
|
||||
should_merge = (
|
||||
requested_date == today
|
||||
and actual_date <= today
|
||||
and now.weekday() < 5
|
||||
and now.time().replace(tzinfo=None) >= dt_time(9, 30)
|
||||
and not (official_today and after_close)
|
||||
)
|
||||
if should_merge:
|
||||
quote = self._ifind_realtime_stock_quote(code)
|
||||
quote = self._resolve_today_daily_quote(code, today, result)
|
||||
if quote and self._valid_realtime_stock_quote(quote, today):
|
||||
self._merge_realtime_stock_detail(result, quote, requested_date)
|
||||
elif self.configured and actual_date < today:
|
||||
client = self._tushare_client()
|
||||
try:
|
||||
resolved_date, _ = client.resolve_trade_context(requested_date)
|
||||
if resolved_date == today:
|
||||
quote = client.realtime_stock_quote(tushare_code(code), requested_date)
|
||||
if self._valid_realtime_stock_quote(quote, today):
|
||||
self._merge_realtime_stock_detail(result, quote, requested_date)
|
||||
except TushareError:
|
||||
pass
|
||||
return self._enrich_stock_detail(result)
|
||||
elif actual_date < today:
|
||||
result["meta"] = {
|
||||
**(result.get("meta") or {}),
|
||||
"notice": TODAY_DAILY_UNAVAILABLE_NOTICE,
|
||||
}
|
||||
return self._enrich_stock_detail(result, requested_date)
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_stock_detail_prices(
|
||||
@@ -886,6 +949,138 @@ class MarketServiceMixin:
|
||||
"quote_time": str(row.get("time") or ""),
|
||||
}
|
||||
|
||||
def _resolve_today_daily_quote(
|
||||
self, code: str, today: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
quote = self._ifind_realtime_stock_quote(code)
|
||||
if quote and self._valid_realtime_stock_quote(quote, today):
|
||||
return quote
|
||||
if self.configured:
|
||||
try:
|
||||
client = self._tushare_client()
|
||||
resolve = getattr(client, "resolve_trade_context", None)
|
||||
resolved = today
|
||||
if callable(resolve):
|
||||
resolved, _ = resolve(today)
|
||||
if str(resolved or "") == today:
|
||||
quote = client.realtime_stock_quote(tushare_code(code), today)
|
||||
if self._valid_realtime_stock_quote(quote, today):
|
||||
return quote
|
||||
except TushareError:
|
||||
pass
|
||||
quote = self._free_realtime_stock_quote(code, today)
|
||||
if quote and self._valid_realtime_stock_quote(quote, today):
|
||||
return quote
|
||||
return self._intraday_realtime_stock_quote(code, today, payload)
|
||||
|
||||
def _free_realtime_stock_quote(self, code: str, today: str) -> dict[str, Any] | None:
|
||||
aggregator = getattr(self, "realtime_aggregator", None)
|
||||
if aggregator is None:
|
||||
return None
|
||||
ts_code = tushare_code(code)
|
||||
for loader in (
|
||||
getattr(aggregator, "tencent_stock_quote", None),
|
||||
getattr(aggregator, "eastmoney_stock_quote", None),
|
||||
):
|
||||
if not callable(loader):
|
||||
continue
|
||||
try:
|
||||
row = loader(ts_code, expected_date=today)
|
||||
except (RealtimeAggregateError, Exception):
|
||||
continue
|
||||
quote = self._quote_from_free_row(code, today, row)
|
||||
if quote:
|
||||
return quote
|
||||
return None
|
||||
|
||||
def _quote_from_free_row(
|
||||
self, code: str, today: str, row: dict[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
price = float(row.get("close") or 0)
|
||||
previous_close = float(row.get("pre_close") or 0)
|
||||
if price <= 0 or previous_close <= 0:
|
||||
return None
|
||||
try:
|
||||
name, sector = self._stock_identity(code, today)
|
||||
except Exception:
|
||||
name, sector = "--", "其他"
|
||||
epoch = int(row.get("quote_time_epoch") or 0)
|
||||
if epoch > 0:
|
||||
quote_time = datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||
else:
|
||||
quote_date = str(row.get("quote_date") or today)
|
||||
quote_time = f"{quote_date[:4]}-{quote_date[4:6]}-{quote_date[6:]}"
|
||||
quote = {
|
||||
"name": str(row.get("name") or name or "--"),
|
||||
"sector": sector,
|
||||
"price": price,
|
||||
"open": float(row.get("open") or 0),
|
||||
"high": float(row.get("high") or 0),
|
||||
"low": float(row.get("low") or 0),
|
||||
"change": round((price / previous_close - 1) * 100, 4),
|
||||
"volume": float(row.get("vol") or 0),
|
||||
"amount_billion": float(row.get("amount") or 0) / 100_000_000,
|
||||
"turnover_rate": float(row.get("turnover_rate") or 0),
|
||||
"quote_time": quote_time,
|
||||
}
|
||||
flow = _moneyflow_payload(row)
|
||||
if flow.get("available"):
|
||||
quote["moneyflow"] = flow
|
||||
return quote
|
||||
|
||||
def _intraday_realtime_stock_quote(
|
||||
self, code: str, today: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any] | None:
|
||||
chart_data = getattr(self, "chart_data", None)
|
||||
if chart_data is None:
|
||||
return None
|
||||
try:
|
||||
chart = chart_data.stock_intraday(code)
|
||||
except (AttributeError, ChartDataError, Exception):
|
||||
return None
|
||||
points = [
|
||||
point
|
||||
for point in list(chart.get("points") or [])
|
||||
if str(point.get("date") or "").replace("-", "") == today
|
||||
]
|
||||
if not points:
|
||||
return None
|
||||
opens = [float(point.get("open") or 0) for point in points if float(point.get("open") or 0) > 0]
|
||||
highs = [float(point.get("high") or 0) for point in points if float(point.get("high") or 0) > 0]
|
||||
lows = [float(point.get("low") or 0) for point in points if float(point.get("low") or 0) > 0]
|
||||
closes = [float(point.get("close") or 0) for point in points if float(point.get("close") or 0) > 0]
|
||||
if not opens or not highs or not lows or not closes:
|
||||
return None
|
||||
price = closes[-1]
|
||||
previous_close = float(chart.get("previous_close") or 0)
|
||||
if previous_close <= 0:
|
||||
history = list(payload.get("prices") or [])
|
||||
previous_close = float((history[-1] if history else {}).get("close") or 0)
|
||||
if previous_close <= 0:
|
||||
return None
|
||||
volume = sum(float(point.get("volume") or 0) for point in points)
|
||||
amount = sum(float(point.get("amount") or 0) for point in points)
|
||||
if volume <= 0 and amount <= 0:
|
||||
return None
|
||||
try:
|
||||
name, sector = self._stock_identity(code, today)
|
||||
except Exception:
|
||||
name, sector = "--", "其他"
|
||||
return {
|
||||
"name": name,
|
||||
"sector": sector,
|
||||
"price": price,
|
||||
"open": opens[0],
|
||||
"high": max(highs),
|
||||
"low": min(lows),
|
||||
"change": round((price / previous_close - 1) * 100, 4),
|
||||
"volume": volume,
|
||||
"volume_unit": "lots",
|
||||
"amount_billion": amount / 100_000_000,
|
||||
"turnover_rate": 0.0,
|
||||
"quote_time": str(points[-1].get("date") or today),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _merge_realtime_stock_detail(
|
||||
payload: dict[str, Any], quote: dict[str, Any], trade_date: str
|
||||
@@ -907,23 +1102,29 @@ class MarketServiceMixin:
|
||||
prices[-1] = realtime_bar
|
||||
else:
|
||||
prices.append(realtime_bar)
|
||||
payload["prices"] = prices[-90:]
|
||||
payload["prices"] = prices[-DAILY_CHART_LIMIT:]
|
||||
stock = dict(payload.get("stock") or {})
|
||||
stock.update(
|
||||
{
|
||||
updates = {
|
||||
"name": quote["name"],
|
||||
"industry": quote["sector"],
|
||||
"price": quote["price"],
|
||||
"change": quote["change"],
|
||||
"amount_billion": quote["amount_billion"],
|
||||
"turnover_rate": quote["turnover_rate"],
|
||||
}
|
||||
)
|
||||
quote_turnover = _optional_number(quote.get("turnover_rate"))
|
||||
if quote_turnover:
|
||||
updates["turnover_rate"] = quote_turnover
|
||||
stock.update(updates)
|
||||
payload["stock"] = stock
|
||||
quote_flow = quote.get("moneyflow")
|
||||
current_flow = payload.get("moneyflow") or {}
|
||||
if isinstance(quote_flow, dict) and quote_flow.get("available") and not current_flow.get("available"):
|
||||
payload["moneyflow"] = quote_flow
|
||||
payload["meta"] = {
|
||||
**(payload.get("meta") or {}),
|
||||
"trade_date": display_date,
|
||||
"realtime": True,
|
||||
"notice": "",
|
||||
"updated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
}
|
||||
|
||||
@@ -957,7 +1158,7 @@ class MarketServiceMixin:
|
||||
intraday_status = "unavailable"
|
||||
intraday_notice = "分时行情暂不可用,请稍后重试。"
|
||||
|
||||
prices = list(detail.get("prices") or [])[-60:]
|
||||
prices = list(detail.get("prices") or [])[-DAILY_CHART_LIMIT:]
|
||||
stock = dict(detail.get("stock") or {"code": code})
|
||||
realtime = bool(detail_meta.get("realtime"))
|
||||
return {
|
||||
@@ -1209,10 +1410,40 @@ class MarketServiceMixin:
|
||||
return item["name"], item["sector"] or "其他"
|
||||
return "--", "其他"
|
||||
|
||||
def _enrich_stock_detail(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
def _enrich_stock_detail(
|
||||
self, payload: dict[str, Any], trade_date: str = ""
|
||||
) -> dict[str, Any]:
|
||||
result = dict(payload)
|
||||
stock = dict(payload.get("stock") or {})
|
||||
code = str(stock.get("code") or "")
|
||||
compact_date = normalize_date(
|
||||
str((payload.get("meta") or {}).get("trade_date") or trade_date)
|
||||
)
|
||||
board = self._limit_event_for_stock(code, compact_date)
|
||||
if board:
|
||||
if not stock.get("first_time") or stock.get("first_time") == "--":
|
||||
stock["first_time"] = board.get("first_time") or "--"
|
||||
if not stock.get("last_time") or stock.get("last_time") == "--":
|
||||
stock["last_time"] = board.get("last_time") or "--"
|
||||
if not stock.get("open_times"):
|
||||
stock["open_times"] = board.get("open_times") or 0
|
||||
if _optional_number(stock.get("seal_amount_million")) is None:
|
||||
stock["seal_amount_million"] = board.get("seal_amount_million")
|
||||
if not _optional_number(stock.get("turnover_rate")) and _optional_number(board.get("turnover_rate")):
|
||||
stock["turnover_rate"] = board.get("turnover_rate")
|
||||
flow = result.get("moneyflow") or {}
|
||||
if not flow.get("available"):
|
||||
live_flow = self._live_moneyflow_for_stock(code, compact_date)
|
||||
if live_flow.get("available"):
|
||||
result["moneyflow"] = live_flow
|
||||
else:
|
||||
result["moneyflow"] = {
|
||||
"available": False,
|
||||
"net_million": None,
|
||||
"large_million": None,
|
||||
"medium_million": None,
|
||||
"small_million": None,
|
||||
}
|
||||
watched = {
|
||||
item["code"]: item
|
||||
for item in self.database.list_watchlist(self.current_user_id)
|
||||
@@ -1222,6 +1453,45 @@ class MarketServiceMixin:
|
||||
result["notes"] = self.database.list_notes(self.current_user_id, code=code)
|
||||
return result
|
||||
|
||||
def _limit_event_for_stock(self, code: str, trade_date: str) -> dict[str, Any]:
|
||||
if not code or not trade_date:
|
||||
return {}
|
||||
ts_code = tushare_code(code)
|
||||
client = self._tushare_client() if self.configured else None
|
||||
rows: list[dict[str, Any]] = []
|
||||
if client is not None:
|
||||
try:
|
||||
rows = client._load_limit_type(trade_date, "U") + client._load_limit_type(trade_date, "Z")
|
||||
except Exception:
|
||||
rows = []
|
||||
if not rows:
|
||||
try:
|
||||
rows = list((client._free_board_map(trade_date) or {}).values())
|
||||
except Exception:
|
||||
rows = []
|
||||
match = next((row for row in rows if str(row.get("ts_code") or "") == ts_code), None)
|
||||
if not match:
|
||||
return {}
|
||||
fd = _optional_number(match.get("fd_amount"))
|
||||
return {
|
||||
"first_time": match.get("first_time") or "--",
|
||||
"last_time": match.get("last_time") or "--",
|
||||
"open_times": match.get("open_times") or 0,
|
||||
"seal_amount_million": None if fd is None else round(fd / 10000, 0),
|
||||
"turnover_rate": _optional_number(match.get("turnover_ratio")),
|
||||
}
|
||||
|
||||
def _live_moneyflow_for_stock(self, code: str, trade_date: str) -> dict[str, Any]:
|
||||
aggregator = getattr(self, "realtime_aggregator", None)
|
||||
loader = getattr(aggregator, "eastmoney_stock_quote", None) if aggregator else None
|
||||
if not callable(loader) or not code:
|
||||
return _moneyflow_payload(None)
|
||||
try:
|
||||
quote = loader(tushare_code(code), expected_date=trade_date)
|
||||
except Exception:
|
||||
return _moneyflow_payload(None)
|
||||
return _moneyflow_payload(quote)
|
||||
|
||||
def _with_storage(self, dashboard: dict[str, Any], cached: bool) -> dict[str, Any]:
|
||||
result = dict(dashboard)
|
||||
result["meta"] = {
|
||||
|
||||
@@ -4,7 +4,14 @@ import re
|
||||
import secrets
|
||||
from typing import Any
|
||||
|
||||
from backend.bootstrap.config import TOKEN_PATTERN, validate_text
|
||||
from backend.bootstrap.config import validate_text
|
||||
|
||||
|
||||
MARKET_SOURCE_SECRET_KEYS = {
|
||||
"tushare_token",
|
||||
"ifind_refresh_token",
|
||||
"ifind_access_token",
|
||||
}
|
||||
|
||||
|
||||
class SystemServiceMixin:
|
||||
@@ -18,9 +25,6 @@ class SystemServiceMixin:
|
||||
first_encrypted = self.database.get_user_credentials(first_user_id)
|
||||
first_personal = self.vault.decrypt_json(first_encrypted) if first_encrypted else {}
|
||||
defaults = {
|
||||
"tushare_token": environment.get("tushare_token") or first_personal.get("tushare_token") or "",
|
||||
"ifind_refresh_token": environment.get("ifind_refresh_token") or "",
|
||||
"ifind_access_token": environment.get("ifind_access_token") or "",
|
||||
"platform_llm_primary_api_key": environment.get("platform_llm_primary_api_key") or first_personal.get("llm_primary_api_key") or "",
|
||||
"platform_llm_primary_base_url": environment.get("platform_llm_primary_base_url") or first_personal.get("llm_primary_base_url") or "https://api.openai.com/v1",
|
||||
"platform_llm_primary_model": environment.get("platform_llm_primary_model") or first_personal.get("llm_primary_model") or "",
|
||||
@@ -34,6 +38,10 @@ class SystemServiceMixin:
|
||||
if key not in current:
|
||||
current[key] = value
|
||||
changed = True
|
||||
for key in MARKET_SOURCE_SECRET_KEYS:
|
||||
if key in current:
|
||||
current.pop(key, None)
|
||||
changed = True
|
||||
if not isinstance(current.get("llm_models"), list):
|
||||
migrated_models: list[dict[str, str]] = []
|
||||
for role, label in (("primary", "原主模型"), ("fallback", "原辅助模型")):
|
||||
@@ -56,26 +64,27 @@ class SystemServiceMixin:
|
||||
self.database.save_system_setting("credentials", self.vault.encrypt_json(current))
|
||||
for row in self.database.list_user_credentials():
|
||||
personal = self.vault.decrypt_json(str(row.get("encrypted_payload") or ""))
|
||||
if "tushare_token" in personal:
|
||||
personal.pop("tushare_token", None)
|
||||
if any(key in personal for key in MARKET_SOURCE_SECRET_KEYS):
|
||||
for key in MARKET_SOURCE_SECRET_KEYS:
|
||||
personal.pop(key, None)
|
||||
self.database.save_user_credentials(
|
||||
int(row["user_id"]), self.vault.encrypt_json(personal)
|
||||
)
|
||||
return current
|
||||
|
||||
def _save_system_credentials(self, credentials: dict[str, Any]) -> None:
|
||||
sanitized = {
|
||||
key: value
|
||||
for key, value in credentials.items()
|
||||
if key not in MARKET_SOURCE_SECRET_KEYS
|
||||
}
|
||||
with self.system_lock:
|
||||
self.database.save_system_setting("credentials", self.vault.encrypt_json(credentials))
|
||||
self._system_credentials = dict(credentials)
|
||||
if hasattr(self, "ifind"):
|
||||
self.ifind.set_credentials(
|
||||
str(credentials.get("ifind_refresh_token") or ""),
|
||||
str(credentials.get("ifind_access_token") or ""),
|
||||
)
|
||||
self.database.save_system_setting("credentials", self.vault.encrypt_json(sanitized))
|
||||
self._system_credentials = dict(sanitized)
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self.token)
|
||||
return bool(self._datahub_status().get("configured"))
|
||||
|
||||
def _credentials(self) -> dict[str, str]:
|
||||
credentials = getattr(self._request_context, "credentials", {})
|
||||
@@ -99,7 +108,7 @@ class SystemServiceMixin:
|
||||
|
||||
@property
|
||||
def token(self) -> str:
|
||||
return str(self._system_credentials.get("tushare_token") or "")
|
||||
return "datahub" if self.configured else ""
|
||||
|
||||
def system_status(self) -> dict[str, Any]:
|
||||
platform = self._platform_llm_profile()
|
||||
@@ -130,6 +139,7 @@ class SystemServiceMixin:
|
||||
),
|
||||
**self.database.status(),
|
||||
"jobs": self.jobs.repository.recent(12),
|
||||
"datahub": self._datahub_status(),
|
||||
},
|
||||
"llm": {
|
||||
"primary_configured": self._profile_configured(platform["primary"]),
|
||||
@@ -145,21 +155,24 @@ class SystemServiceMixin:
|
||||
},
|
||||
}
|
||||
|
||||
def _datahub_status(self) -> dict[str, Any]:
|
||||
gateway = getattr(self, "data_gateway", None)
|
||||
reporter = getattr(gateway, "datahub_status", None)
|
||||
if callable(reporter):
|
||||
return reporter()
|
||||
return {
|
||||
"configured": False,
|
||||
"base_url": "",
|
||||
"enabled_reads": 0,
|
||||
"total_reads": 0,
|
||||
"flags": [],
|
||||
"routes": [],
|
||||
"fallback_count": 0,
|
||||
"fallback_labels": [],
|
||||
}
|
||||
|
||||
def save_system_settings(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = dict(self._system_credentials)
|
||||
token = str(payload.get("tushare_token") or current.get("tushare_token") or "").strip()
|
||||
if token and not TOKEN_PATTERN.fullmatch(token):
|
||||
raise ValueError("Tushare Token 格式不正确。")
|
||||
ifind_refresh_token = str(
|
||||
payload.get("ifind_refresh_token")
|
||||
or current.get("ifind_refresh_token")
|
||||
or ""
|
||||
).strip()
|
||||
if ifind_refresh_token and (
|
||||
len(ifind_refresh_token) > 2048
|
||||
or any(character.isspace() for character in ifind_refresh_token)
|
||||
):
|
||||
raise ValueError("iFinD Refresh Token 格式不正确。")
|
||||
existing_models = {
|
||||
str(item.get("id") or ""): item
|
||||
for item in current.get("llm_models") or []
|
||||
@@ -221,8 +234,6 @@ class SystemServiceMixin:
|
||||
raise ValueError("会员每日额度应为 1 至 1000。") from exc
|
||||
current.update(
|
||||
{
|
||||
"tushare_token": token,
|
||||
"ifind_refresh_token": ifind_refresh_token,
|
||||
"llm_models": models,
|
||||
"primary_model_id": primary_model_id,
|
||||
"fallback_model_id": fallback_model_id,
|
||||
@@ -242,7 +253,7 @@ class SystemServiceMixin:
|
||||
llm_access = self.llm_access_status()
|
||||
return {
|
||||
"configured": self.configured,
|
||||
"mode": "tushare" if self.configured else "unavailable",
|
||||
"mode": "datahub" if self.configured else "unavailable",
|
||||
"llm_configured": self.llm_configured,
|
||||
"llm_model": self.llm_primary_model if self.llm_configured else "",
|
||||
"llm_fallback_configured": self.llm_fallback_configured,
|
||||
|
||||
@@ -41,6 +41,8 @@ def official_catchup_due(today: str, snapshot: dict[str, object]) -> bool:
|
||||
actual == today
|
||||
and meta.get("limit_data_source") != "derived"
|
||||
and not meta.get("carried_forward")
|
||||
and not meta.get("realtime")
|
||||
and meta.get("mode") != "realtime"
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -20,6 +20,8 @@ services:
|
||||
DATAHUB_TOKEN: "${DATAHUB_TOKEN:?DATAHUB_TOKEN must be set}"
|
||||
DATAHUB_ADMIN_PASSWORD: "${DATAHUB_ADMIN_PASSWORD:?DATAHUB_ADMIN_PASSWORD must be set}"
|
||||
TUSHARE_TOKEN: "${TUSHARE_TOKEN:-}"
|
||||
IFIND_REFRESH_TOKEN: "${IFIND_REFRESH_TOKEN:-}"
|
||||
IFIND_ACCESS_TOKEN: "${IFIND_ACCESS_TOKEN:-}"
|
||||
DATAHUB_DB_PATH: /app/data/datahub.db
|
||||
DATAHUB_BACKUP_DIR: /app/data/backups
|
||||
TZ: Asia/Shanghai
|
||||
|
||||
@@ -13,6 +13,26 @@ services:
|
||||
- ./.env
|
||||
environment:
|
||||
APP_ENCRYPTION_KEY: "${APP_ENCRYPTION_KEY:?APP_ENCRYPTION_KEY must be set in .env}"
|
||||
# Provider credentials are consumed only by xiaobai-datahub.
|
||||
TUSHARE_TOKEN: ""
|
||||
IFIND_REFRESH_TOKEN: ""
|
||||
IFIND_ACCESS_TOKEN: ""
|
||||
DATAHUB_BASE_URL: "${DATAHUB_BASE_URL:-http://192.168.200.11:8766}"
|
||||
DATAHUB_READ_CALENDAR: "1"
|
||||
DATAHUB_READ_STOCKS: "1"
|
||||
DATAHUB_READ_DAILY: "1"
|
||||
DATAHUB_READ_INDEX_DAILY: "1"
|
||||
DATAHUB_READ_VALUATION: "1"
|
||||
DATAHUB_READ_MONEYFLOW: "1"
|
||||
DATAHUB_READ_AUCTION: "1"
|
||||
DATAHUB_READ_LIMIT_EVENTS: "1"
|
||||
DATAHUB_READ_POPULARITY: "1"
|
||||
DATAHUB_READ_DRAGON_TIGER: "1"
|
||||
DATAHUB_READ_SECTOR_DAILY: "1"
|
||||
DATAHUB_READ_QUOTES: "1"
|
||||
DATAHUB_READ_INDEX_QUOTES: "1"
|
||||
DATAHUB_READ_INTRADAY: "1"
|
||||
DATAHUB_READ_STATUS: "1"
|
||||
TZ: Asia/Shanghai
|
||||
PYTHONUTF8: "1"
|
||||
volumes:
|
||||
|
||||
+6
-3
@@ -12,9 +12,12 @@ These registries describe the approved product surface of the standalone applica
|
||||
providers, model entry points, CSS layers, and remaining code hotspots.
|
||||
- `data-fields.config.json`: canonical data products, provider eligibility, intended use, and
|
||||
known blocked datasets.
|
||||
- `datahub.config.json`: optional read-only client for `xiaobai-datahub`. Each dataset has its
|
||||
own `read` / `shadow` flag, all default off. Environment variables `DATAHUB_READ_*` and
|
||||
`DATAHUB_SHADOW_*` can override a single dataset without a master switch.
|
||||
- `datahub.config.json`: official read-only client for `xiaobai-datahub`. Each dataset has its
|
||||
own `read` / `shadow` flag; official reads default on. `compose.yaml` pins every
|
||||
`DATAHUB_READ_*` to `"1"` so a leftover `.env` `=0` cannot silently keep official
|
||||
pages on the old APIs. Environment variables can still override a single
|
||||
`DATAHUB_SHADOW_*` without a master switch. The old website APIs stay as
|
||||
emergency fallback only.
|
||||
- `data-quality.config.json`: freshness, coverage, units, adjustment, point-in-time, and
|
||||
fail-closed rules for every canonical data product.
|
||||
- `jobs.config.json`: background schedules, dependencies, lock keys, retry policy, timeouts,
|
||||
|
||||
@@ -207,27 +207,22 @@
|
||||
{
|
||||
"provider": "datahub",
|
||||
"path": "backend/data/datahub/client.py",
|
||||
"runtime_role": "optional official EOD read path behind per-dataset flags"
|
||||
"runtime_role": "website-only read path; official EOD, live quotes, and licensed iFinD"
|
||||
},
|
||||
{
|
||||
"provider": "ifind",
|
||||
"path": "backend/data/providers/ifind_client.py",
|
||||
"runtime_role": "realtime, charts, snapshots, enrichment"
|
||||
"path": "xiaobai-datahub/datahub/adapters/ifind.py",
|
||||
"runtime_role": "licensed iFinD source inside the data hub"
|
||||
},
|
||||
{
|
||||
"provider": "eastmoney",
|
||||
"path": "backend/features/market/charts.py",
|
||||
"runtime_role": "display chart fallback"
|
||||
},
|
||||
{
|
||||
"provider": "eastmoney",
|
||||
"path": "backend/data/realtime.py",
|
||||
"runtime_role": "isolated realtime observation"
|
||||
"path": "xiaobai-datahub/datahub/adapters/eastmoney.py",
|
||||
"runtime_role": "free realtime quotes and shenwan inside the data hub"
|
||||
},
|
||||
{
|
||||
"provider": "tencent",
|
||||
"path": "backend/data/realtime.py",
|
||||
"runtime_role": "index observation fallback"
|
||||
"path": "xiaobai-datahub/datahub/adapters/tencent.py",
|
||||
"runtime_role": "free index and stock quotes inside the data hub"
|
||||
}
|
||||
],
|
||||
"provider_domains": [
|
||||
@@ -296,16 +291,16 @@
|
||||
"owner": "backend/data/gateway.py"
|
||||
},
|
||||
{
|
||||
"client": "IfindHttpClient",
|
||||
"client": "HubIfindProxy",
|
||||
"owner": "backend/data/gateway.py"
|
||||
},
|
||||
{
|
||||
"client": "HubRealtimeProxy",
|
||||
"owner": "backend/data/gateway.py"
|
||||
},
|
||||
{
|
||||
"client": "MarketChartClient",
|
||||
"owner": "backend/data/gateway.py"
|
||||
},
|
||||
{
|
||||
"client": "WebRealtimeAggregator",
|
||||
"owner": "backend/data/gateway.py"
|
||||
}
|
||||
],
|
||||
"heaven_service_owners": {
|
||||
@@ -463,28 +458,33 @@
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/screener/foundation.css",
|
||||
"bytes": 103547,
|
||||
"lines": 6576
|
||||
"bytes": 103683,
|
||||
"lines": 6582
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/heaven/page.js",
|
||||
"bytes": 97268,
|
||||
"lines": 2070
|
||||
"bytes": 97770,
|
||||
"lines": 2079
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/shell.css",
|
||||
"bytes": 63733,
|
||||
"lines": 3767
|
||||
"bytes": 64839,
|
||||
"lines": 3794
|
||||
},
|
||||
{
|
||||
"path": "backend/features/heaven/engine.py",
|
||||
"bytes": 51764,
|
||||
"lines": 1183
|
||||
"bytes": 52046,
|
||||
"lines": 1192
|
||||
},
|
||||
{
|
||||
"path": "frontend/index.html",
|
||||
"bytes": 48254,
|
||||
"lines": 664
|
||||
"bytes": 48403,
|
||||
"lines": 665
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_industries.py",
|
||||
"bytes": 38936,
|
||||
"lines": 894
|
||||
},
|
||||
{
|
||||
"path": "backend/features/screener/catalog.py",
|
||||
@@ -496,6 +496,11 @@
|
||||
"bytes": 35247,
|
||||
"lines": 2416
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_dashboard.py",
|
||||
"bytes": 35001,
|
||||
"lines": 805
|
||||
},
|
||||
{
|
||||
"path": "database.py",
|
||||
"bytes": 32073,
|
||||
@@ -506,16 +511,6 @@
|
||||
"bytes": 31756,
|
||||
"lines": 562
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_dashboard.py",
|
||||
"bytes": 28234,
|
||||
"lines": 648
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_industries.py",
|
||||
"bytes": 26540,
|
||||
"lines": 616
|
||||
},
|
||||
{
|
||||
"path": "backend/features/heaven/manual.py",
|
||||
"bytes": 24521,
|
||||
@@ -533,13 +528,13 @@
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/market/preview.js",
|
||||
"bytes": 18178,
|
||||
"lines": 446
|
||||
"bytes": 18230,
|
||||
"lines": 447
|
||||
},
|
||||
{
|
||||
"path": "backend/features/heaven/trend.py",
|
||||
"bytes": 16772,
|
||||
"lines": 370
|
||||
"bytes": 17005,
|
||||
"lines": 373
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/insights_auction_scoring.py",
|
||||
@@ -548,33 +543,33 @@
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/market/charts.js",
|
||||
"bytes": 15311,
|
||||
"lines": 387
|
||||
"bytes": 15743,
|
||||
"lines": 401
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/dashboard.js",
|
||||
"bytes": 15063,
|
||||
"lines": 321
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/pools/page.html",
|
||||
"bytes": 14942,
|
||||
"lines": 235
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/admin.js",
|
||||
"bytes": 14836,
|
||||
"lines": 283
|
||||
},
|
||||
{
|
||||
"path": "backend/features/screener/data_sync.py",
|
||||
"bytes": 14743,
|
||||
"lines": 342
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/dashboard.js",
|
||||
"bytes": 14740,
|
||||
"lines": 316
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/admin.js",
|
||||
"bytes": 14410,
|
||||
"lines": 268
|
||||
},
|
||||
{
|
||||
"path": "backend/features/heaven/market_context.py",
|
||||
"bytes": 13681,
|
||||
"lines": 338
|
||||
"bytes": 14467,
|
||||
"lines": 356
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/session.js",
|
||||
@@ -588,8 +583,8 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/features/system/service.py",
|
||||
"bytes": 12392,
|
||||
"lines": 254
|
||||
"bytes": 12180,
|
||||
"lines": 265
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/insights_auction.py",
|
||||
@@ -606,6 +601,11 @@
|
||||
"bytes": 9876,
|
||||
"lines": 224
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_daily.py",
|
||||
"bytes": 9539,
|
||||
"lines": 241
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/insights_themes.py",
|
||||
"bytes": 9348,
|
||||
@@ -613,7 +613,7 @@
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/market/entity-detail.js",
|
||||
"bytes": 9119,
|
||||
"bytes": 9139,
|
||||
"lines": 199
|
||||
},
|
||||
{
|
||||
@@ -626,6 +626,11 @@
|
||||
"bytes": 8562,
|
||||
"lines": 238
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_indices.py",
|
||||
"bytes": 8447,
|
||||
"lines": 189
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/mentor/page.html",
|
||||
"bytes": 8357,
|
||||
@@ -636,16 +641,6 @@
|
||||
"bytes": 6983,
|
||||
"lines": 146
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_daily.py",
|
||||
"bytes": 6837,
|
||||
"lines": 160
|
||||
},
|
||||
{
|
||||
"path": "backend/application.py",
|
||||
"bytes": 6751,
|
||||
"lines": 178
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/insights_popularity.py",
|
||||
"bytes": 6739,
|
||||
@@ -662,8 +657,13 @@
|
||||
"lines": 81
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_stocks.py",
|
||||
"bytes": 6244,
|
||||
"path": "backend/application.py",
|
||||
"bytes": 6399,
|
||||
"lines": 161
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/market/stock-detail.js",
|
||||
"bytes": 6325,
|
||||
"lines": 137
|
||||
},
|
||||
{
|
||||
@@ -682,14 +682,9 @@
|
||||
"lines": 85
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/market/stock-detail.js",
|
||||
"bytes": 5690,
|
||||
"lines": 124
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_indices.py",
|
||||
"bytes": 5451,
|
||||
"lines": 118
|
||||
"path": "backend/data/providers/tushare_stocks.py",
|
||||
"bytes": 5592,
|
||||
"lines": 123
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages.config.js",
|
||||
@@ -721,6 +716,11 @@
|
||||
"bytes": 4712,
|
||||
"lines": 106
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_helpers.py",
|
||||
"bytes": 4406,
|
||||
"lines": 124
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/routes.py",
|
||||
"bytes": 4276,
|
||||
@@ -766,16 +766,16 @@
|
||||
"bytes": 3316,
|
||||
"lines": 55
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/rotation/page.html",
|
||||
"bytes": 3299,
|
||||
"lines": 57
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/insights_context.py",
|
||||
"bytes": 3175,
|
||||
"lines": 84
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/rotation/page.html",
|
||||
"bytes": 3031,
|
||||
"lines": 54
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/market/bindings.js",
|
||||
"bytes": 2663,
|
||||
@@ -796,26 +796,21 @@
|
||||
"bytes": 2299,
|
||||
"lines": 57
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_client.py",
|
||||
"bytes": 2263,
|
||||
"lines": 70
|
||||
},
|
||||
{
|
||||
"path": "backend/features/screener/regime.py",
|
||||
"bytes": 2202,
|
||||
"lines": 53
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_client.py",
|
||||
"bytes": 2166,
|
||||
"lines": 68
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/popularity/page.html",
|
||||
"bytes": 2165,
|
||||
"lines": 35
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_helpers.py",
|
||||
"bytes": 2083,
|
||||
"lines": 64
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/market/breadth.js",
|
||||
"bytes": 2071,
|
||||
@@ -827,13 +822,13 @@
|
||||
"lines": 45
|
||||
},
|
||||
{
|
||||
"path": "backend/features/system/routes.py",
|
||||
"bytes": 1791,
|
||||
"lines": 46
|
||||
"path": "backend/jobs/refresh.py",
|
||||
"bytes": 1808,
|
||||
"lines": 48
|
||||
},
|
||||
{
|
||||
"path": "backend/jobs/refresh.py",
|
||||
"bytes": 1728,
|
||||
"path": "backend/features/system/routes.py",
|
||||
"bytes": 1791,
|
||||
"lines": 46
|
||||
},
|
||||
{
|
||||
@@ -846,6 +841,11 @@
|
||||
"bytes": 1642,
|
||||
"lines": 53
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_transport.py",
|
||||
"bytes": 1592,
|
||||
"lines": 50
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/insights.py",
|
||||
"bytes": 1580,
|
||||
@@ -856,11 +856,6 @@
|
||||
"bytes": 1535,
|
||||
"lines": 39
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_transport.py",
|
||||
"bytes": 1455,
|
||||
"lines": 48
|
||||
},
|
||||
{
|
||||
"path": "backend/features/themes/routes.py",
|
||||
"bytes": 1337,
|
||||
|
||||
@@ -6,13 +6,20 @@
|
||||
"page_limit": 5000,
|
||||
"stale_seconds_max": 86400,
|
||||
"datasets": {
|
||||
"calendar": { "read": false, "shadow": false },
|
||||
"stocks": { "read": false, "shadow": false },
|
||||
"daily": { "read": false, "shadow": false },
|
||||
"index_daily": { "read": false, "shadow": false },
|
||||
"valuation": { "read": false, "shadow": false },
|
||||
"moneyflow": { "read": false, "shadow": false },
|
||||
"auction": { "read": false, "shadow": false },
|
||||
"status": { "read": false, "shadow": false }
|
||||
"calendar": { "read": true, "shadow": false },
|
||||
"stocks": { "read": true, "shadow": false },
|
||||
"daily": { "read": true, "shadow": false },
|
||||
"index_daily": { "read": true, "shadow": false },
|
||||
"valuation": { "read": true, "shadow": false },
|
||||
"moneyflow": { "read": true, "shadow": false },
|
||||
"auction": { "read": true, "shadow": false },
|
||||
"limit_events": { "read": true, "shadow": false },
|
||||
"popularity": { "read": true, "shadow": false },
|
||||
"dragon_tiger": { "read": true, "shadow": false },
|
||||
"sector_daily": { "read": true, "shadow": false },
|
||||
"quotes": { "read": true, "shadow": false },
|
||||
"index_quotes": { "read": true, "shadow": false },
|
||||
"intraday": { "read": true, "shadow": false },
|
||||
"status": { "read": true, "shadow": false }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -213,12 +213,12 @@
|
||||
{
|
||||
"provider": "eastmoney",
|
||||
"path": "realtime_aggregator.py",
|
||||
"runtime_role": "isolated realtime observation"
|
||||
"runtime_role": "isolated realtime observation and intraday dashboard fallback"
|
||||
},
|
||||
{
|
||||
"provider": "tencent",
|
||||
"path": "realtime_aggregator.py",
|
||||
"runtime_role": "index observation fallback"
|
||||
"runtime_role": "index observation and intraday quote fallback"
|
||||
}
|
||||
],
|
||||
"llm_entrypoints": [
|
||||
|
||||
@@ -320,6 +320,11 @@ PC端统一采用以下固定骨架:
|
||||
|
||||
### 6.1 数据源职责
|
||||
|
||||
运行边界:下表中的职责全部由独立的 `xiaobai-datahub` 数据中枢执行。主网站只按固定业务
|
||||
协议请求“行情、日K、分时、申万、竞价”等数据,不接触任何提供方参数或凭据,也不决定优先级、
|
||||
重试、降级和回填。数据中枢是主网站唯一的行情出口;中枢暂时取不到新数据时,网站只能读取
|
||||
已经归档的真实快照,不能绕回旧提供方直连接口。
|
||||
|
||||
| 数据源 | 可用于正式计算 | 主要职责 |
|
||||
|---|:---:|---|
|
||||
| Tushare | 是 | 交易日历、股票主表、日线、估值、财务、资金流、申万行业、涨跌停、9:25竞价、热榜、龙虎榜 |
|
||||
|
||||
+4
-3
@@ -607,12 +607,13 @@
|
||||
<div class="admin-panel" data-admin-panel="market">
|
||||
<form id="systemMarketForm" class="settings-section">
|
||||
<div class="settings-section-heading"><h3>公共行情</h3><span id="systemDataStatus">待检查</span></div>
|
||||
<label class="form-field"><span>Tushare Token</span><input id="systemTokenInput" type="password" autocomplete="off" minlength="20" placeholder="留空保留现有 Token"></label>
|
||||
<label class="form-field"><span>iFinD Refresh Token</span><input id="systemIfindTokenInput" type="password" autocomplete="off" maxlength="2048" placeholder="留空保留现有 Token"></label>
|
||||
<label class="form-field"><span>行情来源凭据</span><input id="systemTokenInput" type="text" value="请在数据中枢后台统一管理" disabled></label>
|
||||
<label class="form-field"><span>实时来源凭据</span><input id="systemIfindTokenInput" type="text" value="请在数据中枢后台统一管理" disabled></label>
|
||||
<label class="switch-control"><input id="systemBackgroundRefresh" type="checkbox"><span>启用交易时段后台刷新</span></label>
|
||||
<p class="form-hint">所有用户读取同一份后台快照,页面不会随后台任务自动重绘。</p>
|
||||
<div id="datahubRouteStatus" class="admin-refresh-status" data-tone="idle" role="status" aria-live="polite"><i data-lucide="database"></i><span>数据中枢线路待检查</span></div>
|
||||
<div id="adminRefreshStatus" class="admin-refresh-status" data-tone="idle" role="status" aria-live="polite"><i data-lucide="circle-dot"></i><span>尚未手动刷新</span></div>
|
||||
<div class="dialog-actions admin-inline-actions"><button id="adminRefreshButton" class="button" type="button"><i data-lucide="refresh-cw"></i>立即后台刷新</button><button class="button primary" type="submit">保存行情配置</button></div>
|
||||
<div class="dialog-actions admin-inline-actions"><button id="adminRefreshButton" class="button" type="button"><i data-lucide="refresh-cw"></i>立即后台刷新</button><button class="button primary" type="submit">保存刷新设置</button></div>
|
||||
</form>
|
||||
<section class="settings-section">
|
||||
<div class="settings-section-heading"><h3>历史数据回补</h3><span>管理员任务</span></div>
|
||||
|
||||
+20
-14
@@ -3402,9 +3402,9 @@
|
||||
const payload = detail && detail.payload ? detail.payload : {};
|
||||
const meta = payload.meta || {};
|
||||
if (tab === "daily") {
|
||||
const bars = (payload.prices || []).slice(-48);
|
||||
const bars = (payload.prices || []).slice(-45);
|
||||
const last = bars.length ? bars[bars.length - 1].trade_date : "";
|
||||
return "日线 · 近48根 · 至 " + (displayCompactDate(last) || "--");
|
||||
return "日线 · 近45根 · 至 " + (displayCompactDate(last) || "--");
|
||||
}
|
||||
const d = displayCompactDate(meta.intraday_trade_date) || displayCompactDate(meta.trade_date);
|
||||
return "分时 · " + (d || "--");
|
||||
@@ -3702,7 +3702,7 @@
|
||||
const W = 360, H = 240, padL = 8, padR = 52, padT = 10, padB = 22;
|
||||
const pw = W - padL - padR;
|
||||
const ph = H - padT - padB;
|
||||
const prices = (payload.prices || []).slice(-48);
|
||||
const prices = (payload.prices || []).slice(-45);
|
||||
|
||||
if (prices.length < 2) return emptyChart("日线数据暂不可用");
|
||||
|
||||
@@ -5219,6 +5219,15 @@
|
||||
return '<span class="m-sys-dot' + (ok ? " m-sys-dot--ok" : "") + '"></span>';
|
||||
}
|
||||
|
||||
function datahubStatusText(hub) {
|
||||
const enabled = number(hub.enabled_reads);
|
||||
const total = number(hub.total_reads) || enabled;
|
||||
const fallbacks = hub.fallback_labels || [];
|
||||
if (fallbacks.length) return " 备用 " + fallbacks.join("、");
|
||||
if (hub.configured) return " 主线路 " + enabled + "/" + total;
|
||||
return " 未配置";
|
||||
}
|
||||
|
||||
function renderSystemAdmin(key) {
|
||||
if (key === "system/members") {
|
||||
renderSystemMembers();
|
||||
@@ -5233,15 +5242,16 @@
|
||||
'<div class="m-sys-body" data-system-admin-panel="market">' +
|
||||
'<div class="m-card m-sys-section"><strong>数据源状态</strong>' +
|
||||
'<div class="m-sys-status-list">' +
|
||||
'<div class="m-sys-status-item"><span>Tushare</span><span>' + statusDot(data.configured) + (data.configured ? " 已配置" : " 未配置") + "</span></div>" +
|
||||
'<div class="m-sys-status-item"><span>数据中枢</span><span>' + statusDot(data.configured) + (data.configured ? " 已连接" : " 未连接") + "</span></div>" +
|
||||
'<div class="m-sys-status-item"><span>iFinD</span><span>' + statusDot(ifind.configured) + (ifind.configured ? " 已配置" : " 未配置") + "</span></div>" +
|
||||
'<div class="m-sys-status-item"><span>行情快照</span><strong>' + number(data.snapshot_dates) + " 个交易日</strong></div>" +
|
||||
'<div class="m-sys-status-item"><span>后台刷新</span><span>' + statusDot(data.background_refresh_enabled) + (data.background_refresh_enabled ? " 已启用" : " 已暂停") + "</span></div>" +
|
||||
'<div class="m-sys-status-item"><span>数据中枢</span><span>' + statusDot(Boolean((data.datahub || {}).configured) && !((data.datahub || {}).fallback_count)) + datahubStatusText(data.datahub || {}) + "</span></div>" +
|
||||
"</div></div>" +
|
||||
'<div class="m-card m-sys-section"><strong>数据源密钥</strong>' +
|
||||
formFieldHtml("Tushare Token", '<input id="m-sys-token" type="password" autocomplete="off" minlength="20" placeholder="留空则保留现有 Token">', false) +
|
||||
formFieldHtml("iFinD Refresh Token", '<input id="m-sys-ifind" type="password" autocomplete="off" maxlength="2048" placeholder="留空则保留现有 Token">', false) +
|
||||
'<button class="m-btn-primary" type="button" data-system-save-market>保存密钥</button></div>' +
|
||||
formFieldHtml("行情来源凭据", '<input id="m-sys-token" type="text" value="请在数据中枢后台统一管理" disabled>', false) +
|
||||
formFieldHtml("实时来源凭据", '<input id="m-sys-ifind" type="text" value="请在数据中枢后台统一管理" disabled>', false) +
|
||||
'<button class="m-btn-primary" type="button" data-system-save-market>刷新状态</button></div>' +
|
||||
'<div class="m-card m-sys-section"><strong>后台刷新</strong>' +
|
||||
'<div class="m-sys-switch-row"><div><strong>交易时段自动刷新</strong><p class="m-sys-hint">开启后后台定时更新快照</p></div>' +
|
||||
'<button class="m-theme-switch" type="button" data-system-toggle-refresh role="switch" aria-checked="' + (data.background_refresh_enabled ? "true" : "false") + '" aria-label="交易时段自动刷新"><span class="m-theme-switch-thumb"></span></button></div>' +
|
||||
@@ -5540,14 +5550,10 @@
|
||||
function saveSystemMarket() {
|
||||
const button = document.querySelector("[data-system-save-market]");
|
||||
if (button) button.disabled = true;
|
||||
global.MobileAPI.request("/api/admin/settings", "POST", {
|
||||
tushare_token: ((document.getElementById("m-sys-token") || {}).value || "").trim(),
|
||||
ifind_refresh_token: ((document.getElementById("m-sys-ifind") || {}).value || "").trim(),
|
||||
}).then(function () {
|
||||
showToast("行情密钥已保存");
|
||||
loadSystem();
|
||||
loadSystem().then(function () {
|
||||
showToast("数据中枢状态已刷新");
|
||||
}).catch(function (error) {
|
||||
showToast(error && error.message ? error.message : "系统配置保存失败");
|
||||
showToast(error && error.message ? error.message : "数据中枢状态读取失败");
|
||||
}).then(function () {
|
||||
if (button) button.disabled = false;
|
||||
});
|
||||
|
||||
@@ -68,10 +68,10 @@
|
||||
"/pages/sentiment/page.js?v=20260729-1",
|
||||
"/pages/pools/page.js?v=20260820-1",
|
||||
"/pages/market/breadth.js?v=20260803-1",
|
||||
"/pages/market/charts.js?v=20260803-1",
|
||||
"/pages/market/entity-detail.js?v=20260803-1",
|
||||
"/pages/market/stock-detail.js?v=20260803-1",
|
||||
"/pages/market/preview.js?v=20260806-1",
|
||||
"/pages/market/charts.js?v=20260908-1",
|
||||
"/pages/market/entity-detail.js?v=20260908-1",
|
||||
"/pages/market/stock-detail.js?v=20260908-1",
|
||||
"/pages/market/preview.js?v=20260908-1",
|
||||
"/pages/market/search.js?v=20260803-1",
|
||||
"/pages/market/bindings.js?v=20260803-1",
|
||||
"/pages/ladder/page.js?v=20260820-1",
|
||||
|
||||
@@ -113,9 +113,12 @@ async function loadHeavenSetup(force = false, sector = "", stockCode = "") {
|
||||
document.querySelector("#resetHeavenCalibrationButton"),
|
||||
].filter(Boolean);
|
||||
cancelHeavenPerformance();
|
||||
heavenView?.classList.add("heaven-data-loading");
|
||||
const blocking = !state.heavenSetup;
|
||||
if (blocking) heavenView?.classList.add("heaven-data-loading");
|
||||
if (loadButton) loadButton.disabled = true;
|
||||
calibrationButtons.forEach((button) => { button.disabled = true; });
|
||||
const controller = new AbortController();
|
||||
const timeoutId = window.setTimeout(() => controller.abort(), 25_000);
|
||||
try {
|
||||
if (state.heavenSetup?.requestedKey && state.heavenSetup.requestedKey !== requestedKey) {
|
||||
state.personalField = null;
|
||||
@@ -126,7 +129,7 @@ async function loadHeavenSetup(force = false, sector = "", stockCode = "") {
|
||||
if (sector) query.set("sector", sector);
|
||||
if (stockCode) query.set("stock_code", stockCode);
|
||||
if (manualData) query.set("manual_data", JSON.stringify(manualData));
|
||||
const payload = await apiRequest(`/api/heaven/setup?${query}`);
|
||||
const payload = await apiRequest(`/api/heaven/setup?${query}`, "GET", null, { signal: controller.signal });
|
||||
if (
|
||||
requestSequence !== state.heavenRequestSequence
|
||||
|| calendarDate !== document.querySelector("#qiObservationDate")?.value
|
||||
@@ -152,9 +155,15 @@ async function loadHeavenSetup(force = false, sector = "", stockCode = "") {
|
||||
if (payload.chart.selection_notice) showHeavenNotice(payload.chart.selection_notice);
|
||||
} catch (error) {
|
||||
if (requestSequence !== state.heavenRequestSequence) return;
|
||||
showHeavenNotice(error.message || "问天数据加载失败");
|
||||
showToast(error.message || "问天数据加载失败");
|
||||
const aborted = error?.payload?.aborted || /abort|超时|cancel/i.test(String(error?.message || ""));
|
||||
const message = aborted
|
||||
? "问天数据仍在准备,页面可继续输入和操作"
|
||||
: (error.message || "问天数据加载失败");
|
||||
showHeavenNotice(message);
|
||||
if (!aborted) showToast(message);
|
||||
if (!state.heavenSetup) renderHeavenWorkspace();
|
||||
} finally {
|
||||
window.clearTimeout(timeoutId);
|
||||
if (requestSequence === state.heavenRequestSequence) {
|
||||
heavenView?.classList.remove("heaven-data-loading");
|
||||
if (loadButton) loadButton.disabled = false;
|
||||
|
||||
@@ -1,3 +1,16 @@
|
||||
const DAILY_CHART_BARS = 45;
|
||||
|
||||
function visibleDailyPrices(prices) {
|
||||
return (prices || []).slice(-DAILY_CHART_BARS);
|
||||
}
|
||||
|
||||
function dailyChartSourceLabel(prices, notice) {
|
||||
const count = visibleDailyPrices(prices).length;
|
||||
const base = `日 K 行情 · ${count} 个交易日`;
|
||||
const text = String(notice || "").trim();
|
||||
return text ? `${base} · ${text}` : base;
|
||||
}
|
||||
|
||||
function currentChartPalette() {
|
||||
const style = getComputedStyle(document.documentElement);
|
||||
const color = (token, fallback) => style.getPropertyValue(token).trim() || fallback;
|
||||
@@ -56,7 +69,8 @@ function drawCandlestick(context, x, item, priceY, candleWidth, palette = curren
|
||||
|
||||
function drawPriceChart(prices) {
|
||||
const canvas = elements.priceChart;
|
||||
if (!prices?.length) {
|
||||
const visible = visibleDailyPrices(prices);
|
||||
if (!visible.length) {
|
||||
clearPriceChart("暂无日 K 数据");
|
||||
return;
|
||||
}
|
||||
@@ -81,15 +95,15 @@ function drawPriceChart(prices) {
|
||||
const gap = 12;
|
||||
const priceBottom = height - bottom - volumeHeight - gap;
|
||||
const plotWidth = width - left - right;
|
||||
const highs = prices.map((item) => number(item.high));
|
||||
const lows = prices.map((item) => number(item.low));
|
||||
const highs = visible.map((item) => number(item.high));
|
||||
const lows = visible.map((item) => number(item.low));
|
||||
const maximum = Math.max(...highs);
|
||||
const minimum = Math.min(...lows);
|
||||
const range = Math.max(maximum - minimum, maximum * 0.01, 0.01);
|
||||
const volumes = prices.map((item) => number(item.volume));
|
||||
const volumes = visible.map((item) => number(item.volume));
|
||||
const maxVolume = Math.max(...volumes, 1);
|
||||
const priceY = (value) => top + (maximum - value) / range * (priceBottom - top);
|
||||
const step = plotWidth / prices.length;
|
||||
const step = plotWidth / visible.length;
|
||||
const candleWidth = clamp(step * 0.62, 2, 8);
|
||||
|
||||
context.strokeStyle = palette.grid;
|
||||
@@ -105,7 +119,7 @@ function drawPriceChart(prices) {
|
||||
context.fillText((maximum - range * line / 4).toFixed(2), left - 5, y + 4);
|
||||
}
|
||||
|
||||
prices.forEach((item, index) => {
|
||||
visible.forEach((item, index) => {
|
||||
const x = left + step * index + step / 2;
|
||||
const color = drawCandlestick(context, x, item, priceY, candleWidth, palette);
|
||||
const volumeBarHeight = number(item.volume) / maxVolume * volumeHeight;
|
||||
@@ -117,10 +131,10 @@ function drawPriceChart(prices) {
|
||||
|
||||
context.textAlign = "center";
|
||||
context.fillStyle = palette.axis;
|
||||
const labelIndexes = [0, Math.floor((prices.length - 1) / 2), prices.length - 1];
|
||||
const labelIndexes = [0, Math.floor((visible.length - 1) / 2), visible.length - 1];
|
||||
labelIndexes.forEach((index) => {
|
||||
const x = left + step * index + step / 2;
|
||||
context.fillText(String(prices[index].trade_date).slice(5), x, height - 5);
|
||||
context.fillText(String(visible[index].trade_date).slice(5), x, height - 5);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -301,7 +315,7 @@ function drawIntradayPreviewChart(points, dailyPrices, referenceClose = 0) {
|
||||
|
||||
function drawDailyPreviewChart(prices) {
|
||||
const { context, width, height, palette } = prepareStockPreviewCanvas();
|
||||
const visible = prices.slice(-45);
|
||||
const visible = visibleDailyPrices(prices);
|
||||
const visibleStart = prices.length - visible.length;
|
||||
const left = 45;
|
||||
const right = 10;
|
||||
|
||||
@@ -113,13 +113,13 @@ function renderEntityDetailMetrics(metrics) {
|
||||
}
|
||||
|
||||
function drawEntityDetailChart(series, canvas = elements.entityDetailChart) {
|
||||
const candles = (series || []).filter((item) => number(item.close) > 0).map((item) => {
|
||||
const candles = visibleDailyPrices((series || []).filter((item) => number(item.close) > 0).map((item) => {
|
||||
const close = number(item.close);
|
||||
const open = number(item.open) || close;
|
||||
const high = Math.max(number(item.high) || close, open, close);
|
||||
const low = Math.min(number(item.low) || close, open, close);
|
||||
return { ...item, open, high, low, close };
|
||||
});
|
||||
}));
|
||||
if (!candles.length) {
|
||||
clearEntityDetailChart("暂无日 K 数据", canvas);
|
||||
return;
|
||||
|
||||
@@ -367,7 +367,8 @@ function selectStockPreviewChart(chart) {
|
||||
}
|
||||
} else if ((payload.prices || []).length) {
|
||||
setText("stockPreviewDate", payload.meta?.trade_date || "最新行情");
|
||||
setText("stockPreviewSource", `日 K 行情 · ${payload.prices.length} 个交易日`);
|
||||
const notice = String(payload.meta?.notice || "").trim();
|
||||
setText("stockPreviewSource", dailyChartSourceLabel(payload.prices, notice));
|
||||
drawDailyPreviewChart(payload.prices);
|
||||
} else {
|
||||
setText("stockPreviewDate", payload.meta?.trade_date || "最新行情");
|
||||
|
||||
@@ -20,17 +20,9 @@ async function openStock(code, fallback = null) {
|
||||
setText("detailStreak", row.status === "涨停" ? streakLabel(row.streak) : row.status || "--");
|
||||
setText("detailReason", row.reason || "--");
|
||||
setText("detailSector", row.sector || "其他");
|
||||
setText("detailFirst", row.first_time || "--");
|
||||
setText("detailLast", row.last_time || "--");
|
||||
setText("detailOpen", `${number(row.open_times)} 次`);
|
||||
setText("detailTurnover", `${formatNumber(row.turnover_rate, 2)}%`);
|
||||
setText("detailAmount", `${formatNumber(row.amount_billion, 2)} 亿`);
|
||||
setText("detailSeal", `${formatNumber(row.seal_amount_million, 0)} 万`);
|
||||
setStockBoardFields(row);
|
||||
setText("chartSource", "正在加载行情");
|
||||
setText("flowNet", "--");
|
||||
setText("flowLarge", "--");
|
||||
setText("flowMedium", "--");
|
||||
setText("flowSmall", "--");
|
||||
renderMoneyflow({});
|
||||
document.querySelector("#reasonInput").value = row.reason || "";
|
||||
document.querySelector("#stockNoteContent").value = "";
|
||||
document.querySelector("#stockNotePlan").value = "";
|
||||
@@ -48,11 +40,13 @@ async function openStock(code, fallback = null) {
|
||||
setText("detailName", stock.name || row.name);
|
||||
setText("detailPrice", formatNumber(stock.price || row.price, 2));
|
||||
setText("detailChange", `${signed(stock.change ?? row.change)}%`);
|
||||
setStockBoardFields({ ...row, ...stock });
|
||||
renderMoneyflow(payload.moneyflow || {});
|
||||
renderStockNotes(payload.notes || []);
|
||||
updateWatchButton();
|
||||
if (state.stockDetailChartMode === "daily") {
|
||||
setText("chartSource", `日 K 行情 · ${payload.prices.length} 个交易日`);
|
||||
const notice = String(payload.meta?.notice || "").trim();
|
||||
setText("chartSource", dailyChartSourceLabel(payload.prices, notice));
|
||||
requestAnimationFrame(() => drawPriceChart(payload.prices || []));
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -69,7 +63,11 @@ async function selectStockDetailChart(mode) {
|
||||
syncDetailChartButtons("stock", selected);
|
||||
if (selected === "daily") {
|
||||
const prices = state.stockDetail?.prices || [];
|
||||
setText("chartSource", prices.length ? `日 K 行情 · ${prices.length} 个交易日` : "正在加载行情");
|
||||
const notice = String(state.stockDetail?.meta?.notice || "").trim();
|
||||
setText(
|
||||
"chartSource",
|
||||
prices.length ? dailyChartSourceLabel(prices, notice) : "正在加载行情",
|
||||
);
|
||||
if (prices.length) requestAnimationFrame(() => drawPriceChart(prices));
|
||||
else clearPriceChart("正在加载日 K 数据");
|
||||
return;
|
||||
@@ -111,6 +109,21 @@ function renderStockDetailIntraday(payload) {
|
||||
});
|
||||
}
|
||||
|
||||
function setStockBoardFields(row) {
|
||||
const firstTime = String(row.first_time || "").trim();
|
||||
const lastTime = String(row.last_time || "").trim();
|
||||
setText("detailFirst", firstTime && firstTime !== "--" ? firstTime : "--");
|
||||
setText("detailLast", lastTime && lastTime !== "--" ? lastTime : "--");
|
||||
setText("detailOpen", row.open_times === null || row.open_times === undefined || row.open_times === "" ? "--" : `${number(row.open_times)} 次`);
|
||||
setText("detailTurnover", presentMetric(row.turnover_rate) ? `${formatNumber(row.turnover_rate, 2)}%` : "--");
|
||||
setText("detailAmount", presentMetric(row.amount_billion) ? `${formatNumber(row.amount_billion, 2)} 亿` : "--");
|
||||
setText("detailSeal", presentMetric(row.seal_amount_million) ? `${formatNumber(row.seal_amount_million, 0)} 万` : "--");
|
||||
}
|
||||
|
||||
function presentMetric(value) {
|
||||
return meaningfulNumber(value) && Number(value) !== 0;
|
||||
}
|
||||
|
||||
function openActiveStockInHeaven() {
|
||||
const code = state.activeStock?.code;
|
||||
if (!/^\d{6}$/.test(String(code || ""))) return;
|
||||
|
||||
Vendored
+10
-1
@@ -100,7 +100,12 @@ body[data-active-view="mentorView"] .app-page-context span {
|
||||
}
|
||||
|
||||
#mentorView .mentor-page-header {
|
||||
display: none;
|
||||
flex: 0 0 auto;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
min-height: 34px;
|
||||
margin: 0 0 12px;
|
||||
padding: 0 16px;
|
||||
}
|
||||
|
||||
#mentorView .mentor-page-title {
|
||||
@@ -1360,6 +1365,10 @@ body[data-active-view="mentorView"] .app-page-context span {
|
||||
}
|
||||
|
||||
@media (max-width: 767px) {
|
||||
#mentorView.workspace-view {
|
||||
padding: calc(var(--page-pad-y) + 40px) 0 0;
|
||||
}
|
||||
|
||||
#mentorView .mentor-layout {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
overflow: visible;
|
||||
|
||||
@@ -408,8 +408,18 @@ async function saveReasonOverride(event) {
|
||||
}
|
||||
|
||||
function renderMoneyflow(flow) {
|
||||
for (const [id, value] of [["flowNet", flow.net_million], ["flowLarge", flow.large_million], ["flowMedium", flow.medium_million], ["flowSmall", flow.small_million]]) {
|
||||
const payload = flow || {};
|
||||
const available = payload.available !== false && [
|
||||
payload.net_million, payload.large_million, payload.medium_million, payload.small_million,
|
||||
].some((value) => value !== null && value !== undefined && value !== "");
|
||||
for (const [id, value] of [["flowNet", payload.net_million], ["flowLarge", payload.large_million], ["flowMedium", payload.medium_million], ["flowSmall", payload.small_million]]) {
|
||||
const element = document.getElementById(id);
|
||||
if (!element) continue;
|
||||
if (!available || value === null || value === undefined || value === "") {
|
||||
element.textContent = "--";
|
||||
element.className = "";
|
||||
continue;
|
||||
}
|
||||
element.textContent = formatMoneyMillion(value);
|
||||
element.className = changeClass(value);
|
||||
}
|
||||
|
||||
+137
-79
@@ -28,14 +28,6 @@
|
||||
background: var(--selected);
|
||||
}
|
||||
|
||||
.rotation-sector-chip.selected {
|
||||
border-color: var(--action);
|
||||
|
||||
background: var(--action-soft);
|
||||
|
||||
color: var(--action-hover);
|
||||
}
|
||||
|
||||
.rotation-sector-chip > span {
|
||||
color: var(--text-secondary);
|
||||
|
||||
@@ -158,8 +150,6 @@
|
||||
cursor: pointer;
|
||||
|
||||
border-bottom: 1px dashed var(--border);
|
||||
|
||||
background: color-mix(in srgb, var(--accent) calc(4% + var(--rotation-heat) * 46%), transparent);
|
||||
}
|
||||
|
||||
.redesigned-rotation-view {
|
||||
@@ -397,8 +387,12 @@
|
||||
|
||||
display: flex;
|
||||
|
||||
flex-wrap: wrap;
|
||||
|
||||
align-items: center;
|
||||
|
||||
row-gap: 4px;
|
||||
|
||||
gap: 14px;
|
||||
|
||||
padding: 8px 14px;
|
||||
@@ -434,18 +428,32 @@
|
||||
border-radius: 2px;
|
||||
}
|
||||
|
||||
#rotationView .rotation-swatch.strong {
|
||||
background: var(--accent);
|
||||
#rotationView .rotation-swatch.tier-1 {
|
||||
background: var(--rotation-tier1-bar);
|
||||
}
|
||||
|
||||
#rotationView .rotation-swatch.warm {
|
||||
background: var(--accent-soft);
|
||||
#rotationView .rotation-swatch.tier-2 {
|
||||
background: var(--rotation-tier2-bar);
|
||||
}
|
||||
|
||||
#rotationView .rotation-swatch.mild {
|
||||
background: var(--surface-muted);
|
||||
#rotationView .rotation-swatch.tier-3 {
|
||||
background: var(--rotation-tier3-bar);
|
||||
}
|
||||
|
||||
border: 1px solid var(--border);
|
||||
#rotationView .rotation-swatch.tier-4 {
|
||||
background: var(--rotation-tier4-bar);
|
||||
}
|
||||
|
||||
#rotationView .rotation-swatch.tier-5 {
|
||||
background: var(--rotation-tier5-bar);
|
||||
}
|
||||
|
||||
#rotationView .rotation-legend-note {
|
||||
margin-left: auto;
|
||||
|
||||
color: var(--r2-faint);
|
||||
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
#rotationView .rotation-tracker {
|
||||
@@ -818,10 +826,59 @@
|
||||
|
||||
#rotationView .rotation-history.tracking .rotation-sector-chip.selected {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
border-color: var(--r2-blue);
|
||||
/* 追踪态:命中被追踪板块名、但不是当次点击的那一格 —— 保持档位底色,2px 同档位描边(含 80% 不透明度),
|
||||
连点标记不做动画,仅描边淡入 200ms */
|
||||
#rotationView .rotation-sector-chip.selected:not(.current-cell) {
|
||||
border-width: 2px;
|
||||
|
||||
box-shadow: inset 3px 0 0 var(--r2-blue), 0 0 0 1px var(--accent-soft);
|
||||
border-color: color-mix(in srgb, var(--tier-bar) 80%, transparent);
|
||||
|
||||
transition: border-color 200ms var(--ease-out);
|
||||
}
|
||||
|
||||
/* 选中态:仅当次点击的日期+板块这一格 —— 同档位实色渐变 + 2px 亮描边 + 克制光晕,全页最多一张 */
|
||||
#rotationView .rotation-sector-chip.selected.current-cell {
|
||||
border-width: 2px;
|
||||
|
||||
border-color: var(--tier-text);
|
||||
|
||||
background: linear-gradient(135deg, var(--tier-fill-start), var(--tier-fill-end));
|
||||
|
||||
box-shadow: 0 0 16px color-mix(in srgb, var(--tier-bar) var(--rotation-tier-glow-alpha), transparent);
|
||||
|
||||
color: var(--text-inverse);
|
||||
|
||||
transition: background 160ms var(--ease-out), border-color 160ms var(--ease-out), box-shadow 160ms var(--ease-out);
|
||||
}
|
||||
|
||||
#rotationView .rotation-sector-chip.selected.current-cell strong,
|
||||
#rotationView .rotation-sector-chip.selected.current-cell small,
|
||||
#rotationView .rotation-sector-chip.selected.current-cell small b {
|
||||
color: var(--text-inverse);
|
||||
}
|
||||
|
||||
#rotationView .rotation-track-dot {
|
||||
width: 6px;
|
||||
|
||||
height: 6px;
|
||||
|
||||
position: absolute;
|
||||
|
||||
top: 6px;
|
||||
|
||||
right: 6px;
|
||||
|
||||
border-radius: 50%;
|
||||
|
||||
background: var(--tier-bar);
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
#rotationView .rotation-sector-chip.selected.current-cell {
|
||||
box-shadow: none;
|
||||
}
|
||||
}
|
||||
|
||||
.rotation-cell-tooltip {
|
||||
@@ -1050,40 +1107,79 @@
|
||||
}
|
||||
}
|
||||
|
||||
#rotationView .rotation-sector-chip.heat-mild {
|
||||
border-color: var(--border);
|
||||
|
||||
background: var(--surface-muted);
|
||||
|
||||
box-shadow: none;
|
||||
/* HEL-524: 五档强度色 —— 同一强度值在 9 个日期列中颜色完全一致,仅代表强度档位,与涨跌无关。
|
||||
普通卡只用低透明着色,页面/主容器/日期分栏的层级由 HEL-522 的明度骨架承担,这里不重复叠加。 */
|
||||
#rotationView .rotation-sector-chip.heat-tier-1 {
|
||||
--tier-bar: var(--rotation-tier1-bar);
|
||||
--tier-bg: var(--rotation-tier1-bg);
|
||||
--tier-border: var(--rotation-tier1-border);
|
||||
--tier-text: var(--rotation-tier1-text);
|
||||
--tier-fill-start: var(--rotation-tier1-fill-start);
|
||||
--tier-fill-end: var(--rotation-tier1-fill-end);
|
||||
}
|
||||
|
||||
#rotationView .rotation-sector-chip.heat-strong {
|
||||
border-color: var(--action-line);
|
||||
|
||||
background: var(--accent-soft);
|
||||
|
||||
box-shadow: none;
|
||||
#rotationView .rotation-sector-chip.heat-tier-2 {
|
||||
--tier-bar: var(--rotation-tier2-bar);
|
||||
--tier-bg: var(--rotation-tier2-bg);
|
||||
--tier-border: var(--rotation-tier2-border);
|
||||
--tier-text: var(--rotation-tier2-text);
|
||||
--tier-fill-start: var(--rotation-tier2-fill-start);
|
||||
--tier-fill-end: var(--rotation-tier2-fill-end);
|
||||
}
|
||||
|
||||
#rotationView .rotation-sector-chip.heat-warm {
|
||||
border-color: var(--action-line);
|
||||
|
||||
background: color-mix(in srgb, var(--accent) 12%, var(--surface));
|
||||
|
||||
box-shadow: none;
|
||||
#rotationView .rotation-sector-chip.heat-tier-3 {
|
||||
--tier-bar: var(--rotation-tier3-bar);
|
||||
--tier-bg: var(--rotation-tier3-bg);
|
||||
--tier-border: var(--rotation-tier3-border);
|
||||
--tier-text: var(--rotation-tier3-text);
|
||||
--tier-fill-start: var(--rotation-tier3-fill-start);
|
||||
--tier-fill-end: var(--rotation-tier3-fill-end);
|
||||
}
|
||||
|
||||
#rotationView .rotation-sector-chip:hover {
|
||||
#rotationView .rotation-sector-chip.heat-tier-4 {
|
||||
--tier-bar: var(--rotation-tier4-bar);
|
||||
--tier-bg: var(--rotation-tier4-bg);
|
||||
--tier-border: var(--rotation-tier4-border);
|
||||
--tier-text: var(--rotation-tier4-text);
|
||||
--tier-fill-start: var(--rotation-tier4-fill-start);
|
||||
--tier-fill-end: var(--rotation-tier4-fill-end);
|
||||
}
|
||||
|
||||
#rotationView .rotation-sector-chip.heat-tier-5 {
|
||||
--tier-bar: var(--rotation-tier5-bar);
|
||||
--tier-bg: var(--rotation-tier5-bg);
|
||||
--tier-border: var(--rotation-tier5-border);
|
||||
--tier-text: var(--rotation-tier5-text);
|
||||
--tier-fill-start: var(--rotation-tier5-fill-start);
|
||||
--tier-fill-end: var(--rotation-tier5-fill-end);
|
||||
}
|
||||
|
||||
#rotationView .rotation-sector-chip[class*="heat-tier-"] {
|
||||
border-color: var(--tier-border);
|
||||
|
||||
background: var(--tier-bg);
|
||||
|
||||
box-shadow: inset 4px 0 0 0 var(--tier-bar);
|
||||
|
||||
transition: transform 280ms cubic-bezier(0.22, 1, 0.36, 1), background-color 120ms var(--ease-out), border-color 120ms var(--ease-out), box-shadow 120ms var(--ease-out), filter 240ms, opacity 220ms;
|
||||
}
|
||||
|
||||
#rotationView .rotation-sector-chip[class*="heat-tier-"] .rotation-strength {
|
||||
color: var(--tier-text);
|
||||
}
|
||||
|
||||
/* 悬停态:卡底与描边 alpha 适度提升,120ms ease-out,可打断,不打乱档位色含义;
|
||||
已经处于选中/追踪态的卡片保持其自身样式,不叠加普通悬停处理 */
|
||||
#rotationView .rotation-sector-chip[class*="heat-tier-"]:hover:not(.selected) {
|
||||
z-index: 6;
|
||||
|
||||
border-color: var(--accent);
|
||||
background: color-mix(in srgb, var(--tier-bar) 8%, var(--tier-bg) 92%);
|
||||
|
||||
border-color: color-mix(in srgb, var(--tier-bar) 26%, var(--tier-border) 74%);
|
||||
|
||||
filter: saturate(1.06);
|
||||
|
||||
transform: translate3d(2px, -2px, 0px) scale(1.015);
|
||||
|
||||
box-shadow: none;
|
||||
}
|
||||
|
||||
#rotationView .rotation-table {
|
||||
@@ -1207,30 +1303,6 @@
|
||||
box-shadow: var(--control-shadow);
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] #rotationView .rotation-sector-chip.heat-strong {
|
||||
border-color: var(--blue-line);
|
||||
|
||||
background: var(--heat-strong-bg);
|
||||
|
||||
color: var(--heat-strong-ink);
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] #rotationView .rotation-sector-chip.heat-warm {
|
||||
border-color: var(--border-strong);
|
||||
|
||||
background: var(--heat-warm-bg);
|
||||
|
||||
color: var(--heat-warm-ink);
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] #rotationView .rotation-sector-chip.heat-mild {
|
||||
border-color: var(--border);
|
||||
|
||||
background: var(--heat-mild-bg);
|
||||
|
||||
color: var(--heat-mild-ink);
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] #rotationView :is(.rotation-rank, .rotation-table thead th, .trend-flat) {
|
||||
border-color: var(--border);
|
||||
|
||||
@@ -1243,20 +1315,6 @@
|
||||
background: var(--action-soft);
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] #rotationView :is(.rotation-swatch.warm) {
|
||||
border-color: var(--blue-line);
|
||||
|
||||
background: var(--action-soft);
|
||||
|
||||
color: var(--action);
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] #rotationView :is(.rotation-swatch.mild) {
|
||||
border-color: var(--border);
|
||||
|
||||
background: var(--surface-muted);
|
||||
}
|
||||
|
||||
@media (max-width: 767px) {
|
||||
#rotationView :is(.rotation-table-frame, .rotation-detail-card) {
|
||||
max-height: none;
|
||||
|
||||
@@ -22,9 +22,12 @@
|
||||
<span class="rotation-top-tag">每日 Top 12 热点</span>
|
||||
</header>
|
||||
<div class="rotation-legend" aria-label="板块强度图例">
|
||||
<span><i class="rotation-swatch strong"></i>强度高(90+)</span>
|
||||
<span><i class="rotation-swatch warm"></i>强度中(70–89)</span>
|
||||
<span><i class="rotation-swatch mild"></i>强度低(<70)</span>
|
||||
<span><i class="rotation-swatch tier-1"></i>低·青(<65)</span>
|
||||
<span><i class="rotation-swatch tier-2"></i>中低·蓝(65–74)</span>
|
||||
<span><i class="rotation-swatch tier-3"></i>中高·紫(75–84)</span>
|
||||
<span><i class="rotation-swatch tier-4"></i>高·橙(85–94)</span>
|
||||
<span><i class="rotation-swatch tier-5"></i>最高·红(95+)</span>
|
||||
<span class="rotation-legend-note">颜色=强度档位,与涨跌无关</span>
|
||||
</div>
|
||||
<div id="rotationTracker" class="rotation-tracker" hidden></div>
|
||||
<div id="rotationHistory" class="rotation-history"><div class="empty-state">正在读取轮动历史</div></div>
|
||||
|
||||
@@ -3,6 +3,16 @@ window.XiaobaiPageModules.register("rotation", ["rotationView"], {
|
||||
enter: ["loadRotation"],
|
||||
});
|
||||
|
||||
// HEL-524: 五档强度色固定映射,所有日期列共用同一套阈值 —— <65 青,65-74 蓝,75-84 紫,85-94 橙,95+ 红。
|
||||
function rotationHeatTier(strengthValue) {
|
||||
const value = clamp(number(strengthValue), 0, 100);
|
||||
if (value >= 95) return 5;
|
||||
if (value >= 85) return 4;
|
||||
if (value >= 75) return 3;
|
||||
if (value >= 65) return 2;
|
||||
return 1;
|
||||
}
|
||||
|
||||
async function loadRotationHistory(force = false) {
|
||||
if (!state.dashboard || state.rotationLoading) return;
|
||||
const key = `${elements.tradeDate.value}:9`;
|
||||
@@ -87,10 +97,14 @@ function renderRotationHistory() {
|
||||
<header><time>${escapeHtml(displayCompactDate(day.trade_date).slice(5))}</time><span>${(day.sectors || []).length} 个热点</span></header>
|
||||
<div class="rotation-day-sectors">${(day.sectors || []).map((sector) => {
|
||||
const strength = clamp(number(sector.strength), 0, 100);
|
||||
const heatClass = strength >= 90 ? "heat-strong" : strength >= 70 ? "heat-warm" : "heat-mild";
|
||||
const heatClass = `heat-tier-${rotationHeatTier(strength)}`;
|
||||
const isTrackedName = Boolean(selected) && selected === sector.name;
|
||||
const isCurrentCell = isTrackedName && day.trade_date === state.rotationSelectedDate;
|
||||
const stateClass = isCurrentCell ? "selected current-cell" : isTrackedName ? "selected" : "";
|
||||
return `
|
||||
<button type="button" class="rotation-sector-chip ${heatClass} ${selected === sector.name ? "selected" : ""}" data-rotation-sector="${escapeHtml(sector.name)}" data-rotation-date="${escapeHtml(day.trade_date)}">
|
||||
<span class="rotation-rank rank-${Math.min(number(sector.rank), 4)}">${number(sector.rank)}</span><strong>${escapeHtml(sector.name)}</strong><small><b>${number(sector.count)}</b> 家 · ${formatNumber(sector.strength, 0)}</small>
|
||||
<button type="button" class="rotation-sector-chip ${heatClass} ${stateClass}" data-rotation-sector="${escapeHtml(sector.name)}" data-rotation-date="${escapeHtml(day.trade_date)}">
|
||||
<span class="rotation-rank rank-${Math.min(number(sector.rank), 4)}">${number(sector.rank)}</span><strong>${escapeHtml(sector.name)}</strong><small><b>${number(sector.count)}</b> 家 · <b class="rotation-strength">${formatNumber(sector.strength, 0)}</b></small>
|
||||
${isTrackedName && !isCurrentCell ? '<span class="rotation-track-dot" aria-hidden="true"></span>' : ""}
|
||||
<span class="rotation-cell-tooltip">${escapeHtml(displayCompactDate(day.trade_date).slice(5))} · 第 ${number(sector.rank)} 名 · 涨停 ${number(sector.count)} 家 · 强度 ${formatNumber(sector.strength, 0)}</span>
|
||||
</button>`;
|
||||
}).join("")}</div>
|
||||
|
||||
+17
-11
@@ -2287,8 +2287,6 @@ body[data-active-view="screenerView"] .workspace-view {
|
||||
box-shadow: none;
|
||||
|
||||
overflow: visible;
|
||||
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
:where(#screenerView) .curated-library-pane {
|
||||
@@ -2546,7 +2544,7 @@ body[data-active-view="screenerView"] .workspace-view {
|
||||
|
||||
flex: 0 0 auto;
|
||||
|
||||
gap: 2px;
|
||||
gap: 8px;
|
||||
|
||||
margin: 0px;
|
||||
|
||||
@@ -2566,11 +2564,11 @@ body[data-active-view="screenerView"] .workspace-view {
|
||||
|
||||
padding: 0 14px;
|
||||
|
||||
border: 0px;
|
||||
border: 1px solid var(--control-border);
|
||||
|
||||
border-radius: 0;
|
||||
border-radius: var(--radius-md);
|
||||
|
||||
background: transparent;
|
||||
background: var(--control-surface);
|
||||
|
||||
color: var(--r2-sub);
|
||||
|
||||
@@ -2586,15 +2584,19 @@ body[data-active-view="screenerView"] .workspace-view {
|
||||
}
|
||||
|
||||
#screenerView .screener-mode-tabs button:hover {
|
||||
background: var(--surface-hover);
|
||||
border-color: var(--r2-blue-line);
|
||||
|
||||
background: var(--control-hover);
|
||||
|
||||
color: var(--r2-ink);
|
||||
}
|
||||
|
||||
#screenerView .screener-mode-tabs button.active {
|
||||
background: transparent;
|
||||
border-color: var(--scr-blue);
|
||||
|
||||
color: var(--scr-blue);
|
||||
background: var(--scr-blue);
|
||||
|
||||
color: var(--on-action);
|
||||
|
||||
font-weight: var(--font-weight-semibold);
|
||||
}
|
||||
@@ -3135,7 +3137,11 @@ body[data-active-view="screenerView"] .workspace-view {
|
||||
}
|
||||
|
||||
#screenerView .curated-screener-panel {
|
||||
display: block;
|
||||
display: flex;
|
||||
|
||||
flex-direction: column;
|
||||
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
#screenerView .curated-workspace {
|
||||
@@ -6184,7 +6190,7 @@ body[data-active-view="screenerView"] .workspace-view {
|
||||
background: var(--surface-muted);
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] #screenerView :is(.screener-mode-tabs, .screener-stepper, .regime-selector, .regime-option, .screener-results-view, .result-toolbar) {
|
||||
:root[data-theme="dark"] #screenerView :is(.screener-stepper, .regime-selector, .regime-option, .screener-results-view, .result-toolbar) {
|
||||
border-color: var(--border);
|
||||
|
||||
background: var(--surface);
|
||||
|
||||
+2
-2
@@ -1596,7 +1596,7 @@
|
||||
background: var(--surface);
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] :is(.sentiment-cycle-toolbar, .tbl-tools) {
|
||||
:root[data-theme="dark"] :is(.tbl-tools) {
|
||||
border-color: var(--line-soft);
|
||||
|
||||
background: var(--surface);
|
||||
@@ -1610,7 +1610,7 @@
|
||||
color: var(--warning-color);
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] #sentimentCycleView :is(.sentiment-current-tag, .sentiment-auto-tag, .sentiment-detail-toolbar, .section-toolbar) {
|
||||
:root[data-theme="dark"] #sentimentCycleView :is(.sentiment-current-tag, .sentiment-auto-tag, .sentiment-detail-toolbar) {
|
||||
border-color: var(--border);
|
||||
|
||||
background: var(--surface-muted);
|
||||
|
||||
@@ -41,11 +41,10 @@ async function openAdminSettings(refreshOnly = false) {
|
||||
const ifind = data.ifind || {};
|
||||
const llm = payload.llm || {};
|
||||
const membership = payload.membership || {};
|
||||
status.textContent = `Tushare ${data.configured ? "已配置" : "未配置"} · iFinD ${ifind.configured ? "已配置" : "未配置"} · ${number(data.snapshot_dates)} 个交易日`;
|
||||
status.textContent = `数据中枢 ${data.configured ? "已连接" : "未连接"} · iFinD ${ifind.configured ? "已配置" : "未配置"} · ${number(data.snapshot_dates)} 个交易日`;
|
||||
status.classList.toggle("connected", Boolean(data.configured));
|
||||
setText("systemDataStatus", data.background_refresh_enabled ? "后台刷新已启用" : "后台刷新已暂停");
|
||||
document.querySelector("#systemTokenInput").value = "";
|
||||
document.querySelector("#systemIfindTokenInput").value = "";
|
||||
renderDatahubRouteStatus(data.datahub || {});
|
||||
document.querySelector("#systemBackgroundRefresh").checked = Boolean(data.background_refresh_enabled);
|
||||
document.querySelector("#memberDailyLimit").value = number(membership.member_daily_limit) || 50;
|
||||
renderModelPool(llm.models || [], llm.primary_model_id || "", llm.fallback_model_id || "");
|
||||
@@ -55,6 +54,26 @@ async function openAdminSettings(refreshOnly = false) {
|
||||
}
|
||||
}
|
||||
|
||||
function renderDatahubRouteStatus(hub) {
|
||||
const box = document.querySelector("#datahubRouteStatus");
|
||||
if (!box) return;
|
||||
const label = box.querySelector("span");
|
||||
const enabled = number(hub.enabled_reads);
|
||||
const total = number(hub.total_reads) || enabled;
|
||||
const fallbacks = hub.fallback_labels || [];
|
||||
if (fallbacks.length) {
|
||||
box.dataset.tone = "warning";
|
||||
if (label) label.textContent = `数据中枢主线路 ${enabled}/${total} · 备用 ${fallbacks.length} 类:${fallbacks.join("、")}`;
|
||||
return;
|
||||
}
|
||||
box.dataset.tone = hub.configured ? "success" : "idle";
|
||||
if (label) {
|
||||
label.textContent = hub.configured
|
||||
? `数据中枢主线路 ${enabled}/${total},当前无备用`
|
||||
: "数据中枢未配置,网站只保留已有真实快照";
|
||||
}
|
||||
}
|
||||
|
||||
function selectAdminPanel(panel) {
|
||||
const selected = ["market", "models", "members"].includes(panel) ? panel : "market";
|
||||
document.querySelector("#adminSectionSelect").value = selected;
|
||||
@@ -184,13 +203,9 @@ async function saveMarketSettings(event) {
|
||||
button.disabled = true;
|
||||
try {
|
||||
await apiRequest("/api/admin/settings", "POST", {
|
||||
tushare_token: document.querySelector("#systemTokenInput").value.trim(),
|
||||
ifind_refresh_token: document.querySelector("#systemIfindTokenInput").value.trim(),
|
||||
background_refresh_enabled: document.querySelector("#systemBackgroundRefresh").checked,
|
||||
});
|
||||
document.querySelector("#systemTokenInput").value = "";
|
||||
document.querySelector("#systemIfindTokenInput").value = "";
|
||||
showToast("行情配置已保存");
|
||||
showToast("行情刷新设置已保存");
|
||||
await openAdminSettings(true);
|
||||
} catch (error) {
|
||||
showToast(error.message || "系统配置保存失败");
|
||||
|
||||
@@ -62,6 +62,9 @@
|
||||
try {
|
||||
response = await fetch(url, requestOptions(method, body, options.signal));
|
||||
} catch (error) {
|
||||
if (error?.name === "AbortError") {
|
||||
throw new ApiError("请求已取消或超时", 0, { aborted: true });
|
||||
}
|
||||
throw new ApiError(readableRequestError(error), 0, null);
|
||||
}
|
||||
const payload = await parseJson(response);
|
||||
|
||||
@@ -67,6 +67,11 @@ async function startAdminRefresh() {
|
||||
const actualCompact = actualDate.replaceAll("-", "");
|
||||
const updated = formatTimestamp(meta.updated_at);
|
||||
const freshness = dashboardFreshnessMessage(meta);
|
||||
if (meta.realtime && actualCompact === requestedCompact && !meta.carried_forward) {
|
||||
setAdminRefreshStatus("success", `刷新成功:已获取 ${actualDate} 的盘中行情,更新时间 ${updated}`, "circle-check");
|
||||
showToast(`刷新成功:已获取 ${actualDate} 的盘中行情`);
|
||||
return;
|
||||
}
|
||||
if (freshness || actualCompact !== requestedCompact || meta.carried_forward || meta.limit_data_source === "derived") {
|
||||
setAdminRefreshStatus("warning", freshness || `部分正式数据尚未到齐,当前展示 ${actualDate || "最近可用数据"}`, "triangle-alert");
|
||||
setStatus(freshness || "部分正式数据尚未到齐,当前展示最近可用数据");
|
||||
|
||||
@@ -3765,3 +3765,30 @@ body.mobile-shell[data-active-view="heavenView"] .app-header .overview-strip[dat
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* HEL-524: 侧栏去掉作为整块独立色板的背景,融入页面底层;品牌区与顶栏保持同层;
|
||||
收起侧栏整行不再单独着色。展开/收起态均生效。「问天」冻结路由维持原有外观。 */
|
||||
body:not(.mobile-shell):not([data-active-view="heavenView"]) .module-nav,
|
||||
body.sidebar-collapsed:not(.mobile-shell):not([data-active-view="heavenView"]) .module-nav {
|
||||
background: var(--canvas);
|
||||
}
|
||||
|
||||
body:not(.mobile-shell):not([data-active-view="heavenView"]) .sidebar-brand {
|
||||
background: var(--header-bg);
|
||||
}
|
||||
|
||||
body:not(.mobile-shell):not([data-active-view="heavenView"]) .sidebar-collapse-button {
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] body:not(.mobile-shell):not([data-active-view="heavenView"]) .module-nav {
|
||||
background: var(--canvas);
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] body:not(.mobile-shell):not([data-active-view="heavenView"]) .sidebar-brand {
|
||||
background-color: var(--header-bg);
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] body:not(.mobile-shell):not([data-active-view="heavenView"]) :is(.nav-group, .sidebar-collapse-button) {
|
||||
background-color: transparent;
|
||||
}
|
||||
|
||||
|
||||
@@ -381,6 +381,40 @@
|
||||
--r2-radius: var(--size-radius-lg);
|
||||
--r2-shadow: var(--elevation-card);
|
||||
|
||||
/* HEL-524: 板块轮动五档强度色(日间)。五档单调递进:青→蓝→紫→橙→红,冷到暖;
|
||||
同一强度值在所有日期列使用完全相同的颜色,颜色仅代表强度档位,与涨跌无关。 */
|
||||
--rotation-tier1-bar: #0D9488;
|
||||
--rotation-tier1-bg: rgba(13, 148, 136, .08);
|
||||
--rotation-tier1-border: rgba(13, 148, 136, .25);
|
||||
--rotation-tier1-text: #0F766E;
|
||||
--rotation-tier1-fill-start: #11766D;
|
||||
--rotation-tier1-fill-end: #0C443F;
|
||||
--rotation-tier2-bar: #2563EB;
|
||||
--rotation-tier2-bg: rgba(37, 99, 235, .07);
|
||||
--rotation-tier2-border: rgba(37, 99, 235, .22);
|
||||
--rotation-tier2-text: #1D4ED8;
|
||||
--rotation-tier2-fill-start: #2059D7;
|
||||
--rotation-tier2-fill-end: #1D46A2;
|
||||
--rotation-tier3-bar: #7C3AED;
|
||||
--rotation-tier3-bg: rgba(124, 58, 237, .07);
|
||||
--rotation-tier3-border: rgba(124, 58, 237, .22);
|
||||
--rotation-tier3-text: #6D28D9;
|
||||
--rotation-tier3-fill-start: #6F2DE0;
|
||||
--rotation-tier3-fill-end: #5720B5;
|
||||
--rotation-tier4-bar: #EA580C;
|
||||
--rotation-tier4-bg: rgba(234, 88, 12, .08);
|
||||
--rotation-tier4-border: rgba(234, 88, 12, .25);
|
||||
--rotation-tier4-text: #C2410C;
|
||||
--rotation-tier4-fill-start: #C85215;
|
||||
--rotation-tier4-fill-end: #913E13;
|
||||
--rotation-tier5-bar: #DC2626;
|
||||
--rotation-tier5-bg: rgba(220, 38, 38, .08);
|
||||
--rotation-tier5-border: rgba(220, 38, 38, .25);
|
||||
--rotation-tier5-text: #B91C1C;
|
||||
--rotation-tier5-fill-start: #BE2B2B;
|
||||
--rotation-tier5-fill-end: #8C2424;
|
||||
--rotation-tier-glow-alpha: 22%;
|
||||
|
||||
font-family: "PingFang SC", "Microsoft YaHei", system-ui, sans-serif;
|
||||
font-size: 14px;
|
||||
}
|
||||
@@ -557,6 +591,40 @@
|
||||
--heat-warm-ink: #dfeaf7;
|
||||
--heat-mild-bg: #293440;
|
||||
--heat-mild-ink: #c7d2dc;
|
||||
|
||||
/* HEL-524: 板块轮动五档强度色(夜间)。五档单调递进:青→蓝→紫→橙→红,冷到暖;
|
||||
同一强度值在所有日期列使用完全相同的颜色,颜色仅代表强度档位,与涨跌无关。 */
|
||||
--rotation-tier1-bar: #2DD4BF;
|
||||
--rotation-tier1-bg: rgba(45, 212, 191, .09);
|
||||
--rotation-tier1-border: rgba(45, 212, 191, .20);
|
||||
--rotation-tier1-text: #5EEAD4;
|
||||
--rotation-tier1-fill-start: #32B6A5;
|
||||
--rotation-tier1-fill-end: #2A867A;
|
||||
--rotation-tier2-bar: #60A5FA;
|
||||
--rotation-tier2-bg: rgba(96, 165, 250, .10);
|
||||
--rotation-tier2-border: rgba(96, 165, 250, .22);
|
||||
--rotation-tier2-text: #93C5FD;
|
||||
--rotation-tier2-fill-start: #5098F1;
|
||||
--rotation-tier2-fill-end: #217AE7;
|
||||
--rotation-tier3-bar: #A78BFA;
|
||||
--rotation-tier3-bg: rgba(167, 139, 250, .11);
|
||||
--rotation-tier3-border: rgba(167, 139, 250, .24);
|
||||
--rotation-tier3-text: #C4B5FD;
|
||||
--rotation-tier3-fill-start: #9879F2;
|
||||
--rotation-tier3-fill-end: #724AE9;
|
||||
--rotation-tier4-bar: #FB923C;
|
||||
--rotation-tier4-bg: rgba(251, 146, 60, .12);
|
||||
--rotation-tier4-border: rgba(251, 146, 60, .26);
|
||||
--rotation-tier4-text: #FDBA74;
|
||||
--rotation-tier4-fill-start: #F07C2D;
|
||||
--rotation-tier4-fill-end: #D05A14;
|
||||
--rotation-tier5-bar: #F87171;
|
||||
--rotation-tier5-bg: rgba(248, 113, 113, .13);
|
||||
--rotation-tier5-border: rgba(248, 113, 113, .28);
|
||||
--rotation-tier5-text: #FCA5A5;
|
||||
--rotation-tier5-fill-start: #EF6161;
|
||||
--rotation-tier5-fill-end: #E53333;
|
||||
--rotation-tier-glow-alpha: 35%;
|
||||
--heaven-field-bg: #23241f;
|
||||
--shadow-soft: 0 1px 2px rgba(0, 0, 0, .28), 0 8px 24px rgba(0, 0, 0, .16);
|
||||
--shadow: 0 18px 50px rgba(0, 0, 0, .46);
|
||||
|
||||
@@ -1260,9 +1260,9 @@ test("sector rotation transfers the nine-day matrix, tracking and sortable detai
|
||||
await expect(page.locator("#rotationHistory .rotation-day").last()).toHaveClass(/latest-day/);
|
||||
await expect(page.locator("#rotationView .rotation-legend")).not.toContainText("单元格 =");
|
||||
const firstDayCells = page.locator("#rotationHistory .rotation-day").first().locator(".rotation-sector-chip");
|
||||
await expect(firstDayCells.nth(0)).toHaveClass(/heat-strong/);
|
||||
await expect(firstDayCells.nth(1)).toHaveClass(/heat-warm/);
|
||||
await expect(firstDayCells.nth(2)).toHaveClass(/heat-mild/);
|
||||
await expect(firstDayCells.nth(0)).toHaveClass(/heat-tier-4/); // strength 92 → 85-94 档(高·橙)
|
||||
await expect(firstDayCells.nth(1)).toHaveClass(/heat-tier-3/); // strength 76 → 75-84 档(中高·紫)
|
||||
await expect(firstDayCells.nth(2)).toHaveClass(/heat-tier-1/); // strength 58 → <65 档(低·青)
|
||||
const cellVisuals = await firstDayCells.evaluateAll((cells) => cells.map((cell) => {
|
||||
const style = getComputedStyle(cell);
|
||||
return { background: style.backgroundColor, radius: parseFloat(style.borderRadius), duration: style.transitionDuration };
|
||||
|
||||
@@ -3,7 +3,8 @@ from __future__ import annotations
|
||||
import copy
|
||||
import threading
|
||||
import unittest
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from datetime import date, datetime, timedelta, timezone, time as dt_time
|
||||
from unittest.mock import patch
|
||||
from pathlib import Path
|
||||
|
||||
from backend.features.market.service import MarketServiceMixin
|
||||
@@ -105,18 +106,84 @@ class FakeDerivedClient:
|
||||
}
|
||||
|
||||
|
||||
SHANGHAI = timezone(timedelta(hours=8))
|
||||
TRADE_DAY = date(2026, 9, 8)
|
||||
|
||||
|
||||
def at_clock(hour: int, minute: int, day: date = TRADE_DAY) -> datetime:
|
||||
return datetime(day.year, day.month, day.day, hour, minute, tzinfo=SHANGHAI)
|
||||
|
||||
|
||||
class FakeMissingDailyClient:
|
||||
def __init__(self, open_today: bool = True):
|
||||
self.open_today = open_today
|
||||
|
||||
def dashboard(self, trade_date: str):
|
||||
raise TushareError(f"No daily data returned for {trade_date}")
|
||||
|
||||
def resolve_trade_context(self, requested: str):
|
||||
if self.open_today:
|
||||
return requested, "20260907"
|
||||
return "20260907", "20260904"
|
||||
|
||||
|
||||
class FakeRealtimeTodayClient:
|
||||
def dashboard(self, trade_date: str):
|
||||
return {
|
||||
"meta": {
|
||||
"trade_date": f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:8]}",
|
||||
"requested_date": f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:8]}",
|
||||
"realtime": True,
|
||||
"mode": "realtime",
|
||||
"market_status": "trading",
|
||||
"notice": "盘中行情由 Tushare rt_k 实时计算;涨停原因、封板时间和开板次数以盘后榜单校正为准。",
|
||||
"updated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
},
|
||||
"overview": {"limit_up_count": 15},
|
||||
"limits": [{"code": "000001"}],
|
||||
"broken": [],
|
||||
"down_limits": [],
|
||||
"yesterday_limits": [],
|
||||
}
|
||||
|
||||
def resolve_trade_context(self, requested: str):
|
||||
return requested, "20260907"
|
||||
|
||||
|
||||
class FakeFreeRealtimeTodayClient:
|
||||
def dashboard(self, trade_date: str):
|
||||
return {
|
||||
"meta": {
|
||||
"trade_date": f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:8]}",
|
||||
"requested_date": f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:8]}",
|
||||
"realtime": True,
|
||||
"mode": "realtime",
|
||||
"quote_source": "eastmoney_clist",
|
||||
"source": "eastmoney",
|
||||
"market_status": "trading",
|
||||
"notice": "盘中行情由东财免费实时快照计算;涨停原因、封板时间和开板次数以盘后榜单校正为准。",
|
||||
"updated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
"indices": [{"code": "000001", "price": 3800.1, "change": 0.5}],
|
||||
},
|
||||
"overview": {"limit_up_count": 18, "up_count": 2100, "amount_billion": 12345.6},
|
||||
"limits": [{"code": "000001"}],
|
||||
"broken": [],
|
||||
"down_limits": [],
|
||||
"yesterday_limits": [],
|
||||
}
|
||||
|
||||
def resolve_trade_context(self, requested: str):
|
||||
return requested, "20260907"
|
||||
|
||||
|
||||
class SyncHarness(MarketServiceMixin):
|
||||
def __init__(self, client, latest=None):
|
||||
def __init__(self, client, latest=None, clock=None):
|
||||
self.configured = True
|
||||
self.sync_lock = threading.Lock()
|
||||
self.database = FakeSyncDatabase(latest)
|
||||
self._client = client
|
||||
self.current_user_id = 1
|
||||
self.clock = clock
|
||||
|
||||
def _tushare_client(self):
|
||||
return self._client
|
||||
@@ -142,23 +209,161 @@ class DashboardFreshnessTests(unittest.TestCase):
|
||||
self.assertEqual(harness.database.finished[0][0][1], "success")
|
||||
self.assertEqual(verified_dashboard_result(payload), payload)
|
||||
|
||||
def test_missing_official_data_keeps_previous_day_with_preparing_notice(self):
|
||||
today = date.today()
|
||||
previous = (today - timedelta(days=1)).strftime("%Y-%m-%d")
|
||||
def test_intraday_refresh_keeps_today_and_does_not_fall_back_to_yesterday(self):
|
||||
today = TRADE_DAY.strftime("%Y%m%d")
|
||||
latest = {
|
||||
"meta": {"trade_date": previous, "source": "tushare"},
|
||||
"meta": {"trade_date": "2026-09-07", "source": "tushare"},
|
||||
"overview": {"limit_up_count": 20},
|
||||
}
|
||||
harness = SyncHarness(FakeMissingDailyClient(), latest)
|
||||
payload = harness.sync_dashboard(today.strftime("%Y%m%d"))
|
||||
harness = SyncHarness(
|
||||
FakeRealtimeTodayClient(),
|
||||
latest,
|
||||
clock=lambda: at_clock(10, 5),
|
||||
)
|
||||
payload = harness.sync_dashboard(today)
|
||||
meta = payload["meta"]
|
||||
|
||||
self.assertTrue(meta["carried_forward"])
|
||||
self.assertEqual(meta["data_status"], "preparing")
|
||||
self.assertIn("今日数据正在准备,当前展示", meta["display_notice"])
|
||||
self.assertIn("月", meta["display_notice"])
|
||||
self.assertNotIn("No daily data", meta["display_notice"])
|
||||
self.assertNotEqual(verified_dashboard_result(payload).get("status"), "failed")
|
||||
self.assertFalse(meta.get("carried_forward"))
|
||||
self.assertTrue(meta["realtime"])
|
||||
self.assertEqual(meta["data_status"], "intraday")
|
||||
self.assertEqual(str(meta["trade_date"]).replace("-", ""), today)
|
||||
self.assertNotIn("今日数据正在准备", meta.get("display_notice") or "")
|
||||
self.assertEqual(harness.database.saved[0][0], today)
|
||||
|
||||
def test_intraday_free_source_keeps_today_and_indices(self):
|
||||
today = TRADE_DAY.strftime("%Y%m%d")
|
||||
latest = {
|
||||
"meta": {"trade_date": "2026-09-07", "source": "tushare"},
|
||||
"overview": {"limit_up_count": 20},
|
||||
}
|
||||
harness = SyncHarness(
|
||||
FakeFreeRealtimeTodayClient(),
|
||||
latest,
|
||||
clock=lambda: at_clock(10, 5),
|
||||
)
|
||||
payload = harness.sync_dashboard(today)
|
||||
meta = payload["meta"]
|
||||
self.assertFalse(meta.get("carried_forward"))
|
||||
self.assertTrue(meta["realtime"])
|
||||
self.assertEqual(meta["data_status"], "intraday")
|
||||
self.assertEqual(str(meta["trade_date"]).replace("-", ""), today)
|
||||
self.assertEqual(meta["quote_source"], "eastmoney_clist")
|
||||
self.assertEqual(payload["overview"]["amount_billion"], 12345.6)
|
||||
self.assertEqual(meta["indices"][0]["price"], 3800.1)
|
||||
self.assertEqual(harness.database.saved[0][0], today)
|
||||
|
||||
def test_intraday_missing_quotes_do_not_carry_yesterday(self):
|
||||
today = TRADE_DAY.strftime("%Y%m%d")
|
||||
latest = {
|
||||
"meta": {"trade_date": "2026-09-07", "source": "tushare"},
|
||||
"overview": {"limit_up_count": 20},
|
||||
}
|
||||
harness = SyncHarness(
|
||||
FakeMissingDailyClient(),
|
||||
latest,
|
||||
clock=lambda: at_clock(10, 5),
|
||||
)
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
harness.sync_dashboard(today)
|
||||
self.assertIn("当天盘中行情", str(ctx.exception))
|
||||
self.assertFalse(harness.database.saved)
|
||||
|
||||
def test_intraday_keeps_existing_today_snapshot_when_refresh_fails(self):
|
||||
today = TRADE_DAY.strftime("%Y%m%d")
|
||||
existing = {
|
||||
"meta": {
|
||||
"trade_date": "2026-09-08",
|
||||
"realtime": True,
|
||||
"mode": "realtime",
|
||||
"source": "tushare",
|
||||
},
|
||||
"overview": {"limit_up_count": 11},
|
||||
"limits": [{"code": "600000"}],
|
||||
"broken": [],
|
||||
"down_limits": [],
|
||||
"yesterday_limits": [],
|
||||
}
|
||||
harness = SyncHarness(
|
||||
FakeMissingDailyClient(),
|
||||
clock=lambda: at_clock(10, 5),
|
||||
)
|
||||
harness.database.get_snapshot = lambda *_args, **_kwargs: copy.deepcopy(existing)
|
||||
payload = harness.sync_dashboard(today)
|
||||
meta = payload["meta"]
|
||||
self.assertEqual(str(meta["trade_date"]).replace("-", ""), today)
|
||||
self.assertTrue(meta["realtime"])
|
||||
self.assertEqual(meta["data_status"], "intraday")
|
||||
self.assertFalse(meta.get("carried_forward"))
|
||||
|
||||
def test_lunch_and_after_hours_keep_today_until_official_arrives(self):
|
||||
today = TRADE_DAY.strftime("%Y%m%d")
|
||||
for clock in (lambda: at_clock(12, 0), lambda: at_clock(16, 10)):
|
||||
harness = SyncHarness(
|
||||
FakeRealtimeTodayClient(),
|
||||
clock=clock,
|
||||
)
|
||||
payload = harness.sync_dashboard(today)
|
||||
self.assertEqual(str(payload["meta"]["trade_date"]).replace("-", ""), today)
|
||||
self.assertFalse(payload["meta"].get("carried_forward"))
|
||||
|
||||
def test_preopen_and_weekend_still_carry_last_session(self):
|
||||
latest = {
|
||||
"meta": {"trade_date": "2026-09-07", "source": "tushare"},
|
||||
"overview": {"limit_up_count": 20},
|
||||
}
|
||||
preopen = SyncHarness(
|
||||
FakeMissingDailyClient(),
|
||||
latest,
|
||||
clock=lambda: at_clock(8, 30),
|
||||
)
|
||||
preopen_payload = preopen.sync_dashboard(TRADE_DAY.strftime("%Y%m%d"))
|
||||
self.assertTrue(preopen_payload["meta"]["carried_forward"])
|
||||
self.assertEqual(preopen_payload["meta"]["data_status"], "preparing")
|
||||
self.assertIn("今日数据正在准备,当前展示", preopen_payload["meta"]["display_notice"])
|
||||
|
||||
weekend = SyncHarness(
|
||||
FakeMissingDailyClient(open_today=False),
|
||||
latest,
|
||||
clock=lambda: at_clock(10, 5, date(2026, 9, 5)),
|
||||
)
|
||||
weekend_payload = weekend.sync_dashboard("20260905")
|
||||
self.assertTrue(weekend_payload["meta"]["carried_forward"])
|
||||
|
||||
def test_history_date_still_uses_official_or_preparing_notice(self):
|
||||
latest = {
|
||||
"meta": {"trade_date": "2026-09-01", "source": "tushare"},
|
||||
"overview": {"limit_up_count": 8},
|
||||
}
|
||||
harness = SyncHarness(
|
||||
FakeMissingDailyClient(),
|
||||
latest,
|
||||
clock=lambda: at_clock(10, 5),
|
||||
)
|
||||
payload = harness.sync_dashboard("20260902")
|
||||
self.assertTrue(payload["meta"]["carried_forward"])
|
||||
self.assertIn("所选日期数据尚未到齐", payload["meta"]["display_notice"])
|
||||
|
||||
def test_carried_today_snapshot_is_retried_immediately_in_session(self):
|
||||
today = TRADE_DAY.strftime("%Y%m%d")
|
||||
snapshot = {
|
||||
"meta": {
|
||||
"source": "tushare",
|
||||
"trade_date": "2026-09-07",
|
||||
"carried_forward": True,
|
||||
"requested_date": "2026-09-08",
|
||||
"updated_at": at_clock(10, 0).isoformat(),
|
||||
},
|
||||
"overview": {"limit_up_count": 1},
|
||||
}
|
||||
harness = SyncHarness(
|
||||
FakeRealtimeTodayClient(),
|
||||
clock=lambda: at_clock(10, 5),
|
||||
)
|
||||
harness.database.get_snapshot = lambda *_args, **_kwargs: copy.deepcopy(snapshot)
|
||||
payload = harness.get_dashboard(today)
|
||||
self.assertEqual(str(payload["meta"]["trade_date"]).replace("-", ""), today)
|
||||
self.assertEqual(payload["meta"]["data_status"], "intraday")
|
||||
self.assertTrue(harness.database.saved)
|
||||
|
||||
def test_weekend_carry_is_not_labeled_as_preparing(self):
|
||||
snapshot = {
|
||||
@@ -200,19 +405,43 @@ class DashboardFreshnessTests(unittest.TestCase):
|
||||
{"meta": {"trade_date": iso, "limit_data_source": "derived"}},
|
||||
)
|
||||
now = datetime.now().astimezone().time().replace(tzinfo=None)
|
||||
if datetime.strptime("15:05", "%H:%M").time() <= now < datetime.strptime("22:00", "%H:%M").time():
|
||||
if dt_time(15, 5) <= now < dt_time(22, 0):
|
||||
self.assertFalse(due)
|
||||
self.assertTrue(derived_due)
|
||||
else:
|
||||
self.assertFalse(due)
|
||||
self.assertFalse(derived_due)
|
||||
|
||||
def test_official_catchup_is_due_for_intraday_snapshot_after_close(self):
|
||||
today = TRADE_DAY.strftime("%Y%m%d")
|
||||
snapshot = {
|
||||
"meta": {
|
||||
"trade_date": "2026-09-08",
|
||||
"realtime": True,
|
||||
"mode": "realtime",
|
||||
}
|
||||
}
|
||||
with patch("backend.jobs.refresh.datetime") as mocked:
|
||||
mocked.now.return_value = at_clock(16, 10)
|
||||
mocked.strptime = datetime.strptime
|
||||
self.assertTrue(official_catchup_due(today, snapshot))
|
||||
official = {
|
||||
"meta": {
|
||||
"trade_date": "2026-09-08",
|
||||
"limit_data_source": "official",
|
||||
"realtime": False,
|
||||
}
|
||||
}
|
||||
self.assertFalse(official_catchup_due(today, official))
|
||||
|
||||
|
||||
class FrontendRefreshCopyTests(unittest.TestCase):
|
||||
def test_dashboard_script_distinguishes_partial_from_failure(self):
|
||||
script = (Path(__file__).resolve().parents[1] / "frontend" / "shared" / "dashboard.js").read_text(encoding="utf-8")
|
||||
self.assertIn("今日数据正在准备,当前展示", script)
|
||||
self.assertIn("部分正式数据尚未到齐", script)
|
||||
self.assertIn("盘中行情", script)
|
||||
self.assertIn("meta.realtime && actualCompact === requestedCompact", script)
|
||||
self.assertIn('job.status === "failed"', script)
|
||||
failed_block = script.split("if (job.status === \"failed\")", 1)[1].split("const query", 1)[0]
|
||||
self.assertIn("后台刷新失败", failed_block)
|
||||
|
||||
@@ -10,7 +10,7 @@ from database import ReviewDatabase
|
||||
|
||||
|
||||
class BootstrapContainerTests(unittest.TestCase):
|
||||
def test_environment_credentials_preserve_legacy_model_fallbacks(self) -> None:
|
||||
def test_environment_credentials_exclude_provider_secrets_and_preserve_llm_fallbacks(self) -> None:
|
||||
result = environment_credentials(
|
||||
{
|
||||
"TUSHARE_TOKEN": " tushare ",
|
||||
@@ -20,8 +20,8 @@ class BootstrapContainerTests(unittest.TestCase):
|
||||
"LLM_MODEL": "legacy-model",
|
||||
}
|
||||
)
|
||||
self.assertEqual(result["tushare_token"], "tushare")
|
||||
self.assertEqual(result["ifind_refresh_token"], "refresh")
|
||||
self.assertNotIn("tushare_token", result)
|
||||
self.assertNotIn("ifind_refresh_token", result)
|
||||
self.assertEqual(result["platform_llm_primary_api_key"], "legacy-key")
|
||||
self.assertEqual(result["platform_llm_primary_base_url"], "https://legacy.example/v1")
|
||||
self.assertEqual(result["platform_llm_primary_model"], "legacy-model")
|
||||
@@ -45,8 +45,9 @@ class BootstrapContainerTests(unittest.TestCase):
|
||||
self.assertIs(container.strategy_tracking.repository.database, database)
|
||||
self.assertIs(container.alert_service.repository.database, database)
|
||||
self.assertIs(container.trade_journal.repository.database, database)
|
||||
self.assertIs(container.chart_data.ifind, container.ifind)
|
||||
self.assertTrue(container.ifind.configured)
|
||||
self.assertIs(container.ifind, container.data_gateway.ifind)
|
||||
self.assertIs(container.chart_data.datahub, container.data_gateway.datahub)
|
||||
self.assertIsNone(container.chart_data.ifind)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import logging
|
||||
import unittest
|
||||
|
||||
from backend.bootstrap.runtime import configure_logging
|
||||
|
||||
|
||||
class ConfigureLoggingTest(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._saved_handlers = logging.getLogger().handlers[:]
|
||||
self._saved_level = logging.getLogger().level
|
||||
logging.getLogger().handlers.clear()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
logging.getLogger().handlers[:] = self._saved_handlers
|
||||
logging.getLogger().setLevel(self._saved_level)
|
||||
|
||||
def test_configures_root_logger_at_info(self) -> None:
|
||||
configure_logging()
|
||||
root = logging.getLogger()
|
||||
self.assertTrue(root.handlers)
|
||||
self.assertEqual(root.level, logging.INFO)
|
||||
with self.assertLogs("xiaobai.datahub", level="INFO") as captured:
|
||||
logging.getLogger("xiaobai.datahub").info("datahub shadow %s", {"dataset": "daily"})
|
||||
self.assertIn("datahub shadow", captured.output[0])
|
||||
|
||||
def test_keeps_existing_configuration(self) -> None:
|
||||
handler = logging.NullHandler()
|
||||
logging.getLogger().addHandler(handler)
|
||||
configure_logging()
|
||||
self.assertEqual(logging.getLogger().handlers, [handler])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from backend.features.market.charts import ChartDataError, EastmoneyChartClient
|
||||
from backend.features.market.charts import ChartDataError, EastmoneyChartClient, HIS_TRENDS_URL, MarketChartClient, TRENDS_URL
|
||||
from server import DashboardService
|
||||
|
||||
|
||||
@@ -72,6 +72,170 @@ class ChartDataProviderTests(unittest.TestCase):
|
||||
self.client.stock_intraday("abc")
|
||||
|
||||
|
||||
class LookbackChartClient(EastmoneyChartClient):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(cache_ttl_seconds=20)
|
||||
self.requests: list[tuple[str, dict[str, str]]] = []
|
||||
|
||||
def _request_json(self, url, params, referer):
|
||||
self.requests.append((url, params))
|
||||
if url == TRENDS_URL and params.get("ndays") == "1":
|
||||
return {"data": {"code": "601318", "name": "中国平安", "preClose": 56.0, "trends": []}}
|
||||
if url == TRENDS_URL and params.get("ndays") == "5":
|
||||
return {"data": {"code": "601318", "name": "中国平安", "preClose": 56.0, "trends": []}}
|
||||
if url == HIS_TRENDS_URL:
|
||||
return {
|
||||
"data": {
|
||||
"code": "601318",
|
||||
"name": "中国平安",
|
||||
"preClose": 55.8,
|
||||
"trends": [
|
||||
"2026-09-07 09:30,55.80,55.90,56.00,55.70,100,5580.00,55.900",
|
||||
"2026-09-07 15:00,56.10,56.20,56.30,56.00,200,11240.00,56.150",
|
||||
"2026-09-08 09:30,0,0,0,0,0,0.00,0",
|
||||
],
|
||||
}
|
||||
}
|
||||
raise ChartDataError("unexpected url")
|
||||
|
||||
|
||||
class ChartLookbackTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
EastmoneyChartClient._cache.clear()
|
||||
self.client = LookbackChartClient()
|
||||
|
||||
def test_empty_today_falls_back_to_latest_available_session(self):
|
||||
payload = self.client.stock_intraday("601318")
|
||||
urls = [url for url, _ in self.client.requests]
|
||||
self.assertEqual(urls[0], TRENDS_URL)
|
||||
self.assertEqual(self.client.requests[0][1]["ndays"], "1")
|
||||
self.assertEqual(urls[1], TRENDS_URL)
|
||||
self.assertEqual(self.client.requests[1][1]["ndays"], "5")
|
||||
self.assertEqual(urls[2], HIS_TRENDS_URL)
|
||||
self.assertEqual(payload["trade_date"], "2026-09-07")
|
||||
self.assertEqual([point["time"] for point in payload["points"]], ["09:30", "15:00"])
|
||||
self.assertEqual(payload["points"][0]["close"], 55.9)
|
||||
|
||||
def test_delay_multiday_can_recover_without_his(self):
|
||||
class DelayFive(EastmoneyChartClient):
|
||||
def __init__(self):
|
||||
super().__init__(cache_ttl_seconds=20)
|
||||
self.requests = []
|
||||
|
||||
def _request_json(self, url, params, referer):
|
||||
self.requests.append((url, params))
|
||||
if params.get("ndays") == "1":
|
||||
return {"data": {"code": "000001", "name": "平安银行", "preClose": 11.7, "trends": []}}
|
||||
return {
|
||||
"data": {
|
||||
"code": "000001",
|
||||
"name": "平安银行",
|
||||
"preClose": 11.5,
|
||||
"trends": [
|
||||
"2026-09-07 09:30,11.50,11.60,11.70,11.40,100,1160.00,11.600",
|
||||
"2026-09-07 15:00,11.70,11.80,11.90,11.60,200,2360.00,11.750",
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
EastmoneyChartClient._cache.clear()
|
||||
client = DelayFive()
|
||||
payload = client.stock_intraday("000001")
|
||||
self.assertEqual(payload["trade_date"], "2026-09-07")
|
||||
self.assertEqual(len(payload["points"]), 2)
|
||||
self.assertEqual([url for url, _ in client.requests], [TRENDS_URL, TRENDS_URL])
|
||||
|
||||
def test_sh_sz_cyb_codes_use_correct_secid(self):
|
||||
for code, secid in (("601318", "1.601318"), ("000001", "0.000001"), ("300750", "0.300750")):
|
||||
EastmoneyChartClient._cache.clear()
|
||||
client = LookbackChartClient()
|
||||
client.stock_intraday(code)
|
||||
self.assertEqual(client.requests[0][1]["secid"], secid)
|
||||
|
||||
|
||||
class FakeHub:
|
||||
def __init__(self, chart=None, error=None, daily=None):
|
||||
self.chart = chart
|
||||
self.error = error
|
||||
self.daily = daily
|
||||
self.calls: list[str] = []
|
||||
self.legacy: list[str] = []
|
||||
|
||||
def try_intraday(self, code):
|
||||
self.calls.append(code)
|
||||
if self.error:
|
||||
raise self.error
|
||||
return self.chart
|
||||
|
||||
def try_daily_chart(self, code, end_date, limit=90, dataset="daily"):
|
||||
self.calls.append(f"{dataset}:{code}")
|
||||
if self.error:
|
||||
raise self.error
|
||||
return self.daily
|
||||
|
||||
def record_legacy(self, dataset, source="", error=""):
|
||||
self.legacy.append(dataset)
|
||||
|
||||
|
||||
class DatahubChartFallbackTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
EastmoneyChartClient._cache.clear()
|
||||
|
||||
def test_datahub_success_skips_old_channel(self):
|
||||
hub = FakeHub(
|
||||
{
|
||||
"entity_type": "stock",
|
||||
"identifier": "601318",
|
||||
"name": "中国平安",
|
||||
"code": "601318",
|
||||
"trade_date": "2026-09-08",
|
||||
"previous_close": 56.36,
|
||||
"points": [{"date": "2026-09-08", "time": "09:30", "close": 56.5, "average": 56.4}],
|
||||
"source": "datahub",
|
||||
}
|
||||
)
|
||||
fallback = LookbackChartClient()
|
||||
client = MarketChartClient(hub)
|
||||
payload = client.stock_intraday("601318")
|
||||
self.assertEqual(payload["source"], "datahub")
|
||||
self.assertEqual(hub.calls, ["601318"])
|
||||
self.assertEqual(fallback.requests, [])
|
||||
|
||||
def test_datahub_timeout_or_empty_does_not_use_old_channel(self):
|
||||
fallback = LookbackChartClient()
|
||||
for hub in (
|
||||
FakeHub(chart=None),
|
||||
FakeHub(error=RuntimeError("timeout")),
|
||||
FakeHub(error=RuntimeError("datahub exploded")),
|
||||
FakeHub(chart={"points": []}),
|
||||
):
|
||||
EastmoneyChartClient._cache.clear()
|
||||
fallback.requests.clear()
|
||||
client = MarketChartClient(hub)
|
||||
with self.assertRaises(ChartDataError):
|
||||
client.stock_intraday("000001")
|
||||
self.assertEqual(fallback.requests, [])
|
||||
|
||||
def test_datahub_daily_skips_ifind(self):
|
||||
hub = FakeHub(
|
||||
daily=[
|
||||
{
|
||||
"trade_date": "2026-09-07",
|
||||
"open": 10.0,
|
||||
"high": 10.4,
|
||||
"low": 9.9,
|
||||
"close": 10.2,
|
||||
"volume": 1000,
|
||||
"amount_billion": 0.02,
|
||||
}
|
||||
]
|
||||
)
|
||||
client = MarketChartClient(hub)
|
||||
rows = client.stock_daily("600000", "20260907")
|
||||
self.assertEqual(rows[-1]["trade_date"], "2026-09-07")
|
||||
self.assertIn("daily:600000", hub.calls)
|
||||
|
||||
|
||||
class ChartServiceStub:
|
||||
@staticmethod
|
||||
def _payload(code: str, name: str):
|
||||
|
||||
+38
-15
@@ -12,6 +12,7 @@ from backend.data import (
|
||||
QualityEvidence,
|
||||
build_data_gateway,
|
||||
)
|
||||
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
||||
from backend.data.quality import market_timezone
|
||||
|
||||
|
||||
@@ -35,16 +36,30 @@ class DataGatewayTests(unittest.TestCase):
|
||||
with self.assertRaises(DataPolicyError):
|
||||
policy.assert_allowed("market.level2", "unresolved", "display")
|
||||
|
||||
def test_gateway_uses_live_token_supplier_and_shared_ifind(self) -> None:
|
||||
token = {"value": "first"}
|
||||
gateway = build_data_gateway(
|
||||
{"ifind_refresh_token": "refresh", "ifind_access_token": "access"},
|
||||
lambda: token["value"],
|
||||
def test_gateway_uses_hub_facade_and_proxies(self) -> None:
|
||||
settings = DatahubSettings(
|
||||
base_url="http://127.0.0.1:8766",
|
||||
token="hub-token",
|
||||
datasets={name: DatasetFlags(name, read=True) for name in DATASETS},
|
||||
)
|
||||
self.assertEqual(gateway.tushare().token, "first")
|
||||
token["value"] = "second"
|
||||
self.assertEqual(gateway.tushare().token, "second")
|
||||
self.assertIs(gateway.chart_data.ifind, gateway.ifind)
|
||||
gateway = build_data_gateway(
|
||||
{},
|
||||
datahub_settings=settings,
|
||||
)
|
||||
client = gateway.tushare()
|
||||
self.assertEqual(client.token, "datahub")
|
||||
self.assertIsNone(client.realtime_aggregator)
|
||||
self.assertFalse(hasattr(client, "_legacy"))
|
||||
self.assertIs(gateway.ifind, gateway.ifind_provider.client)
|
||||
self.assertIs(gateway.chart_data.datahub, gateway.datahub)
|
||||
self.assertIsNone(gateway.chart_data.ifind)
|
||||
from backend.data.datahub.bridge import DatahubAwareTushareClient
|
||||
from backend.data.datahub.ifind_proxy import HubIfindProxy
|
||||
from backend.data.datahub.realtime_proxy import HubRealtimeProxy
|
||||
|
||||
self.assertIsInstance(client, DatahubAwareTushareClient)
|
||||
self.assertIsInstance(gateway.ifind, HubIfindProxy)
|
||||
self.assertIsInstance(gateway.realtime_observer, HubRealtimeProxy)
|
||||
|
||||
def test_server_has_no_direct_runtime_tushare_construction(self) -> None:
|
||||
source = (
|
||||
@@ -54,24 +69,29 @@ class DataGatewayTests(unittest.TestCase):
|
||||
/ "market"
|
||||
/ "service.py"
|
||||
).read_text(encoding="utf-8")
|
||||
self.assertEqual(source.count("TushareClient(self.token)"), 1)
|
||||
self.assertNotIn("TushareClient(self.token)", source)
|
||||
self.assertIn("return gateway.tushare()", source)
|
||||
|
||||
def test_provider_construction_has_unique_declared_owners(self) -> None:
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
owners = {
|
||||
"EastmoneyChartClient": {"backend/data/gateway.py"},
|
||||
"IfindHttpClient": {"backend/data/gateway.py"},
|
||||
"IfindProvider": {"backend/data/gateway.py"},
|
||||
"MarketChartClient": {"backend/data/gateway.py"},
|
||||
"TushareClient": {"backend/features/market/service.py"},
|
||||
"TushareProvider": {"backend/data/gateway.py"},
|
||||
"WebRealtimeAggregator": {"backend/data/gateway.py"},
|
||||
"TushareClient": set(),
|
||||
"DatahubClient": {"backend/data/gateway.py"},
|
||||
"DatahubAwareTushareClient": {"backend/data/gateway.py"},
|
||||
"DatahubBridge": {"backend/data/gateway.py"},
|
||||
"HubIfindProxy": {"backend/data/gateway.py"},
|
||||
"HubRealtimeProxy": {"backend/data/gateway.py"},
|
||||
}
|
||||
found = {name: set() for name in owners}
|
||||
forbidden = {
|
||||
"IfindHttpClient": set(),
|
||||
"EastmoneyChartClient": set(),
|
||||
"WebRealtimeAggregator": set(),
|
||||
"TushareProvider": set(),
|
||||
}
|
||||
found_forbidden = {name: set() for name in forbidden}
|
||||
for path in (root / "backend").rglob("*.py"):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
@@ -81,7 +101,10 @@ class DataGatewayTests(unittest.TestCase):
|
||||
name = getattr(node.func, "id", None) or getattr(node.func, "attr", None)
|
||||
if name in found:
|
||||
found[name].add(relative)
|
||||
if name in found_forbidden:
|
||||
found_forbidden[name].add(relative)
|
||||
self.assertEqual(found, owners)
|
||||
self.assertEqual(found_forbidden, forbidden)
|
||||
provider_source = (root / "backend/data/providers/tushare.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
+391
-35
@@ -12,6 +12,8 @@ from backend.data.datahub.client import DatahubClient, DatahubResponse
|
||||
from backend.data.datahub.compare import compare_rows
|
||||
from backend.data.datahub.errors import DatahubError
|
||||
from backend.data.datahub.native import to_canonical_row, to_native_row
|
||||
from backend.data.datahub.route_state import LEDGER
|
||||
from backend.data.providers.tushare_transport import TushareError
|
||||
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -64,10 +66,19 @@ class FakeClient(DatahubClient):
|
||||
meta={"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0},
|
||||
)
|
||||
self.paths: list[str] = []
|
||||
self.calls: list[tuple[str, dict[str, Any]]] = []
|
||||
|
||||
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
|
||||
return self._record(path, params)
|
||||
|
||||
def post(self, path: str, body: dict[str, Any] | None = None) -> DatahubResponse:
|
||||
return self._record(path, body)
|
||||
|
||||
def _record(self, path: str, payload: dict[str, Any] | None) -> DatahubResponse:
|
||||
self.paths.append(path)
|
||||
if TOKEN in json.dumps(params or {}) or TOKEN in path:
|
||||
self.calls.append((path, {key: value for key, value in (payload or {}).items()}))
|
||||
packed = json.dumps(payload or {})
|
||||
if TOKEN in packed or TOKEN in path:
|
||||
raise AssertionError("token leaked into url")
|
||||
if self.error:
|
||||
raise self.error
|
||||
@@ -82,17 +93,21 @@ def flags(**enabled: tuple[bool, bool]) -> DatahubSettings:
|
||||
|
||||
|
||||
class DatahubBridgeTests(unittest.TestCase):
|
||||
def test_default_config_keeps_legacy_and_does_not_call_datahub(self) -> None:
|
||||
def setUp(self) -> None:
|
||||
LEDGER.clear()
|
||||
|
||||
def test_default_config_enables_official_reads(self) -> None:
|
||||
settings = DatahubSettings.load(environ={}, credentials={})
|
||||
self.assertFalse(settings.any_enabled())
|
||||
self.assertTrue(all(not settings.flags(name).read and not settings.flags(name).shadow for name in DATASETS))
|
||||
client = FakeClient(error=DatahubError("INTERNAL", "should not be called"))
|
||||
self.assertTrue(settings.any_enabled())
|
||||
self.assertTrue(all(settings.flags(name).read and not settings.flags(name).shadow for name in DATASETS))
|
||||
client = FakeClient()
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(settings, client))
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,close,vol,amount")
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(client.paths, [])
|
||||
self.assertEqual(len(legacy.calls), 1)
|
||||
self.assertEqual(client.paths, ["/v1/bars/daily"])
|
||||
self.assertEqual(legacy.calls, [])
|
||||
self.assertEqual(LEDGER.snapshot()[0]["route"], "datahub")
|
||||
|
||||
def test_each_dataset_has_independent_read_flag(self) -> None:
|
||||
settings = flags(daily=(True, False), auction=(False, False))
|
||||
@@ -102,6 +117,13 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
source = (ROOT / "config" / "datahub.config.json").read_text(encoding="utf-8")
|
||||
self.assertNotIn("master", source)
|
||||
self.assertNotIn("DATAHUB_READ_ALL", source)
|
||||
compose = (ROOT / "compose.yaml").read_text(encoding="utf-8")
|
||||
for env_key in (
|
||||
"CALENDAR", "STOCKS", "DAILY", "INDEX_DAILY", "VALUATION", "MONEYFLOW",
|
||||
"AUCTION", "LIMIT_EVENTS", "POPULARITY", "DRAGON_TIGER", "SECTOR_DAILY",
|
||||
"QUOTES", "INDEX_QUOTES", "INTRADAY", "STATUS",
|
||||
):
|
||||
self.assertIn(f'DATAHUB_READ_{env_key}: "1"', compose)
|
||||
|
||||
def test_read_flag_replaces_only_that_dataset_and_converts_units(self) -> None:
|
||||
shadows: list[dict[str, Any]] = []
|
||||
@@ -117,16 +139,22 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
self.assertEqual(legacy.calls, [])
|
||||
self.assertEqual(client.paths, ["/v1/bars/daily"])
|
||||
calendar_legacy = FakeLegacy([{"cal_date": "20240902", "is_open": 1}])
|
||||
calendar_client = FakeClient(error=DatahubError("INTERNAL", "nope"))
|
||||
calendar_client = FakeClient(
|
||||
response=DatahubResponse(
|
||||
data=[{"cal_date": "20240902", "is_open": 1, "pretrade_date": "20240830"}],
|
||||
meta={"source": "datahub", "stale": False, "staleness_seconds": 0},
|
||||
)
|
||||
)
|
||||
calendar_wrapped = DatahubAwareTushareClient(
|
||||
calendar_legacy,
|
||||
DatahubBridge(flags(daily=(True, False)), calendar_client),
|
||||
)
|
||||
calendar = calendar_wrapped.query("trade_cal", {"start_date": "20240902", "end_date": "20240902"}, "")
|
||||
self.assertEqual(calendar[0]["is_open"], 1)
|
||||
self.assertEqual(calendar_client.paths, [])
|
||||
self.assertEqual(calendar_legacy.calls, [])
|
||||
self.assertEqual(calendar_client.paths, ["/v1/query"])
|
||||
|
||||
def test_fallback_on_down_401_timeout_empty_unpublished_and_stale(self) -> None:
|
||||
def test_hub_failure_does_not_call_website_legacy(self) -> None:
|
||||
cases = [
|
||||
DatahubError("UNAVAILABLE", "down"),
|
||||
DatahubError("UNAUTHORIZED", "401"),
|
||||
@@ -134,27 +162,25 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
DatahubError("EMPTY", "no rows"),
|
||||
DatahubError("DATASET_NOT_PUBLISHED", "not ready"),
|
||||
DatahubError("STALE", "old"),
|
||||
DatahubError("INCOMPLETE", "truncated"),
|
||||
]
|
||||
for error in cases:
|
||||
with self.subTest(error=error.code):
|
||||
if error.code == "EMPTY":
|
||||
client = FakeClient(response=DatahubResponse(data=[], meta={"stale": False, "staleness_seconds": 0}))
|
||||
elif error.code == "STALE":
|
||||
client = FakeClient(response=DatahubResponse(
|
||||
data=[dict(HUB_DAILY)],
|
||||
meta={"stale": True, "staleness_seconds": 999999},
|
||||
))
|
||||
else:
|
||||
client = FakeClient(error=error)
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(flags(daily=(True, False)), client))
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(len(legacy.calls), 1)
|
||||
with self.assertRaises(TushareError):
|
||||
wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
|
||||
self.assertEqual(legacy.calls, [])
|
||||
|
||||
def test_shadow_compares_without_replacing_and_survives_hub_failure(self) -> None:
|
||||
def test_shadow_mode_no_longer_calls_website_tushare(self) -> None:
|
||||
reports: list[dict[str, Any]] = []
|
||||
client = FakeClient()
|
||||
client = FakeClient(
|
||||
response=DatahubResponse(
|
||||
data=[dict(LEGACY_DAILY)],
|
||||
meta={"source": "tushare", "stale": False, "staleness_seconds": 0, "row_shape": "tushare"},
|
||||
)
|
||||
)
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(
|
||||
legacy,
|
||||
@@ -162,21 +188,19 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
)
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount,vol")
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(len(legacy.calls), 1)
|
||||
self.assertEqual(reports[0]["equal"], True)
|
||||
self.assertEqual(reports[0]["matched"], 1)
|
||||
self.assertEqual(legacy.calls, [])
|
||||
self.assertEqual(client.paths, ["/v1/query"])
|
||||
|
||||
failed = FakeClient(error=DatahubError("UNAVAILABLE", TOKEN))
|
||||
fail_reports: list[dict[str, Any]] = []
|
||||
fail_legacy = FakeLegacy([LEGACY_DAILY])
|
||||
fail_wrapped = DatahubAwareTushareClient(
|
||||
fail_legacy,
|
||||
DatahubBridge(flags(daily=(False, True)), failed, shadow_sink=fail_reports.append),
|
||||
DatahubBridge(flags(daily=(False, True)), failed, shadow_sink=reports.append),
|
||||
)
|
||||
again = fail_wrapped.query("daily", {"trade_date": "20240902"}, "amount")
|
||||
self.assertEqual(again[0]["amount"], 2000.0)
|
||||
self.assertTrue(fail_reports[0]["hub_error"])
|
||||
self.assertNotIn(TOKEN, json.dumps(fail_reports[0]))
|
||||
with self.assertRaises(TushareError):
|
||||
fail_wrapped.query("daily", {"trade_date": "20240902"}, "amount")
|
||||
self.assertEqual(fail_legacy.calls, [])
|
||||
self.assertNotIn(TOKEN, str(failed.calls))
|
||||
|
||||
def test_compare_classifies_unit_conversion_missing_row_and_value_diff(self) -> None:
|
||||
equal = compare_rows("daily", [LEGACY_DAILY], [HUB_DAILY], {"stale": False, "staleness_seconds": 0})
|
||||
@@ -190,6 +214,86 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
skew = compare_rows("daily", [LEGACY_DAILY], [HUB_DAILY], {"stale": False, "staleness_seconds": 12})
|
||||
self.assertTrue(skew["time_skew"])
|
||||
|
||||
def test_shadow_extra_hub_columns_are_not_false_diffs_when_projected(self) -> None:
|
||||
hub_full = {**HUB_DAILY, "adj_factor": 1.1}
|
||||
legacy_close_only = {k: LEGACY_DAILY[k] for k in ("ts_code", "trade_date", "close")}
|
||||
report = compare_rows(
|
||||
"daily", [legacy_close_only], [hub_full],
|
||||
{"stale": False, "staleness_seconds": 0},
|
||||
fields="ts_code,trade_date,close",
|
||||
)
|
||||
self.assertTrue(report["equal"])
|
||||
self.assertEqual(report["value_diff_count"], 0)
|
||||
self.assertEqual(report["fields_compared"], ["close", "trade_date", "ts_code"])
|
||||
# without projection the same pair shows the historic false diff
|
||||
unprojected = compare_rows("daily", [legacy_close_only], [hub_full])
|
||||
self.assertFalse(unprojected["equal"])
|
||||
|
||||
legacy_stocks = {"ts_code": "600000.SH", "name": "浦发银行"}
|
||||
hub_stocks = {
|
||||
"ts_code": "600000.SH", "symbol": "600000", "name": "浦发银行", "area": "上海",
|
||||
"industry": "银行", "market": "主板", "list_status": "L", "list_date": "19991110",
|
||||
}
|
||||
stocks = compare_rows("stocks", [legacy_stocks], [hub_stocks], {}, fields="ts_code,name")
|
||||
self.assertTrue(stocks["equal"])
|
||||
|
||||
legacy_cal = {"cal_date": "20240902", "is_open": 1}
|
||||
hub_cal = {
|
||||
"cal_date": "20240902", "is_open": True,
|
||||
"pretrade_date": "20240830", "prev_open": "20240830",
|
||||
}
|
||||
calendar = compare_rows(
|
||||
"calendar", [legacy_cal], [hub_cal], {}, fields="cal_date,is_open"
|
||||
)
|
||||
self.assertTrue(calendar["equal"])
|
||||
|
||||
def test_shadow_projection_still_alarms_on_requested_field_problems(self) -> None:
|
||||
hub_missing_field = {k: v for k, v in HUB_DAILY.items() if k != "close"}
|
||||
legacy_close_only = {k: LEGACY_DAILY[k] for k in ("ts_code", "trade_date", "close")}
|
||||
lost = compare_rows(
|
||||
"daily", [legacy_close_only], [hub_missing_field], fields="ts_code,trade_date,close"
|
||||
)
|
||||
self.assertFalse(lost["equal"])
|
||||
self.assertEqual(lost["value_diff_count"], 1)
|
||||
|
||||
changed = compare_rows(
|
||||
"daily", [legacy_close_only], [{**HUB_DAILY, "close": 99.0}],
|
||||
fields="ts_code,trade_date,close",
|
||||
)
|
||||
self.assertFalse(changed["equal"])
|
||||
self.assertEqual(changed["value_diff_count"], 1)
|
||||
self.assertEqual(changed["value_diffs"][0]["fields"][0]["field"], "close")
|
||||
|
||||
gone = compare_rows("daily", [LEGACY_DAILY], [], fields="ts_code,trade_date,close")
|
||||
self.assertEqual(gone["missing_hub_count"], 1)
|
||||
self.assertFalse(gone["equal"])
|
||||
|
||||
unit = compare_rows(
|
||||
"daily", [LEGACY_DAILY], [{**HUB_DAILY, "amount": 2000.0, "volume": 1000.0}],
|
||||
fields="ts_code,trade_date,vol,amount",
|
||||
)
|
||||
self.assertGreater(unit["unit_conversion_count"], 0)
|
||||
self.assertFalse(unit["equal"])
|
||||
|
||||
def test_bridge_shadow_report_uses_website_request_fields(self) -> None:
|
||||
hub_full = {**HUB_DAILY, "adj_factor": 1.1}
|
||||
legacy_close_only = {k: LEGACY_DAILY[k] for k in ("ts_code", "trade_date", "close", "vol", "amount")}
|
||||
reports: list[dict[str, Any]] = []
|
||||
client = FakeClient(
|
||||
response=DatahubResponse(
|
||||
data=[hub_full],
|
||||
meta={"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0},
|
||||
)
|
||||
)
|
||||
wrapped = DatahubAwareTushareClient(
|
||||
FakeLegacy([legacy_close_only]),
|
||||
DatahubBridge(flags(daily=(True, False)), client, shadow_sink=reports.append),
|
||||
)
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,trade_date,close,vol,amount")
|
||||
self.assertEqual(rows[0]["close"], 10.20)
|
||||
self.assertEqual(rows[0]["vol"], 1000.0)
|
||||
self.assertEqual(client.paths, ["/v1/bars/daily"])
|
||||
|
||||
def test_native_roundtrip_matches_known_scales(self) -> None:
|
||||
native = to_native_row("daily", HUB_DAILY)
|
||||
self.assertEqual(native["vol"], 1000.0)
|
||||
@@ -198,8 +302,8 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
self.assertEqual(canonical["vol"], 100000.0)
|
||||
self.assertEqual(canonical["amount"], 2000000.0)
|
||||
|
||||
def test_heaven_keeps_legacy_on_first_batch_even_when_read_flag_is_on(self) -> None:
|
||||
"""问天未永久冻结;首批只读接入仍走旧链路,后续迁移可以纳入。"""
|
||||
def test_heaven_can_use_hub_when_dataset_flag_is_on(self) -> None:
|
||||
"""问天按数据依赖接入:已映射 API 跟随开关,不再整栈强制旧链路。"""
|
||||
self.assertTrue(looks_like_heaven("backend.features.heaven.market_context", "backend/features/heaven/market_context.py"))
|
||||
self.assertFalse(looks_like_heaven("backend.features.market.service", "backend/features/market/service.py"))
|
||||
client = FakeClient()
|
||||
@@ -210,7 +314,8 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
)
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "amount")
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(client.paths, [])
|
||||
self.assertEqual(client.paths, ["/v1/bars/daily"])
|
||||
self.assertEqual(legacy.calls, [])
|
||||
|
||||
def test_status_flag_does_not_run_when_off_and_falls_back_when_on(self) -> None:
|
||||
off = DatahubBridge(flags(), FakeClient(error=DatahubError("UNAVAILABLE", "down")))
|
||||
@@ -235,6 +340,257 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
self.assertIsInstance(client, DatahubAwareTushareClient)
|
||||
self.assertFalse(gateway.datahub.settings.any_enabled())
|
||||
|
||||
def test_stock_detail_range_query_is_not_silently_accepted_when_incomplete(self) -> None:
|
||||
source = (ROOT / "backend" / "data" / "providers" / "tushare_stocks.py").read_text(encoding="utf-8")
|
||||
self.assertIn('"daily"', source)
|
||||
self.assertIn("start_date", source)
|
||||
self.assertIn("end_date", source)
|
||||
client = FakeClient(error=DatahubError("INCOMPLETE", "truncated"))
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(flags(daily=(True, False)), client))
|
||||
with self.assertRaises(TushareError):
|
||||
wrapped.query(
|
||||
"daily",
|
||||
{"ts_code": "600000.SH", "start_date": "20240301", "end_date": "20240902"},
|
||||
"ts_code,amount",
|
||||
)
|
||||
self.assertEqual(legacy.calls, [])
|
||||
self.assertIn("/v1/query", client.paths)
|
||||
|
||||
def test_try_intraday_respects_switch_and_falls_back_on_bad_payload(self) -> None:
|
||||
closed = DatahubBridge(flags(), FakeClient(error=DatahubError("INTERNAL", "should not run")))
|
||||
self.assertIsNone(closed.try_intraday("601318"))
|
||||
|
||||
empty = DatahubBridge(
|
||||
flags(intraday=(True, False)),
|
||||
FakeClient(response=DatahubResponse(data={"points": []}, meta={"stale": False})),
|
||||
)
|
||||
self.assertIsNone(empty.try_intraday("601318"))
|
||||
|
||||
stale = DatahubBridge(
|
||||
flags(intraday=(True, False)),
|
||||
FakeClient(response=DatahubResponse(
|
||||
data={
|
||||
"entity_type": "stock",
|
||||
"code": "601318",
|
||||
"trade_date": "2026-09-07",
|
||||
"previous_close": 55.8,
|
||||
"points": [{"date": "2026-09-07", "time": "09:30", "close": 55.9, "avg_price": 55.85}],
|
||||
},
|
||||
meta={"stale": True},
|
||||
)),
|
||||
)
|
||||
self.assertIsNone(stale.try_intraday("601318"))
|
||||
|
||||
ok = DatahubBridge(
|
||||
flags(intraday=(True, False)),
|
||||
FakeClient(response=DatahubResponse(
|
||||
data={
|
||||
"entity_type": "stock",
|
||||
"identifier": "601318",
|
||||
"name": "中国平安",
|
||||
"code": "601318",
|
||||
"trade_date": "2026-09-08",
|
||||
"previous_close": 56.36,
|
||||
"points": [
|
||||
{"date": "2026-09-08", "time": "09:30", "close": 0},
|
||||
{"date": "2026-09-08", "time": "09:31", "close": 56.5, "avg_price": 56.4},
|
||||
],
|
||||
},
|
||||
meta={"stale": False},
|
||||
)),
|
||||
)
|
||||
chart = ok.try_intraday("601318")
|
||||
self.assertEqual(chart["source"], "datahub")
|
||||
self.assertEqual(len(chart["points"]), 1)
|
||||
self.assertEqual(chart["points"][0]["average"], 56.4)
|
||||
self.assertEqual(ok.client.paths, ["/v1/intraday/points"])
|
||||
self.assertEqual(ok.client.calls, [("/v1/intraday/points", {"code": "601318"})])
|
||||
self.assertNotIn("date", ok.client.calls[0][1])
|
||||
|
||||
timeout = DatahubBridge(
|
||||
flags(intraday=(True, False)),
|
||||
FakeClient(error=DatahubError("TIMEOUT", "datahub request timed out")),
|
||||
)
|
||||
self.assertIsNone(timeout.try_intraday("601318"))
|
||||
broken = DatahubBridge(
|
||||
flags(intraday=(True, False)),
|
||||
FakeClient(error=DatahubError("INTERNAL", "datahub exploded")),
|
||||
)
|
||||
self.assertIsNone(broken.try_intraday("601318"))
|
||||
self.assertTrue(DatahubSettings.load(environ={}, credentials={}).flags("intraday").read)
|
||||
|
||||
def test_try_market_quotes_and_visible_fallback(self) -> None:
|
||||
quotes = [
|
||||
{
|
||||
"ts_code": f"{600000 + index:06d}.SH",
|
||||
"name": f"股票{index}",
|
||||
"close": 10.2,
|
||||
"pre_close": 10.0,
|
||||
"open": 10.1,
|
||||
"high": 10.3,
|
||||
"low": 9.9,
|
||||
"vol": 1000,
|
||||
"amount": 2000000,
|
||||
"quote_date": "20240902",
|
||||
}
|
||||
for index in range(220)
|
||||
]
|
||||
ok = DatahubBridge(
|
||||
flags(quotes=(True, False)),
|
||||
FakeClient(
|
||||
response=DatahubResponse(
|
||||
data=quotes,
|
||||
meta={"stale": False, "staleness_seconds": 0, "source": "eastmoney:clist"},
|
||||
)
|
||||
),
|
||||
)
|
||||
rows = ok.try_market_quotes("20240902")
|
||||
self.assertEqual(len(rows), 220)
|
||||
self.assertEqual(rows[0]["pre_close"], 10.0)
|
||||
self.assertEqual(ok.client.paths, ["/v1/quotes/latest"])
|
||||
self.assertEqual(LEDGER.snapshot()[0]["route"], "datahub")
|
||||
|
||||
failed = DatahubBridge(
|
||||
flags(quotes=(True, False)),
|
||||
FakeClient(error=DatahubError("UNAVAILABLE", "down")),
|
||||
)
|
||||
self.assertIsNone(failed.try_market_quotes("20240902"))
|
||||
snap = next(item for item in LEDGER.snapshot() if item["dataset"] == "quotes")
|
||||
self.assertEqual(snap["route"], "datahub")
|
||||
self.assertEqual(snap["source"], "unavailable")
|
||||
|
||||
gateway = build_data_gateway({}, datahub_settings=flags(quotes=(True, False)))
|
||||
status = gateway.datahub_status()
|
||||
self.assertEqual(status["enabled_reads"], 1)
|
||||
self.assertEqual(status["total_reads"], len(DATASETS))
|
||||
self.assertEqual(status["fallback_count"], 0)
|
||||
|
||||
def test_try_daily_chart_converts_hub_bars(self) -> None:
|
||||
rows = [
|
||||
{
|
||||
"ts_code": "600000.SH",
|
||||
"trade_date": "20240901",
|
||||
"open": 10.0,
|
||||
"high": 10.4,
|
||||
"low": 9.9,
|
||||
"close": 10.2,
|
||||
"volume": 100000,
|
||||
"amount": 2000000,
|
||||
},
|
||||
{
|
||||
"ts_code": "600000.SH",
|
||||
"trade_date": "20240902",
|
||||
"open": 10.2,
|
||||
"high": 10.5,
|
||||
"low": 10.1,
|
||||
"close": 10.4,
|
||||
"volume": 120000,
|
||||
"amount": 2400000,
|
||||
},
|
||||
]
|
||||
hub = DatahubBridge(
|
||||
flags(daily=(True, False)),
|
||||
FakeClient(
|
||||
response=DatahubResponse(
|
||||
data=rows,
|
||||
meta={"stale": False, "staleness_seconds": 0, "source": "tushare:daily"},
|
||||
)
|
||||
),
|
||||
)
|
||||
chart = hub.try_daily_chart("600000.SH", "20240902", 90, "daily")
|
||||
self.assertEqual(chart[-1]["trade_date"], "2024-09-02")
|
||||
self.assertEqual(chart[-1]["close"], 10.4)
|
||||
self.assertAlmostEqual(chart[-1]["amount_billion"], 0.024)
|
||||
|
||||
def test_try_daily_chart_keeps_usable_bars_when_coverage_incomplete(self) -> None:
|
||||
rows = [
|
||||
{
|
||||
"ts_code": "000001.SZ",
|
||||
"trade_date": "20240901",
|
||||
"open": 10.0,
|
||||
"high": 10.4,
|
||||
"low": 9.9,
|
||||
"close": 10.2,
|
||||
"volume": 100000,
|
||||
"amount": 2000000,
|
||||
},
|
||||
{
|
||||
"ts_code": "000001.SZ",
|
||||
"trade_date": "20240902",
|
||||
"open": 10.2,
|
||||
"high": 10.5,
|
||||
"low": 10.1,
|
||||
"close": 10.4,
|
||||
"volume": 120000,
|
||||
"amount": 2400000,
|
||||
},
|
||||
]
|
||||
hub = DatahubBridge(
|
||||
flags(daily=(True, False)),
|
||||
FakeClient(
|
||||
response=DatahubResponse(
|
||||
data=rows,
|
||||
meta={
|
||||
"stale": False,
|
||||
"staleness_seconds": 0,
|
||||
"incomplete": True,
|
||||
"coverage": {"complete": False, "missing_count": 127},
|
||||
"source": "tushare:daily",
|
||||
},
|
||||
)
|
||||
),
|
||||
)
|
||||
chart = hub.try_daily_chart("000001.SZ", "20240902", 90, "daily")
|
||||
self.assertIsNotNone(chart)
|
||||
self.assertEqual(chart[-1]["trade_date"], "2024-09-02")
|
||||
self.assertEqual(chart[-1]["close"], 10.4)
|
||||
|
||||
def test_gateway_tushare_facade_has_no_legacy_client(self) -> None:
|
||||
quotes = [
|
||||
{
|
||||
"ts_code": f"{index:06d}.SZ",
|
||||
"name": f"S{index}",
|
||||
"pre_close": 10.0,
|
||||
"open": 10.0,
|
||||
"high": 10.5,
|
||||
"low": 9.8,
|
||||
"close": 10.2,
|
||||
"vol": 100.0,
|
||||
"amount": 1000.0,
|
||||
"quote_date": "20240902",
|
||||
}
|
||||
for index in range(1, 221)
|
||||
]
|
||||
hub_client = FakeClient(
|
||||
response=DatahubResponse(
|
||||
data=quotes,
|
||||
meta={"stale": False, "staleness_seconds": 0, "source": "eastmoney_clist"},
|
||||
)
|
||||
)
|
||||
gateway = build_data_gateway(
|
||||
{"tushare_token": "tok"},
|
||||
datahub_settings=flags(quotes=(True, False), daily=(True, False)),
|
||||
)
|
||||
gateway.datahub.client = hub_client
|
||||
wrapped = gateway.tushare()
|
||||
self.assertFalse(hasattr(wrapped, "_legacy"))
|
||||
self.assertIsNone(getattr(type(wrapped), "__getattr__", None))
|
||||
self.assertTrue(callable(getattr(type(wrapped), "try_market_quotes", None)))
|
||||
self.assertTrue(callable(getattr(type(wrapped), "try_index_quotes", None)))
|
||||
self.assertTrue(callable(getattr(type(wrapped), "record_datahub_legacy", None)))
|
||||
self.assertTrue(callable(getattr(type(wrapped), "dashboard", None)))
|
||||
rows = wrapped.try_market_quotes("20240902")
|
||||
self.assertGreaterEqual(len(rows or []), 200)
|
||||
self.assertIn("/v1/quotes/latest", hub_client.paths)
|
||||
hub_client.response = DatahubResponse(
|
||||
data=[dict(HUB_DAILY)],
|
||||
meta={"stale": False, "staleness_seconds": 0, "source": "tushare:daily"},
|
||||
)
|
||||
daily = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
|
||||
self.assertEqual(daily[0]["amount"], 2000.0)
|
||||
self.assertIn("/v1/bars/daily", hub_client.paths)
|
||||
|
||||
def test_features_do_not_import_datahub_client(self) -> None:
|
||||
violations = []
|
||||
for path in (ROOT / "backend" / "features").rglob("*.py"):
|
||||
|
||||
@@ -459,6 +459,19 @@ class FrontendContractTests(unittest.TestCase):
|
||||
self.assertIn('payload.question_preset = state.heartQuestionPreset;', self.script)
|
||||
self.assertIn('payload.cast_at = state.heartCastAt;', self.script)
|
||||
|
||||
def test_heaven_loading_timeout_clears_dimmed_state(self):
|
||||
self.assertIn("controller.abort()", self.script)
|
||||
self.assertIn('heavenView?.classList.remove("heaven-data-loading")', self.script)
|
||||
self.assertIn("问天数据仍在准备,页面可继续输入和操作", self.script)
|
||||
self.assertIn("const blocking = !state.heavenSetup;", self.script)
|
||||
self.assertIn("payload?.aborted", self.script)
|
||||
|
||||
def test_stock_detail_does_not_display_missing_metrics_as_zero(self):
|
||||
self.assertIn("function setStockBoardFields(row)", self.script)
|
||||
self.assertIn("function presentMetric(value)", self.script)
|
||||
self.assertIn("payload.available !== false", self.script)
|
||||
self.assertIn('element.textContent = "--"', self.script)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -94,7 +94,7 @@ class GlobalSearchTests(unittest.TestCase):
|
||||
self.assertIn('event.key.toLowerCase() !== "k"', script)
|
||||
self.assertIn('openStock(item.id, { code: item.code', script)
|
||||
self.assertNotIn('include_notes', script)
|
||||
self.assertIn('const candles = (series || [])', script)
|
||||
self.assertIn('const candles = visibleDailyPrices((series || [])', script)
|
||||
self.assertIn('renderStockNotes(payload.notes || [])', script)
|
||||
|
||||
|
||||
|
||||
@@ -3,9 +3,10 @@ from __future__ import annotations
|
||||
import http.client
|
||||
import json
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from backend.data.realtime import WebRealtimeAggregator
|
||||
from backend.data.realtime import RealtimeAggregateError, WebRealtimeAggregator
|
||||
from backend.features.heaven.engine import _market_line_scores, build_manual_market_hexagram
|
||||
from server import DashboardService
|
||||
from backend.data.providers.tushare_client import (
|
||||
@@ -377,6 +378,87 @@ class RealtimeAggregatorTests(unittest.TestCase):
|
||||
self.assertEqual(rows[0]["quote_time"][:10], "2026-07-20")
|
||||
self.assertAlmostEqual(rows[0]["amount_billion"], 12946.52)
|
||||
|
||||
@patch.object(WebRealtimeAggregator, "_get_json")
|
||||
def test_eastmoney_market_quotes_normalize_and_keep_expected_date(self, get_json: MagicMock):
|
||||
epoch = datetime(2026, 7, 20, 10, 5).timestamp()
|
||||
rows = []
|
||||
for index in range(200):
|
||||
sz = index < 100
|
||||
rows.append(
|
||||
{
|
||||
"f12": f"{index:06d}" if sz else f"{600000 + index - 100:06d}",
|
||||
"f13": 0 if sz else 1,
|
||||
"f14": f"股票{index}",
|
||||
"f2": 11.2,
|
||||
"f3": 2.0,
|
||||
"f5": 10,
|
||||
"f6": 50000000,
|
||||
"f15": 11.3,
|
||||
"f16": 11.0,
|
||||
"f17": 11.1,
|
||||
"f18": 11.0,
|
||||
"f124": epoch,
|
||||
}
|
||||
)
|
||||
def fake_get_json(_url, params, referer=""):
|
||||
page = int(params.get("pn") or 1)
|
||||
start = (page - 1) * 100
|
||||
return {"rc": 0, "data": {"total": 200, "diff": rows[start:start + 100]}}
|
||||
|
||||
get_json.side_effect = fake_get_json
|
||||
aggregator = WebRealtimeAggregator()
|
||||
aggregator._response_cache.clear()
|
||||
quotes = aggregator.eastmoney_market_quotes("20260720")
|
||||
self.assertEqual(len(quotes), 200)
|
||||
self.assertEqual(quotes[0]["ts_code"], "000000.SZ")
|
||||
self.assertTrue(quotes[100]["ts_code"].endswith(".SH"))
|
||||
self.assertEqual(quotes[0]["vol"], 1000)
|
||||
self.assertEqual(quotes[0]["quote_date"], "20260720")
|
||||
|
||||
@patch.object(WebRealtimeAggregator, "_get_text")
|
||||
def test_tencent_stock_quote_keeps_expected_date(self, get_text: MagicMock):
|
||||
fields = [""] * 38
|
||||
fields[1] = "浦发银行"
|
||||
fields[2] = "600000"
|
||||
fields[3] = "11.20"
|
||||
fields[4] = "11.00"
|
||||
fields[5] = "11.10"
|
||||
fields[6] = "1234"
|
||||
fields[30] = "20260720103000"
|
||||
fields[33] = "11.30"
|
||||
fields[34] = "11.00"
|
||||
fields[37] = "1380"
|
||||
get_text.return_value = (f'v_sh600000="{"~".join(fields)}";', 0)
|
||||
|
||||
quote = WebRealtimeAggregator().tencent_stock_quote("600000", "20260720")
|
||||
|
||||
self.assertEqual(quote["ts_code"], "600000.SH")
|
||||
self.assertEqual(quote["quote_date"], "20260720")
|
||||
self.assertEqual(quote["vol"], 123400)
|
||||
self.assertAlmostEqual(quote["amount"], 13_800_000)
|
||||
|
||||
@patch.object(WebRealtimeAggregator, "_get_json")
|
||||
def test_eastmoney_stock_quote_rejects_stale_date(self, get_json: MagicMock):
|
||||
epoch = datetime(2026, 7, 19, 15, 0).timestamp()
|
||||
get_json.return_value = {
|
||||
"rc": 0,
|
||||
"data": {
|
||||
"f43": 11.2,
|
||||
"f44": 11.3,
|
||||
"f45": 11.0,
|
||||
"f46": 11.1,
|
||||
"f47": 10,
|
||||
"f48": 50000000,
|
||||
"f57": "300750",
|
||||
"f58": "宁德时代",
|
||||
"f60": 11.0,
|
||||
"f86": epoch,
|
||||
},
|
||||
}
|
||||
|
||||
with self.assertRaises(RealtimeAggregateError):
|
||||
WebRealtimeAggregator().eastmoney_stock_quote("300750.SZ", "20260720")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,422 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from backend.data.providers.tushare_client import TushareClient, TushareError
|
||||
from backend.data.providers.tushare_helpers import _moneyflow_payload
|
||||
from backend.data.realtime import (
|
||||
WebRealtimeAggregator,
|
||||
_normalize_eastmoney_limit_row,
|
||||
_normalize_eastmoney_stock_quote,
|
||||
)
|
||||
from backend.data.providers.tushare_daily import DailyMarketMixin
|
||||
|
||||
|
||||
class MoneyflowPayloadTests(unittest.TestCase):
|
||||
def test_missing_row_is_not_zero(self) -> None:
|
||||
payload = _moneyflow_payload(None)
|
||||
self.assertFalse(payload["available"])
|
||||
self.assertIsNone(payload["net_million"])
|
||||
self.assertIsNone(payload["large_million"])
|
||||
|
||||
def test_empty_row_is_not_zero(self) -> None:
|
||||
payload = _moneyflow_payload({})
|
||||
self.assertFalse(payload["available"])
|
||||
self.assertIsNone(payload["net_million"])
|
||||
|
||||
def test_real_zero_net_is_kept_when_source_exists(self) -> None:
|
||||
payload = _moneyflow_payload(
|
||||
{
|
||||
"net_mf_amount": 0,
|
||||
"buy_lg_amount": 1,
|
||||
"sell_lg_amount": 1,
|
||||
"buy_elg_amount": 0,
|
||||
"sell_elg_amount": 0,
|
||||
"buy_md_amount": 0,
|
||||
"sell_md_amount": 0,
|
||||
"buy_sm_amount": 0,
|
||||
"sell_sm_amount": 0,
|
||||
}
|
||||
)
|
||||
self.assertTrue(payload["available"])
|
||||
self.assertEqual(payload["net_million"], 0)
|
||||
|
||||
|
||||
class LimitOverlayTests(unittest.TestCase):
|
||||
def test_normalize_limit_keeps_missing_seal_as_none(self) -> None:
|
||||
row = DailyMarketMixin._normalize_limit(
|
||||
{
|
||||
"ts_code": "000737.SZ",
|
||||
"name": "北方铜业",
|
||||
"close": 12.3,
|
||||
"pct_chg": 10,
|
||||
"amount": 1e8,
|
||||
"amount_unit": "yuan",
|
||||
},
|
||||
"涨停",
|
||||
)
|
||||
self.assertIsNone(row["seal_amount_million"])
|
||||
self.assertEqual(row["first_time"], "--")
|
||||
|
||||
def test_overlay_fills_board_times_from_official_list(self) -> None:
|
||||
mixin = DailyMarketMixin()
|
||||
mixin._load_limit_lists = lambda trade_date: [
|
||||
{
|
||||
"ts_code": "000737.SZ",
|
||||
"first_time": "09:31:02",
|
||||
"last_time": "10:18:11",
|
||||
"fd_amount": 82000000,
|
||||
"open_times": 1,
|
||||
"turnover_ratio": 18.4,
|
||||
}
|
||||
]
|
||||
mixin.realtime_aggregator = None
|
||||
rows = mixin._overlay_board_fields(
|
||||
[{"ts_code": "000737.SZ", "close": 12.3, "limit_type": "U"}],
|
||||
"20260908",
|
||||
)
|
||||
self.assertEqual(rows[0]["first_time"], "09:31:02")
|
||||
self.assertEqual(rows[0]["fd_amount"], 82000000)
|
||||
self.assertEqual(rows[0]["turnover_ratio"], 18.4)
|
||||
|
||||
def test_overlay_replaces_provisional_first_board_with_observed_streak(self) -> None:
|
||||
mixin = DailyMarketMixin()
|
||||
mixin._load_limit_lists = lambda trade_date: []
|
||||
mixin.try_limit_pool = lambda trade_date: [
|
||||
{"ts_code": "000737.SZ", "limit_times": 3}
|
||||
]
|
||||
|
||||
rows = mixin._overlay_board_fields(
|
||||
[{"ts_code": "000737.SZ", "limit_times": 1, "limit_type": "U"}],
|
||||
"20260909",
|
||||
)
|
||||
|
||||
self.assertEqual(rows[0]["limit_times"], 3)
|
||||
|
||||
def test_daily_fallback_extends_yesterday_streak(self) -> None:
|
||||
mixin = DailyMarketMixin()
|
||||
mixin.query = lambda *args, **kwargs: []
|
||||
mixin._load_limit_lists = lambda trade_date: []
|
||||
|
||||
rows = mixin._derive_limits(
|
||||
"20260909",
|
||||
[
|
||||
{
|
||||
"ts_code": "000737.SZ",
|
||||
"trade_date": "20260909",
|
||||
"close": 11.0,
|
||||
"high": 11.0,
|
||||
"pct_chg": 10.0,
|
||||
"amount": 100000,
|
||||
}
|
||||
],
|
||||
price_limits=[
|
||||
{"ts_code": "000737.SZ", "up_limit": 11.0, "down_limit": 9.0}
|
||||
],
|
||||
basic_rows=[
|
||||
{"ts_code": "000737.SZ", "name": "北方铜业", "industry": "有色"}
|
||||
],
|
||||
previous_limit_rows=[
|
||||
{"ts_code": "000737.SZ", "limit_times": 2}
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual(rows[0]["limit_times"], 3)
|
||||
|
||||
|
||||
class ShenwanRealtimeSourceTests(unittest.TestCase):
|
||||
def test_transport_refuses_rt_sw_k(self) -> None:
|
||||
client = TushareClient(token="demo")
|
||||
with self.assertRaisesRegex(TushareError, "rt_sw_k is disabled"):
|
||||
client.query("rt_sw_k", {"ts_code": "801074.SI"})
|
||||
|
||||
def test_outer_realtime_uses_hub_sector_quote_not_rt_sw_k(self) -> None:
|
||||
client = TushareClient(token="demo")
|
||||
client.query = MagicMock(side_effect=AssertionError("should not call tushare"))
|
||||
client.try_sector_quote = MagicMock(return_value={
|
||||
"code": "801074.SI",
|
||||
"name": "工业金属",
|
||||
"close": 1234.5,
|
||||
"pre_close": 1200,
|
||||
"change": 2.88,
|
||||
"pct_change": 2.88,
|
||||
"quote_date": "20260908",
|
||||
"quote_time": "2026-09-08T14:50:00+08:00",
|
||||
"source": "eastmoney_sw",
|
||||
})
|
||||
row, source, error = client._sw_outer_realtime("801074.SI", "工业金属", "20260908")
|
||||
self.assertEqual(source, "eastmoney_sw")
|
||||
self.assertEqual(error, "")
|
||||
self.assertEqual(row["change"], 2.88)
|
||||
client.query.assert_not_called()
|
||||
|
||||
def test_outer_waiting_state_has_no_permission_error(self) -> None:
|
||||
client = TushareClient(token="demo")
|
||||
client.realtime_aggregator = None
|
||||
row, source, error = client._sw_outer_realtime(
|
||||
"801074.SI", "工业金属", "20260908", finalized=True
|
||||
)
|
||||
self.assertEqual(row, {})
|
||||
self.assertIn("尚未入库", error)
|
||||
self.assertNotIn("权限", error)
|
||||
self.assertNotIn("rt_sw_k", error)
|
||||
|
||||
|
||||
class EastmoneyHelperTests(unittest.TestCase):
|
||||
def test_limit_pool_row_keeps_board_clock(self) -> None:
|
||||
parsed = _normalize_eastmoney_limit_row(
|
||||
{
|
||||
"c": "000737",
|
||||
"m": 0,
|
||||
"n": "北方铜业",
|
||||
"fbt": 93102,
|
||||
"lbt": 101811,
|
||||
"zbc": 1,
|
||||
"lbc": 2,
|
||||
"hs": 18.4,
|
||||
"fund": 82000000,
|
||||
},
|
||||
"U",
|
||||
)
|
||||
self.assertEqual(parsed["ts_code"], "000737.SZ")
|
||||
self.assertEqual(parsed["first_time"], "09:31:02")
|
||||
self.assertEqual(parsed["last_time"], "10:18:11")
|
||||
self.assertEqual(parsed["fd_amount"], 82000000)
|
||||
|
||||
def test_stock_quote_keeps_moneyflow_when_present(self) -> None:
|
||||
quote = _normalize_eastmoney_stock_quote(
|
||||
{
|
||||
"f43": 12.3,
|
||||
"f60": 11.18,
|
||||
"f46": 11.2,
|
||||
"f44": 12.3,
|
||||
"f45": 11.1,
|
||||
"f47": 1000,
|
||||
"f48": 150000000,
|
||||
"f58": "北方铜业",
|
||||
"f86": 0,
|
||||
"f168": 8.5,
|
||||
"f62": 25000000,
|
||||
"f78": 3000000,
|
||||
"f84": -1000000,
|
||||
},
|
||||
"000737.SZ",
|
||||
)
|
||||
self.assertEqual(quote["net_mf_amount"], 2500)
|
||||
payload = _moneyflow_payload(quote)
|
||||
self.assertTrue(payload["available"])
|
||||
self.assertEqual(payload["net_million"], 25)
|
||||
|
||||
@patch.object(WebRealtimeAggregator, "_get_json")
|
||||
def test_shenwan_quote_uses_eastmoney_90_prefix(self, get_json: MagicMock) -> None:
|
||||
get_json.return_value = {
|
||||
"rc": 0,
|
||||
"data": {
|
||||
"diff": [
|
||||
{
|
||||
"f12": "801074",
|
||||
"f14": "工业金属",
|
||||
"f2": 1234.5,
|
||||
"f3": 2.88,
|
||||
"f18": 1200,
|
||||
"f17": 1205,
|
||||
"f15": 1240,
|
||||
"f16": 1198,
|
||||
"f6": 1,
|
||||
"f124": 1757319000,
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
quote = WebRealtimeAggregator().eastmoney_shenwan_quote("801074.SI")
|
||||
self.assertEqual(quote["source"], "eastmoney_sw")
|
||||
self.assertAlmostEqual(quote["change"], 2.88)
|
||||
params = get_json.call_args.args[1]
|
||||
self.assertEqual(params["secids"], "90.801074")
|
||||
|
||||
|
||||
class ChartWindowTests(unittest.TestCase):
|
||||
def test_display_window_is_45_not_250(self) -> None:
|
||||
from backend.features.market.charts import DAILY_CHART_LIMIT
|
||||
|
||||
self.assertEqual(DAILY_CHART_LIMIT, 45)
|
||||
|
||||
|
||||
class MemberQuoteCoverageTests(unittest.TestCase):
|
||||
def test_prefers_full_hub_market_over_truncated_named_quotes(self) -> None:
|
||||
client = TushareClient(token="demo")
|
||||
wanted = [f"{index:06d}.SZ" for index in range(205)]
|
||||
market = [
|
||||
{"ts_code": code, "close": 10.0, "pre_close": 9.0}
|
||||
for code in wanted
|
||||
]
|
||||
client.try_market_quotes = MagicMock(return_value=market)
|
||||
client.try_quotes = MagicMock(return_value=market[:60])
|
||||
client.realtime_aggregator = MagicMock()
|
||||
rows, source = client._load_member_realtime_quotes(wanted, "20260908")
|
||||
self.assertEqual(len(rows), 205)
|
||||
self.assertEqual(source, "datahub")
|
||||
client.try_quotes.assert_not_called()
|
||||
|
||||
def test_hub_named_quotes_cover_members_when_market_missing(self) -> None:
|
||||
client = TushareClient(token="demo")
|
||||
wanted = ["000737.SZ", "000630.SZ"]
|
||||
client.try_market_quotes = MagicMock(return_value=None)
|
||||
client.try_quotes = MagicMock(return_value=[
|
||||
{"ts_code": "000737.SZ", "close": 12.3, "pre_close": 11.2},
|
||||
{"ts_code": "000630.SZ", "close": 4.5, "pre_close": 4.4},
|
||||
])
|
||||
client.realtime_aggregator = MagicMock()
|
||||
rows, source = client._load_member_realtime_quotes(wanted, "20260908")
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertEqual(source, "datahub")
|
||||
client.try_quotes.assert_called()
|
||||
client.realtime_aggregator.eastmoney_stock_quotes.assert_not_called()
|
||||
client.realtime_aggregator.tencent_stock_quotes.assert_not_called()
|
||||
|
||||
def test_delayed_hub_quotes_are_kept_not_zeroed(self) -> None:
|
||||
client = TushareClient(token="demo")
|
||||
delayed = [
|
||||
{
|
||||
"ts_code": "000737.SZ",
|
||||
"close": 12.3,
|
||||
"pre_close": 11.2,
|
||||
"delayed": True,
|
||||
"delay_seconds": 90,
|
||||
"delay_notice": "主备免费行情均暂不可用,显示 90 秒前的真实快照",
|
||||
}
|
||||
]
|
||||
client.try_market_quotes = MagicMock(return_value=delayed)
|
||||
client.try_quotes = MagicMock()
|
||||
client.realtime_aggregator = MagicMock()
|
||||
rows, source = client._load_member_realtime_quotes(["000737.SZ"], "20260908")
|
||||
self.assertEqual(source, "datahub_delayed")
|
||||
self.assertEqual(rows[0]["close"], 12.3)
|
||||
client.try_quotes.assert_not_called()
|
||||
|
||||
def test_ignores_non_member_quotes_from_market_snapshot(self) -> None:
|
||||
client = TushareClient(token="demo")
|
||||
client.try_market_quotes = MagicMock(
|
||||
return_value=[
|
||||
{"ts_code": "000737.SZ", "close": 12.3, "pre_close": 11.2},
|
||||
{"ts_code": "600000.SH", "close": 10.0, "pre_close": 9.9},
|
||||
]
|
||||
)
|
||||
client.try_quotes = MagicMock(return_value=[])
|
||||
client._free_realtime_quotes = MagicMock(return_value=([], "empty"))
|
||||
rows, _source = client._load_member_realtime_quotes(
|
||||
["000737.SZ", "000630.SZ"], "20260908"
|
||||
)
|
||||
self.assertEqual([row["ts_code"] for row in rows], ["000737.SZ"])
|
||||
|
||||
def test_local_sw_members_survive_tushare_outage(self) -> None:
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from backend.data.providers import tushare_industries as module
|
||||
|
||||
client = TushareClient(token="demo")
|
||||
stored = [
|
||||
{
|
||||
"ts_code": "000737.SZ",
|
||||
"name": "北方铜业",
|
||||
"l2_code": "801074.SI",
|
||||
"in_date": "20200101",
|
||||
"out_date": "",
|
||||
}
|
||||
]
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
original = module._SW_MEMBER_DIR
|
||||
module._SW_MEMBER_DIR = Path(tmp)
|
||||
try:
|
||||
client._sw_member_cache.clear()
|
||||
client._write_local_sw_members("801074.SI", stored)
|
||||
client.query = MagicMock(side_effect=TushareError("index_member_all down"))
|
||||
members = client._sw_sector_members("801074.SI", "20260908")
|
||||
finally:
|
||||
module._SW_MEMBER_DIR = original
|
||||
client._sw_member_cache.clear()
|
||||
self.assertEqual([item["ts_code"] for item in members], ["000737.SZ"])
|
||||
client.query.assert_not_called()
|
||||
|
||||
def test_closed_keeps_daily_inner_when_sw_daily_missing(self) -> None:
|
||||
client = TushareClient(token="demo")
|
||||
client.resolve_trade_context = lambda _date: ("20260908", "20260907")
|
||||
client.sw_stock_industry = MagicMock(
|
||||
return_value={"l2_code": "801074.SI", "l2_name": "工业金属"}
|
||||
)
|
||||
client._sw_sector_members = MagicMock(
|
||||
return_value=[
|
||||
{"ts_code": "000737.SZ", "name": "北方铜业"},
|
||||
{"ts_code": "000630.SZ", "name": "铜陵有色"},
|
||||
]
|
||||
)
|
||||
client._stock_listing_reference = MagicMock(return_value={})
|
||||
client._load_daily = MagicMock(
|
||||
return_value=[
|
||||
{"ts_code": "000737.SZ", "name": "北方铜业", "pct_chg": 2, "amount": 1e8},
|
||||
{"ts_code": "000630.SZ", "name": "铜陵有色", "pct_chg": 1, "amount": 1e8},
|
||||
]
|
||||
)
|
||||
client._confirmed_suspended_members = MagicMock(return_value=[])
|
||||
client.query = MagicMock(return_value=[])
|
||||
client._sw_realtime_sector_snapshot = MagicMock(
|
||||
side_effect=AssertionError("daily inner should be kept")
|
||||
)
|
||||
client.try_sector_quote = MagicMock(return_value={
|
||||
"code": "801074.SI",
|
||||
"name": "工业金属",
|
||||
"change": 1.5,
|
||||
"pct_change": 1.5,
|
||||
"quote_date": "20260908",
|
||||
"quote_time": "2026-09-08T15:00:00+08:00",
|
||||
"source": "eastmoney_sw",
|
||||
})
|
||||
snapshot = client.sw_sector_snapshot(
|
||||
"000737.SZ", "20260908", allow_realtime_close=True
|
||||
)
|
||||
self.assertEqual(snapshot["quote_count"], 2)
|
||||
self.assertEqual(snapshot["member_count"], 2)
|
||||
self.assertTrue(snapshot["inner_precise"])
|
||||
self.assertTrue(snapshot["outer_precise"])
|
||||
self.assertEqual(snapshot["inner_source"], "tushare_member_daily")
|
||||
self.assertEqual(snapshot["change"], 1.5)
|
||||
self.assertNotIn("权限", snapshot.get("outer_error") or "")
|
||||
self.assertNotIn("rt_sw_k", snapshot.get("outer_error") or "")
|
||||
|
||||
def test_closed_uses_complete_member_daily_when_sector_quote_is_wrong(self) -> None:
|
||||
client = TushareClient(token="demo")
|
||||
client.resolve_trade_context = lambda _date: ("20260909", "20260908")
|
||||
client.sw_stock_industry = MagicMock(
|
||||
return_value={"l2_code": "801074.SI", "l2_name": "专用设备"}
|
||||
)
|
||||
client._sw_sector_members = MagicMock(return_value=[
|
||||
{"ts_code": "603011.SH", "name": "合锻智能"},
|
||||
{"ts_code": "000821.SZ", "name": "京山轻机"},
|
||||
])
|
||||
client._stock_listing_reference = MagicMock(return_value={})
|
||||
client._load_daily = MagicMock(return_value=[
|
||||
{"ts_code": "603011.SH", "name": "合锻智能", "pct_chg": 2, "amount": 1e8},
|
||||
{"ts_code": "000821.SZ", "name": "京山轻机", "pct_chg": -1, "amount": 1e8},
|
||||
])
|
||||
client._confirmed_suspended_members = MagicMock(return_value=[])
|
||||
client.query = MagicMock(return_value=[])
|
||||
client.try_sector_quote = MagicMock(return_value={
|
||||
"code": "801074.SI",
|
||||
"name": "托育服务",
|
||||
"change": -2.19,
|
||||
"quote_date": "20260909",
|
||||
"quote_time": "2026-09-09T15:00:00+08:00",
|
||||
"source": "eastmoney_sw",
|
||||
})
|
||||
|
||||
snapshot = client.sw_sector_snapshot(
|
||||
"603011.SH", "20260909", allow_realtime_close=True
|
||||
)
|
||||
|
||||
self.assertTrue(snapshot["precise"])
|
||||
self.assertEqual(snapshot["name"], "专用设备")
|
||||
self.assertEqual(snapshot["change"], 0.5)
|
||||
self.assertEqual(snapshot["outer_source"], "sw_member_equal_daily")
|
||||
self.assertEqual(snapshot["outer_error"], "")
|
||||
@@ -138,7 +138,7 @@ class HttpDispatchContractTests(unittest.TestCase):
|
||||
self.assertTrue(claimed.isdisjoint(methods))
|
||||
claimed.update(methods)
|
||||
self.assertLessEqual(len(path.read_text(encoding="utf-8").splitlines()), line_limit)
|
||||
self.assertEqual(len(claimed), 27)
|
||||
self.assertEqual(len(claimed), 28)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,434 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from backend.data import build_data_gateway
|
||||
from backend.data.datahub.bridge import DatahubAwareTushareClient, DatahubBridge
|
||||
from backend.data.datahub.client import DatahubClient
|
||||
from backend.data.datahub.ifind_proxy import HubIfindProxy
|
||||
from backend.data.datahub.realtime_proxy import HubRealtimeProxy
|
||||
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
||||
from backend.data.providers.tushare_transport import TushareError
|
||||
from backend.features.market.charts import MarketChartClient
|
||||
from backend.features.system.service import SystemServiceMixin
|
||||
from tests.test_datahub_bridge import FakeClient, FakeLegacy, flags
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BLOCKED_HOSTS = (
|
||||
"api.tushare.pro",
|
||||
"push2.eastmoney.com",
|
||||
"push2delay.eastmoney.com",
|
||||
"push2his.eastmoney.com",
|
||||
"push2ex.eastmoney.com",
|
||||
"qt.gtimg.cn",
|
||||
"hq.sinajs.cn",
|
||||
"10jqka.com.cn",
|
||||
"xuangubao.cn",
|
||||
"quantapi.51ifind.com",
|
||||
"51ifind.com",
|
||||
)
|
||||
LEFTOVER_WEBSITE_FILES = {
|
||||
"backend/data/providers/ifind_client.py",
|
||||
"backend/data/realtime.py",
|
||||
"backend/features/market/charts.py",
|
||||
"backend/data/providers/tushare_transport.py",
|
||||
}
|
||||
HUB_BASE = "http://127.0.0.1:8766"
|
||||
|
||||
|
||||
def _enabled_settings() -> DatahubSettings:
|
||||
return DatahubSettings(
|
||||
base_url=HUB_BASE,
|
||||
token="hub-token",
|
||||
datasets={name: DatasetFlags(name, read=True) for name in DATASETS},
|
||||
)
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, payload: dict) -> None:
|
||||
self.status = 200
|
||||
self._raw = json.dumps(payload).encode("utf-8")
|
||||
|
||||
def read(self):
|
||||
return self._raw
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
|
||||
def hub_payload(request) -> dict:
|
||||
url = str(getattr(request, "full_url", None) or request)
|
||||
if any(host in url for host in BLOCKED_HOSTS):
|
||||
raise AssertionError(f"website opened blocked host: {url}")
|
||||
if HUB_BASE not in url:
|
||||
raise AssertionError(f"unexpected url: {url}")
|
||||
path = url.split(HUB_BASE, 1)[1].split("?", 1)[0]
|
||||
if path == "/v1/bars/daily":
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"data": [
|
||||
{
|
||||
"ts_code": "600000.SH",
|
||||
"trade_date": "20240902",
|
||||
"open": 10.0,
|
||||
"high": 10.4,
|
||||
"low": 9.9,
|
||||
"close": 10.2,
|
||||
"volume": 100000,
|
||||
"amount": 2000000,
|
||||
}
|
||||
],
|
||||
"meta": {"stale": False, "staleness_seconds": 0, "source": "datahub"},
|
||||
}
|
||||
if path == "/v1/quotes/latest":
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"data": [
|
||||
{
|
||||
"ts_code": "600000.SH",
|
||||
"code": "600000",
|
||||
"name": "浦发银行",
|
||||
"close": 10.2,
|
||||
"price": 10.2,
|
||||
"pre_close": 10.0,
|
||||
"open": 10.1,
|
||||
"high": 10.3,
|
||||
"low": 9.9,
|
||||
"vol": 1000,
|
||||
"amount": 2000000,
|
||||
"quote_date": "20240902",
|
||||
"source": "datahub",
|
||||
}
|
||||
],
|
||||
"meta": {"stale": False, "staleness_seconds": 0, "source": "eastmoney"},
|
||||
}
|
||||
if path == "/v1/indexes/quotes":
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"data": [
|
||||
{
|
||||
"ts_code": "000001.SH",
|
||||
"code": "000001",
|
||||
"name": "上证指数",
|
||||
"price": 3000,
|
||||
"pct_chg": 1.2,
|
||||
"quote_time_epoch": 1725249600,
|
||||
"source": "datahub",
|
||||
},
|
||||
{
|
||||
"ts_code": "399001.SZ",
|
||||
"code": "399001",
|
||||
"name": "深证成指",
|
||||
"price": 9000,
|
||||
"pct_chg": 0.8,
|
||||
"quote_time_epoch": 1725249600,
|
||||
"source": "datahub",
|
||||
},
|
||||
{
|
||||
"ts_code": "399006.SZ",
|
||||
"code": "399006",
|
||||
"name": "创业板指",
|
||||
"price": 1800,
|
||||
"pct_chg": 0.5,
|
||||
"quote_time_epoch": 1725249600,
|
||||
"source": "datahub",
|
||||
},
|
||||
],
|
||||
"meta": {"stale": False, "staleness_seconds": 0, "source": "tencent"},
|
||||
}
|
||||
if path == "/v1/auction":
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"data": [
|
||||
{
|
||||
"ts_code": "600000.SH",
|
||||
"trade_date": "20240902",
|
||||
"close": 10.2,
|
||||
"vol": 1000.0,
|
||||
"amount": 2000.0,
|
||||
}
|
||||
],
|
||||
"meta": {"stale": False, "staleness_seconds": 0, "source": "datahub"},
|
||||
}
|
||||
if path == "/v1/credentials/ifind":
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"data": {"configured": True, "access_ready": True, "access_expires_at": ""},
|
||||
"meta": {"source": "ifind"},
|
||||
}
|
||||
if path == "/v1/intraday/points":
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"data": {
|
||||
"entity_type": "stock",
|
||||
"identifier": "600000",
|
||||
"code": "600000",
|
||||
"trade_date": "2024-09-02",
|
||||
"previous_close": 10.0,
|
||||
"points": [
|
||||
{"date": "2024-09-02", "time": "09:30", "close": 10.2, "average": 10.1, "open": 10.1, "high": 10.2, "low": 10.0, "volume": 100, "amount": 1000}
|
||||
],
|
||||
},
|
||||
"meta": {"stale": False, "source": "datahub"},
|
||||
}
|
||||
if path == "/v1/query":
|
||||
body = json.loads(request.data.decode("utf-8") if request.data else "{}")
|
||||
api_name = body.get("api_name")
|
||||
if api_name == "ifind_status":
|
||||
return {"schema_version": 1, "data": [{"configured": True, "access_ready": True, "access_expires_at": ""}], "meta": {"source": "ifind"}}
|
||||
if api_name == "ifind_wencai":
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"data": [{"股票代码": "000001.SZ", "涨停原因": "重组", "首次涨停时间": "09:42:00"}],
|
||||
"meta": {"source": "ifind"},
|
||||
}
|
||||
if api_name == "ifind_snapshots":
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"data": [
|
||||
{
|
||||
"time": "2024-09-02 09:21:00",
|
||||
"thscode": "000001.SZ",
|
||||
"latest": 10.5,
|
||||
"preClose": 10,
|
||||
"volume": 2000,
|
||||
"amount": 21000,
|
||||
}
|
||||
],
|
||||
"meta": {"source": "ifind"},
|
||||
}
|
||||
if api_name in {"daily", "rt_k", "stk_auction"}:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"data": [{"ts_code": "600000.SH", "trade_date": "20240902", "close": 10.2, "amount": 2000.0}],
|
||||
"meta": {"source": "datahub", "stale": False, "row_shape": "tushare"},
|
||||
}
|
||||
raise AssertionError(f"unexpected query api: {api_name}")
|
||||
raise AssertionError(f"unexpected path: {path}")
|
||||
|
||||
|
||||
def blocked_urlopen(request, timeout=None):
|
||||
return _Resp(hub_payload(request))
|
||||
|
||||
|
||||
class HubExclusiveWebsiteTests(unittest.TestCase):
|
||||
def test_website_availability_depends_on_hub_not_provider_credentials(self) -> None:
|
||||
service = SystemServiceMixin()
|
||||
service._system_credentials = {}
|
||||
service.data_gateway = build_data_gateway({}, datahub_settings=_enabled_settings())
|
||||
self.assertTrue(service.configured)
|
||||
self.assertEqual(service.token, "datahub")
|
||||
|
||||
def test_website_vault_scrubs_historical_provider_credentials(self) -> None:
|
||||
class Vault:
|
||||
@staticmethod
|
||||
def decrypt_json(payload):
|
||||
if isinstance(payload, str):
|
||||
return json.loads(payload)
|
||||
return dict(payload)
|
||||
|
||||
@staticmethod
|
||||
def encrypt_json(payload):
|
||||
return dict(payload)
|
||||
|
||||
class Database:
|
||||
system = {
|
||||
"tushare_token": "old-tushare",
|
||||
"ifind_refresh_token": "old-refresh",
|
||||
"ifind_access_token": "old-access",
|
||||
}
|
||||
personal = {"tushare_token": "old-personal", "llm_primary_model": "model"}
|
||||
|
||||
def get_system_setting(self, _name):
|
||||
return dict(self.system)
|
||||
|
||||
@staticmethod
|
||||
def first_user_id():
|
||||
return 1
|
||||
|
||||
def get_user_credentials(self, _user_id):
|
||||
return dict(self.personal)
|
||||
|
||||
@staticmethod
|
||||
def list_user_credentials():
|
||||
return [{"user_id": 1, "encrypted_payload": '{"tushare_token":"old-personal"}'}]
|
||||
|
||||
def save_system_setting(self, _name, payload):
|
||||
self.system = dict(payload)
|
||||
|
||||
def save_user_credentials(self, _user_id, payload):
|
||||
self.personal = dict(payload)
|
||||
|
||||
service = SystemServiceMixin()
|
||||
service.database = Database()
|
||||
service.vault = Vault()
|
||||
service.system_lock = threading.Lock()
|
||||
loaded = service._load_system_credentials({})
|
||||
for key in ("tushare_token", "ifind_refresh_token", "ifind_access_token"):
|
||||
self.assertNotIn(key, loaded)
|
||||
self.assertNotIn(key, service.database.system)
|
||||
self.assertNotIn(key, service.database.personal)
|
||||
|
||||
def test_query_never_calls_website_tushare_transport(self) -> None:
|
||||
client = FakeClient()
|
||||
legacy = FakeLegacy(TushareError("website tushare must stay dark"))
|
||||
wrapped = DatahubAwareTushareClient(
|
||||
legacy,
|
||||
DatahubBridge(flags(daily=(True, False)), client),
|
||||
)
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(legacy.calls, [])
|
||||
|
||||
def test_blocked_external_hosts_still_read_hub(self) -> None:
|
||||
settings = _enabled_settings()
|
||||
hub_client = DatahubClient(settings, urlopen=blocked_urlopen)
|
||||
legacy = FakeLegacy(TushareError("blocked"))
|
||||
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(settings, hub_client))
|
||||
with patch("urllib.request.urlopen", blocked_urlopen):
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,close,amount")
|
||||
self.assertEqual(rows[0]["close"], 10.2)
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(legacy.calls, [])
|
||||
|
||||
def test_site_features_only_reach_hub_when_external_hosts_blocked(self) -> None:
|
||||
settings = _enabled_settings()
|
||||
hub_client = DatahubClient(settings, urlopen=blocked_urlopen)
|
||||
bridge = DatahubBridge(settings, hub_client)
|
||||
with patch("urllib.request.urlopen", blocked_urlopen):
|
||||
quotes = bridge.try_quotes(["600000.SH"])
|
||||
indices = bridge.try_index_quotes()
|
||||
chart = MarketChartClient(bridge).stock_daily("600000", "20240902")
|
||||
intraday = MarketChartClient(bridge).stock_intraday("600000")
|
||||
ifind = HubIfindProxy(bridge)
|
||||
wencai = ifind.wencai("2024年9月2日涨停股票")
|
||||
snapshots = ifind.snapshots(["000001.SZ"], ["latest"], "2024-09-02 09:15:00", "2024-09-02 09:22:00")
|
||||
realtime = HubRealtimeProxy(bridge)
|
||||
index_rows = realtime.tencent_indices()
|
||||
stock = realtime.tencent_stock_quote("600000.SH", "20240902")
|
||||
health = realtime.health_snapshot()
|
||||
|
||||
self.assertEqual(quotes[0]["close"], 10.2)
|
||||
self.assertEqual(len(indices), 3)
|
||||
self.assertEqual(chart[-1]["close"], 10.2)
|
||||
self.assertEqual(intraday["source"], "datahub")
|
||||
self.assertEqual(wencai[0]["股票代码"], "000001.SZ")
|
||||
self.assertEqual(snapshots[0]["latest"], 10.5)
|
||||
self.assertEqual([row["code"] for row in index_rows], ["000001", "399001", "399006"])
|
||||
self.assertEqual(stock["close"], 10.2)
|
||||
self.assertTrue(health["ready"])
|
||||
self.assertTrue(ifind.configured)
|
||||
|
||||
def test_production_gateway_does_not_construct_external_clients(self) -> None:
|
||||
source = (ROOT / "backend" / "data" / "gateway.py").read_text(encoding="utf-8")
|
||||
self.assertNotIn("IfindHttpClient", source)
|
||||
self.assertNotIn("EastmoneyChartClient", source)
|
||||
self.assertNotIn("WebRealtimeAggregator", source)
|
||||
self.assertNotIn("TushareProvider", source)
|
||||
self.assertIsNone(re.search(r"(?<![A-Za-z])TushareClient\(", source))
|
||||
self.assertIn("HubIfindProxy", source)
|
||||
self.assertIn("HubRealtimeProxy", source)
|
||||
self.assertIn("DatahubAwareTushareClient", source)
|
||||
facade = (ROOT / "backend" / "data" / "datahub" / "bridge.py").read_text(encoding="utf-8")
|
||||
tree = ast.parse(facade)
|
||||
cls = next(
|
||||
node
|
||||
for node in tree.body
|
||||
if isinstance(node, ast.ClassDef) and node.name == "DatahubAwareTushareClient"
|
||||
)
|
||||
methods = {item.name for item in cls.body if isinstance(item, ast.FunctionDef)}
|
||||
self.assertNotIn("__getattr__", methods)
|
||||
self.assertIn("query", methods)
|
||||
self.assertTrue(any(base.id == "DashboardMixin" for base in cls.bases if isinstance(base, ast.Name)))
|
||||
|
||||
def test_production_python_does_not_embed_blocked_hosts(self) -> None:
|
||||
violations = []
|
||||
for path in (ROOT / "backend").rglob("*.py"):
|
||||
relative = path.relative_to(ROOT).as_posix()
|
||||
if relative in LEFTOVER_WEBSITE_FILES:
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8")
|
||||
for host in BLOCKED_HOSTS:
|
||||
if host in text:
|
||||
violations.append(f"{relative} -> {host}")
|
||||
self.assertEqual(violations, [])
|
||||
|
||||
def test_website_runtime_does_not_call_blocked_hosts_from_gateway(self) -> None:
|
||||
gateway_src = (ROOT / "backend" / "data" / "gateway.py").read_text(encoding="utf-8")
|
||||
self.assertNotIn("TushareProvider", gateway_src)
|
||||
self.assertIsNone(re.search(r"(?<![A-Za-z])TushareClient\(", gateway_src))
|
||||
self.assertIn("DatahubAwareTushareClient", gateway_src)
|
||||
|
||||
def test_bridge_query_has_no_legacy_call(self) -> None:
|
||||
source = (ROOT / "backend" / "data" / "datahub" / "bridge.py").read_text(encoding="utf-8")
|
||||
tree = ast.parse(source)
|
||||
query_fn = next(
|
||||
node
|
||||
for node in tree.body
|
||||
if isinstance(node, ast.ClassDef) and node.name == "DatahubBridge"
|
||||
for item in node.body
|
||||
if isinstance(item, ast.FunctionDef) and item.name == "query"
|
||||
)
|
||||
called = [
|
||||
ast.unparse(item.func) if hasattr(ast, "unparse") else ""
|
||||
for item in ast.walk(query_fn)
|
||||
if isinstance(item, ast.Call)
|
||||
]
|
||||
self.assertTrue(any("query_api" in text for text in called))
|
||||
self.assertFalse(any("legacy_query" in text for text in called))
|
||||
|
||||
def test_build_gateway_uses_hub_proxies_without_opening_external_hosts(self) -> None:
|
||||
settings = _enabled_settings()
|
||||
with patch("urllib.request.urlopen", blocked_urlopen):
|
||||
gateway = build_data_gateway({"tushare_token": "tok"}, datahub_settings=settings)
|
||||
hub_client = DatahubClient(settings, urlopen=blocked_urlopen)
|
||||
gateway.datahub.client = hub_client
|
||||
rows = gateway.ifind.wencai("涨停")
|
||||
quotes = gateway.realtime_observer.tencent_indices()
|
||||
chart = gateway.chart_data.stock_daily("600000", "20240902")
|
||||
market = gateway.tushare()
|
||||
market_quotes = market.try_quotes(["600000.SH"])
|
||||
auction = market.query("stk_auction", {"trade_date": "20240902"}, "")
|
||||
self.assertEqual(rows[0]["涨停原因"], "重组")
|
||||
self.assertEqual(len(quotes), 3)
|
||||
self.assertEqual(chart[-1]["close"], 10.2)
|
||||
self.assertEqual(market_quotes[0]["close"], 10.2)
|
||||
self.assertEqual(auction[0]["close"], 10.2)
|
||||
self.assertIsNone(market.realtime_aggregator)
|
||||
self.assertEqual(market.token, "datahub")
|
||||
|
||||
def test_website_does_not_accept_or_forward_provider_credentials(self) -> None:
|
||||
proxy = HubIfindProxy(DatahubBridge(_enabled_settings(), FakeClient()))
|
||||
self.assertFalse(hasattr(proxy, "set_credentials"))
|
||||
client_source = (ROOT / "backend" / "data" / "datahub" / "client.py").read_text(encoding="utf-8")
|
||||
application_source = (ROOT / "backend" / "application.py").read_text(encoding="utf-8")
|
||||
self.assertNotIn("/v1/credentials", client_source)
|
||||
self.assertNotIn("ifind_refresh_token", application_source)
|
||||
|
||||
def test_site_configuration_depends_on_hub_not_provider_credentials(self) -> None:
|
||||
service = SystemServiceMixin()
|
||||
service._system_credentials = {}
|
||||
service.data_gateway = build_data_gateway({}, datahub_settings=_enabled_settings())
|
||||
self.assertTrue(service.configured)
|
||||
self.assertEqual(service.token, "datahub")
|
||||
|
||||
def test_compose_passes_ifind_env_to_hub(self) -> None:
|
||||
overlay = (ROOT / "compose.datahub.yaml").read_text(encoding="utf-8")
|
||||
standalone = (ROOT / "xiaobai-datahub" / "compose.yaml").read_text(encoding="utf-8")
|
||||
for text in (overlay, standalone):
|
||||
self.assertIn('IFIND_REFRESH_TOKEN: "${IFIND_REFRESH_TOKEN:-}"', text)
|
||||
self.assertIn('IFIND_ACCESS_TOKEN: "${IFIND_ACCESS_TOKEN:-}"', text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -6,12 +6,26 @@ from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from backend.features.market.charts import EastmoneyChartClient, MarketChartClient
|
||||
from backend.features.market.charts import MarketChartClient
|
||||
from database import ReviewDatabase
|
||||
from backend.features.market.insights import MarketInsightsService
|
||||
from server import DashboardService
|
||||
|
||||
|
||||
class _FakeDailyHub:
|
||||
def __init__(self, rows: list) -> None:
|
||||
self.rows = rows
|
||||
|
||||
def try_daily_chart(self, code, end_date, limit, dataset="daily"):
|
||||
return list(self.rows)
|
||||
|
||||
def try_quotes(self, codes):
|
||||
return None
|
||||
|
||||
def try_index_quotes(self):
|
||||
return None
|
||||
|
||||
|
||||
class FakeIfind:
|
||||
configured = True
|
||||
|
||||
@@ -128,13 +142,50 @@ class IfindFeatureTests(unittest.TestCase):
|
||||
self.assertEqual(database.list_wencai_saved_queries(second["id"]), [])
|
||||
|
||||
def test_ifind_daily_chart_normalizes_change(self):
|
||||
client = MarketChartClient(FakeIfind(), EastmoneyChartClient())
|
||||
hub = _FakeDailyHub(
|
||||
[
|
||||
{
|
||||
"trade_date": "2026-07-27",
|
||||
"open": 10,
|
||||
"high": 10.5,
|
||||
"low": 9.8,
|
||||
"close": 10.2,
|
||||
"volume": 100,
|
||||
"amount_billion": 0.01,
|
||||
"change": 0,
|
||||
},
|
||||
{
|
||||
"trade_date": "2026-07-28",
|
||||
"open": 10.2,
|
||||
"high": 10.8,
|
||||
"low": 10.1,
|
||||
"close": 10.5,
|
||||
"volume": 120,
|
||||
"amount_billion": 0.012,
|
||||
"change": 2.9412,
|
||||
},
|
||||
]
|
||||
)
|
||||
client = MarketChartClient(hub)
|
||||
rows = client.stock_daily("000001", "20260728")
|
||||
self.assertEqual(rows[-1]["trade_date"], "2026-07-28")
|
||||
self.assertAlmostEqual(rows[-1]["change"], 2.9412, places=4)
|
||||
|
||||
def test_ifind_daily_chart_keeps_last_traded_bar_before_market_open(self):
|
||||
client = MarketChartClient(FakeIfindStalePreopen(), EastmoneyChartClient())
|
||||
hub = _FakeDailyHub(
|
||||
[
|
||||
{
|
||||
"trade_date": "2026-07-28",
|
||||
"open": 10.2,
|
||||
"high": 10.8,
|
||||
"low": 10.1,
|
||||
"close": 10.5,
|
||||
"volume": 120,
|
||||
"amount_billion": 0.012,
|
||||
}
|
||||
]
|
||||
)
|
||||
client = MarketChartClient(hub)
|
||||
with patch("backend.features.market.charts.datetime", FixedPreopenDatetime):
|
||||
rows = client.stock_daily("000001", "20260729")
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ class MobileSystemPagesRegressionTests(unittest.TestCase):
|
||||
"data-system-edit-model",
|
||||
"data-system-open-member",
|
||||
"管理员专区",
|
||||
"保存密钥",
|
||||
"刷新状态",
|
||||
"保存分工",
|
||||
'location.assign("/login/")',
|
||||
):
|
||||
|
||||
@@ -1,8 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from backend.data.providers.tushare_client import TushareClient
|
||||
from backend.data.providers.tushare_helpers import calendar_is_open
|
||||
from backend.data.providers.tushare_transport import TushareError
|
||||
from backend.data.realtime import (
|
||||
RealtimeAggregateError,
|
||||
_normalize_eastmoney_quote,
|
||||
_parse_tencent_stock_quote,
|
||||
)
|
||||
|
||||
|
||||
class FakeRealtimeClient(TushareClient):
|
||||
@@ -81,6 +89,72 @@ class FakeRealtimeClient(TushareClient):
|
||||
raise AssertionError(f"Unexpected API call: {api_name} {params}")
|
||||
|
||||
|
||||
FREE_QUOTES = [
|
||||
{
|
||||
"ts_code": "000001.SZ", "name": "甲", "pre_close": 10.0,
|
||||
"open": 10.1, "high": 11.0, "low": 10.0, "close": 11.0,
|
||||
"vol": 1000, "amount": 100000000, "num": 10,
|
||||
"quote_date": "20260720",
|
||||
},
|
||||
{
|
||||
"ts_code": "000002.SZ", "name": "乙", "pre_close": 20.0,
|
||||
"open": 19.5, "high": 20.0, "low": 18.0, "close": 18.0,
|
||||
"vol": 2000, "amount": 200000000, "num": 20,
|
||||
"quote_date": "20260720",
|
||||
},
|
||||
{
|
||||
"ts_code": "000003.SZ", "name": "丙", "pre_close": 30.0,
|
||||
"open": 31.0, "high": 33.0, "low": 30.0, "close": 32.0,
|
||||
"vol": 3000, "amount": 300000000, "num": 30,
|
||||
"quote_date": "20260720",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class FakeFreeAggregator:
|
||||
def __init__(self, quotes=None, fail=False):
|
||||
self.quotes = list(quotes if quotes is not None else FREE_QUOTES)
|
||||
self.fail = fail
|
||||
self.calls = 0
|
||||
|
||||
def eastmoney_market_quotes(self, expected_date=""):
|
||||
self.calls += 1
|
||||
if self.fail:
|
||||
raise RealtimeAggregateError("eastmoney down")
|
||||
if expected_date and self.quotes:
|
||||
dated = [
|
||||
row for row in self.quotes
|
||||
if str(row.get("quote_date") or "") == str(expected_date).replace("-", "")
|
||||
]
|
||||
if dated:
|
||||
return dated
|
||||
return list(self.quotes)
|
||||
|
||||
def tencent_market_quotes(self, codes, expected_date=""):
|
||||
return self.eastmoney_market_quotes(expected_date)
|
||||
|
||||
def tencent_stock_quotes(self, codes, expected_date="", minimum=None):
|
||||
wanted = {str(code).strip() for code in (codes or []) if str(code).strip()}
|
||||
rows = self.tencent_market_quotes(codes, expected_date)
|
||||
if not wanted:
|
||||
return rows
|
||||
return [row for row in rows if str(row.get("ts_code") or "") in wanted]
|
||||
|
||||
def eastmoney_indices(self):
|
||||
return [
|
||||
{
|
||||
"code": "000001",
|
||||
"name": "上证指数",
|
||||
"price": 3800.12,
|
||||
"change": 0.85,
|
||||
"previous_close": 3768.0,
|
||||
"amount_billion": 4200.5,
|
||||
"quote_time": "2026-07-20T10:05:00+08:00",
|
||||
"source": "eastmoney_push2",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class RealtimeDashboardTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
TushareClient._realtime_reference_cache.clear()
|
||||
@@ -103,13 +177,29 @@ class RealtimeDashboardTests(unittest.TestCase):
|
||||
|
||||
def test_realtime_stock_quote_uses_cached_industry(self):
|
||||
self.client._load_realtime_reference("20260720", "20260717")
|
||||
quote = self.client.realtime_stock_quote("000003.SZ")
|
||||
self.client.try_quotes = lambda codes: [
|
||||
row for row in FREE_QUOTES if row["ts_code"] in set(codes)
|
||||
]
|
||||
calls = []
|
||||
original_query = self.client.query
|
||||
self.client.query = lambda api, params=None, fields="": (
|
||||
calls.append((api, dict(params or {})))
|
||||
or original_query(api, params, fields)
|
||||
)
|
||||
quote = self.client.realtime_stock_quote("000003.SZ", "20260720")
|
||||
|
||||
self.assertEqual(quote["name"], "丙")
|
||||
self.assertEqual(quote["sector"], "元器件")
|
||||
self.assertAlmostEqual(quote["change"], 6.6667)
|
||||
self.assertEqual(quote["amount_billion"], 3.0)
|
||||
self.assertAlmostEqual(quote["turnover_rate"], 0.01)
|
||||
self.assertEqual(quote["trade_date"], "20260720")
|
||||
history_calls = [
|
||||
params for api, params in calls
|
||||
if api in {"daily", "daily_basic"} and params.get("start_date")
|
||||
]
|
||||
self.assertTrue(history_calls)
|
||||
self.assertTrue(all(params.get("end_date") == "20260719" for params in history_calls))
|
||||
|
||||
def test_close_dashboard_marks_official_limit_data(self):
|
||||
dashboard = self.client.dashboard("20260720")
|
||||
@@ -130,6 +220,255 @@ class RealtimeDashboardTests(unittest.TestCase):
|
||||
self.assertEqual(dashboard["meta"]["limit_data_source"], "derived")
|
||||
self.assertIn("日线数据推算", dashboard["meta"]["notice"])
|
||||
|
||||
def test_calendar_open_flag_accepts_string_and_bool(self):
|
||||
self.assertTrue(calendar_is_open(1))
|
||||
self.assertTrue(calendar_is_open("1"))
|
||||
self.assertTrue(calendar_is_open(True))
|
||||
self.assertFalse(calendar_is_open(0))
|
||||
self.assertFalse(calendar_is_open("0"))
|
||||
self.assertFalse(calendar_is_open(False))
|
||||
|
||||
original_query = self.client.query
|
||||
|
||||
def query(api_name, params=None, fields=""):
|
||||
if api_name == "trade_cal":
|
||||
return [
|
||||
{
|
||||
"cal_date": params.get("start_date"),
|
||||
"is_open": "1",
|
||||
"pretrade_date": "20260907",
|
||||
}
|
||||
]
|
||||
return original_query(api_name, params, fields)
|
||||
|
||||
self.client.query = query
|
||||
trade_date, previous = self.client.resolve_trade_context("20260908")
|
||||
self.assertEqual(trade_date, "20260908")
|
||||
self.assertEqual(previous, "20260907")
|
||||
|
||||
def test_session_clock_uses_realtime_until_official_window(self):
|
||||
today = "20260908"
|
||||
self.client.clock = lambda: datetime(
|
||||
2026, 9, 8, 10, 5, tzinfo=timezone(timedelta(hours=8))
|
||||
)
|
||||
self.assertTrue(self.client.should_use_realtime(today, today))
|
||||
self.client.clock = lambda: datetime(
|
||||
2026, 9, 8, 16, 10, tzinfo=timezone(timedelta(hours=8))
|
||||
)
|
||||
self.assertFalse(self.client.should_use_realtime(today, today))
|
||||
|
||||
def test_realtime_dashboard_survives_missing_limit_table(self):
|
||||
original_query = self.client.query
|
||||
|
||||
def query(api_name, params=None, fields=""):
|
||||
if api_name == "stk_limit":
|
||||
return []
|
||||
return original_query(api_name, params, fields)
|
||||
|
||||
self.client.query = query
|
||||
TushareClient._realtime_reference_cache.clear()
|
||||
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
||||
self.assertTrue(dashboard["meta"]["realtime"])
|
||||
self.assertEqual(dashboard["meta"]["quote_count"], 3)
|
||||
self.assertEqual(dashboard["overview"]["limit_up_count"], 0)
|
||||
|
||||
def test_hub_quotes_used_when_rt_k_denied(self):
|
||||
original_query = self.client.query
|
||||
|
||||
def query(api_name, params=None, fields=""):
|
||||
if api_name == "rt_k":
|
||||
raise TushareError("没有接口访问权限")
|
||||
return original_query(api_name, params, fields)
|
||||
|
||||
self.client.query = query
|
||||
self.client.try_market_quotes = lambda trade_date: list(FREE_QUOTES)
|
||||
TushareClient._realtime_reference_cache.clear()
|
||||
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
||||
|
||||
self.assertTrue(dashboard["meta"]["realtime"])
|
||||
self.assertEqual(dashboard["meta"]["quote_source"], "datahub")
|
||||
self.assertEqual(dashboard["meta"]["trade_date"], "2026-07-20")
|
||||
self.assertEqual(dashboard["meta"]["quote_count"], 3)
|
||||
self.assertEqual(dashboard["overview"]["limit_up_count"], 1)
|
||||
self.assertEqual(dashboard["overview"]["limit_down_count"], 1)
|
||||
self.assertEqual(dashboard["overview"]["amount_billion"], 6.0)
|
||||
self.assertIn("数据中枢", dashboard["meta"]["notice"])
|
||||
|
||||
def test_hub_quotes_used_when_rt_k_empty(self):
|
||||
original_query = self.client.query
|
||||
|
||||
def query(api_name, params=None, fields=""):
|
||||
if api_name == "rt_k":
|
||||
return []
|
||||
return original_query(api_name, params, fields)
|
||||
|
||||
self.client.query = query
|
||||
self.client.try_market_quotes = lambda trade_date: list(FREE_QUOTES)
|
||||
TushareClient._realtime_reference_cache.clear()
|
||||
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
||||
self.assertEqual(dashboard["meta"]["quote_source"], "datahub")
|
||||
self.assertEqual(str(dashboard["meta"]["trade_date"]).replace("-", ""), "20260720")
|
||||
|
||||
def test_hub_failure_keeps_today_error(self):
|
||||
original_query = self.client.query
|
||||
|
||||
def query(api_name, params=None, fields=""):
|
||||
if api_name == "rt_k":
|
||||
raise TushareError("数据中枢行情暂不可用")
|
||||
return original_query(api_name, params, fields)
|
||||
|
||||
self.client.query = query
|
||||
TushareClient._realtime_reference_cache.clear()
|
||||
with self.assertRaises(TushareError) as ctx:
|
||||
self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
||||
self.assertIn("当天盘中实时行情不可用", str(ctx.exception))
|
||||
|
||||
def test_hub_failover_is_invisible_to_website(self):
|
||||
original_query = self.client.query
|
||||
|
||||
def query(api_name, params=None, fields=""):
|
||||
if api_name == "rt_k":
|
||||
raise TushareError("没有接口访问权限")
|
||||
return original_query(api_name, params, fields)
|
||||
|
||||
self.client.query = query
|
||||
self.client.try_market_quotes = lambda trade_date: list(FREE_QUOTES)
|
||||
TushareClient._realtime_reference_cache.clear()
|
||||
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
||||
self.assertEqual(dashboard["meta"]["quote_source"], "datahub")
|
||||
self.assertEqual(str(dashboard["meta"]["trade_date"]).replace("-", ""), "20260720")
|
||||
self.assertIn("数据中枢", dashboard["meta"]["notice"])
|
||||
self.assertEqual(dashboard["overview"]["amount_billion"], 6.0)
|
||||
|
||||
def test_normalize_eastmoney_quote_maps_units_and_exchange(self):
|
||||
quote = _normalize_eastmoney_quote(
|
||||
{
|
||||
"f12": "600000",
|
||||
"f13": 1,
|
||||
"f14": "浦发银行",
|
||||
"f2": 10.5,
|
||||
"f5": 12.0,
|
||||
"f6": 200000000,
|
||||
"f15": 10.8,
|
||||
"f16": 10.2,
|
||||
"f17": 10.3,
|
||||
"f18": 10.0,
|
||||
"f124": 1752986700,
|
||||
}
|
||||
)
|
||||
self.assertEqual(quote["ts_code"], "600000.SH")
|
||||
self.assertEqual(quote["vol"], 1200)
|
||||
self.assertEqual(quote["close"], 10.5)
|
||||
self.assertEqual(quote["pre_close"], 10.0)
|
||||
self.assertEqual(quote["source"], "eastmoney_clist")
|
||||
|
||||
def test_parse_tencent_stock_quote_keeps_today_and_units(self):
|
||||
line = (
|
||||
'v_sz000001="51~平安银行~000001~11.73~11.70~11.66~346232~0~0~0~0~0~0~0~0~0~0~0~0~0~0~0~0~0~0~0~0~0~0~'
|
||||
'~20260720100500~0.03~0.26~11.79~11.65~11.73/346232/406045563~346232~40605~0.18~5.24~~11.79~11.65~1.20~'
|
||||
'2276.29~2276.31~0.49~12.87~10.53~0.95~-3076~11.73~4.43~5.34~~~0.18~40604.5563~0.0000~0~";'
|
||||
)
|
||||
quote = _parse_tencent_stock_quote(line)
|
||||
self.assertEqual(quote["ts_code"], "000001.SZ")
|
||||
self.assertEqual(quote["quote_date"], "20260720")
|
||||
self.assertEqual(quote["close"], 11.73)
|
||||
self.assertEqual(quote["pre_close"], 11.70)
|
||||
self.assertEqual(quote["vol"], 34623200)
|
||||
self.assertEqual(quote["amount"], 406050000)
|
||||
self.assertEqual(quote["source"], "tencent_qt")
|
||||
|
||||
def test_datahub_market_quotes_used_before_legacy(self):
|
||||
calls = []
|
||||
|
||||
def try_market_quotes(trade_date):
|
||||
calls.append(trade_date)
|
||||
return list(FREE_QUOTES)
|
||||
|
||||
self.client.try_market_quotes = try_market_quotes
|
||||
self.client.realtime_aggregator = FakeFreeAggregator(fail=True)
|
||||
TushareClient._realtime_reference_cache.clear()
|
||||
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
||||
self.assertEqual(calls, ["20260720"])
|
||||
self.assertEqual(dashboard["meta"]["quote_source"], "datahub")
|
||||
self.assertEqual(dashboard["meta"]["source"], "datahub")
|
||||
self.assertEqual(dashboard["meta"]["quote_count"], 3)
|
||||
self.assertIn("数据中枢", dashboard["meta"]["notice"])
|
||||
|
||||
def test_gateway_dashboard_uses_bound_market_quotes(self) -> None:
|
||||
from backend.data import build_data_gateway
|
||||
from backend.data.datahub.bridge import DatahubAwareTushareClient
|
||||
from backend.data.datahub.client import DatahubResponse
|
||||
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
||||
|
||||
quotes = [
|
||||
{
|
||||
"ts_code": item["ts_code"],
|
||||
"name": item["name"],
|
||||
"pre_close": item["pre_close"],
|
||||
"open": item["open"],
|
||||
"high": item["high"],
|
||||
"low": item["low"],
|
||||
"close": item["close"],
|
||||
"vol": item["vol"],
|
||||
"amount": item["amount"],
|
||||
"quote_date": "20260720",
|
||||
}
|
||||
for item in FREE_QUOTES
|
||||
]
|
||||
extras = [
|
||||
{
|
||||
"ts_code": f"{index:06d}.SZ",
|
||||
"name": f"X{index}",
|
||||
"pre_close": 10.0,
|
||||
"open": 10.0,
|
||||
"high": 10.2,
|
||||
"low": 9.8,
|
||||
"close": 10.1,
|
||||
"vol": 100.0,
|
||||
"amount": 1000.0,
|
||||
"quote_date": "20260720",
|
||||
}
|
||||
for index in range(10, 230)
|
||||
]
|
||||
|
||||
class QuoteHub:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def query_api(self, api_name, params=None, fields=""):
|
||||
rows = FakeRealtimeClient("tok").query(api_name, params or {}, fields)
|
||||
return DatahubResponse(
|
||||
data=rows,
|
||||
meta={"source": "datahub", "stale": False, "staleness_seconds": 0, "row_shape": "tushare"},
|
||||
)
|
||||
|
||||
def quotes_latest(self, **params):
|
||||
return self.get("/v1/quotes/latest", params)
|
||||
|
||||
def get(self, path, params=None):
|
||||
self.calls.append(path)
|
||||
if path == "/v1/quotes/latest":
|
||||
return DatahubResponse(
|
||||
data=quotes + extras,
|
||||
meta={"stale": False, "staleness_seconds": 0, "source": "eastmoney_clist"},
|
||||
)
|
||||
raise AssertionError(path)
|
||||
|
||||
datasets = {name: DatasetFlags(name) for name in DATASETS}
|
||||
datasets["quotes"] = DatasetFlags("quotes", read=True, shadow=False)
|
||||
settings = DatahubSettings(base_url="http://127.0.0.1:9", token="tok", datasets=datasets)
|
||||
gateway = build_data_gateway({"tushare_token": "tok"}, datahub_settings=settings)
|
||||
gateway.datahub.client = QuoteHub()
|
||||
wrapped = gateway.tushare()
|
||||
wrapped.clock = lambda: datetime(2026, 7, 20, 10, 30, tzinfo=timezone(timedelta(hours=8)))
|
||||
wrapped.realtime_aggregator = FakeFreeAggregator(fail=True)
|
||||
DatahubAwareTushareClient._realtime_reference_cache.clear()
|
||||
dashboard = wrapped.dashboard("20260720")
|
||||
self.assertEqual(dashboard["meta"]["quote_source"], "datahub")
|
||||
self.assertIn("/v1/quotes/latest", gateway.datahub.client.calls)
|
||||
self.assertTrue(callable(getattr(type(wrapped), "try_market_quotes", None)))
|
||||
self.assertFalse(hasattr(wrapped, "_legacy"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -5,6 +5,10 @@ import unittest
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
from backend.data.providers.tushare_client import TushareError
|
||||
from backend.data.realtime import RealtimeAggregateError
|
||||
from backend.features.market.charts import ChartDataError
|
||||
from backend.features.market.service import TODAY_DAILY_UNAVAILABLE_NOTICE
|
||||
from server import DashboardService
|
||||
|
||||
|
||||
@@ -17,6 +21,10 @@ class DetailDatabaseStub:
|
||||
def list_notes(user_id, code=""):
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def get_snapshot(trade_date):
|
||||
return {}
|
||||
|
||||
|
||||
class RealtimeClientStub:
|
||||
quote_calls = 0
|
||||
@@ -45,6 +53,18 @@ class RealtimeClientStub:
|
||||
}
|
||||
|
||||
|
||||
class DataGatewayStub:
|
||||
def __init__(self, client):
|
||||
self.client = client
|
||||
|
||||
def tushare(self):
|
||||
return self.client
|
||||
|
||||
@staticmethod
|
||||
def datahub_status():
|
||||
return {"configured": True}
|
||||
|
||||
|
||||
class FixedMarketDatetime(datetime):
|
||||
fixed_now = datetime(2026, 7, 31, 10, 30).astimezone()
|
||||
|
||||
@@ -61,6 +81,122 @@ class FixedPreopenDatetime(datetime):
|
||||
return cls.fixed_now
|
||||
|
||||
|
||||
class FixedLunchDatetime(datetime):
|
||||
fixed_now = datetime(2026, 7, 31, 11, 45).astimezone()
|
||||
|
||||
@classmethod
|
||||
def now(cls, tz=None):
|
||||
return cls.fixed_now
|
||||
|
||||
|
||||
class FixedAfterCloseDatetime(datetime):
|
||||
fixed_now = datetime(2026, 7, 31, 15, 30).astimezone()
|
||||
|
||||
@classmethod
|
||||
def now(cls, tz=None):
|
||||
return cls.fixed_now
|
||||
|
||||
|
||||
class DeniedRealtimeClientStub:
|
||||
quote_calls = 0
|
||||
|
||||
def __init__(self, token):
|
||||
self.token = token
|
||||
|
||||
@staticmethod
|
||||
def resolve_trade_context(requested_date):
|
||||
return requested_date, requested_date
|
||||
|
||||
@classmethod
|
||||
def realtime_stock_quote(cls, ts_code, reference_date=""):
|
||||
cls.quote_calls += 1
|
||||
raise TushareError("没有接口访问权限")
|
||||
|
||||
|
||||
class FreeQuoteAggregator:
|
||||
def __init__(self, quote=None, fail=False):
|
||||
self.quote = quote
|
||||
self.fail = fail
|
||||
self.tencent_calls = 0
|
||||
self.eastmoney_calls = 0
|
||||
|
||||
def tencent_stock_quote(self, code, expected_date=""):
|
||||
self.tencent_calls += 1
|
||||
if self.fail:
|
||||
raise RealtimeAggregateError("tencent down")
|
||||
if self.quote and self.quote.get("source") == "eastmoney_stock":
|
||||
raise RealtimeAggregateError("tencent empty")
|
||||
if self.quote:
|
||||
return self.quote
|
||||
raise RealtimeAggregateError("tencent empty")
|
||||
|
||||
def eastmoney_stock_quote(self, code, expected_date=""):
|
||||
self.eastmoney_calls += 1
|
||||
if self.fail:
|
||||
raise RealtimeAggregateError("eastmoney down")
|
||||
if self.quote and self.quote.get("source") == "eastmoney_stock":
|
||||
return self.quote
|
||||
raise RealtimeAggregateError("eastmoney empty")
|
||||
|
||||
|
||||
class IntradayChartStub:
|
||||
def __init__(self, points, previous_close=10.0, trade_date="2026-07-31"):
|
||||
self.points = points
|
||||
self.previous_close = previous_close
|
||||
self.trade_date = trade_date
|
||||
|
||||
def stock_daily(self, code, end_date, limit=90):
|
||||
raise ChartDataError("iFinD daily unavailable")
|
||||
|
||||
def stock_intraday(self, code):
|
||||
return {
|
||||
"trade_date": self.trade_date,
|
||||
"previous_close": self.previous_close,
|
||||
"points": self.points,
|
||||
}
|
||||
|
||||
|
||||
def _history_payload(code="002141"):
|
||||
yesterday = (FixedMarketDatetime.fixed_now - timedelta(days=1)).strftime("%Y-%m-%d")
|
||||
return {
|
||||
"meta": {"trade_date": yesterday, "source": "tushare"},
|
||||
"stock": {"code": code, "name": "旧名称", "price": 10, "change": 7.1},
|
||||
"prices": [
|
||||
{
|
||||
"trade_date": yesterday,
|
||||
"open": 9.5,
|
||||
"high": 10.1,
|
||||
"low": 9.4,
|
||||
"close": 10,
|
||||
"change": 7.1,
|
||||
"volume": 100,
|
||||
"amount_billion": 1.1,
|
||||
}
|
||||
],
|
||||
"moneyflow": {},
|
||||
}
|
||||
|
||||
|
||||
def _free_quote(source="tencent_qt", **overrides):
|
||||
quote = {
|
||||
"ts_code": "002141.SZ",
|
||||
"name": "贤程科技",
|
||||
"pre_close": 10.0,
|
||||
"open": 10.2,
|
||||
"high": 10.8,
|
||||
"low": 10.1,
|
||||
"close": 10.6,
|
||||
"vol": 250000,
|
||||
"amount": 26_500_000,
|
||||
"quote_date": "20260731",
|
||||
"quote_time_epoch": int(datetime(2026, 7, 31, 10, 31).timestamp()),
|
||||
"source": source,
|
||||
"turnover_rate": 2.5,
|
||||
}
|
||||
quote.update(overrides)
|
||||
return quote
|
||||
|
||||
|
||||
class StockDetailRealtimeTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.service = DashboardService.__new__(DashboardService)
|
||||
@@ -68,7 +204,13 @@ class StockDetailRealtimeTests(unittest.TestCase):
|
||||
self.service.database = DetailDatabaseStub()
|
||||
self.service._request_context = threading.local()
|
||||
self.service._request_context.user_id = 1
|
||||
self.service.ifind = None
|
||||
self.service.realtime_aggregator = None
|
||||
self.service.chart_data = None
|
||||
self.service._market_client_override = RealtimeClientStub("datahub")
|
||||
self.service.data_gateway = DataGatewayStub(self.service._market_client_override)
|
||||
RealtimeClientStub.quote_calls = 0
|
||||
DeniedRealtimeClientStub.quote_calls = 0
|
||||
|
||||
def test_today_detail_merges_rt_quote_without_mutating_daily_cache(self):
|
||||
today = FixedMarketDatetime.fixed_now.strftime("%Y%m%d")
|
||||
@@ -90,9 +232,7 @@ class StockDetailRealtimeTests(unittest.TestCase):
|
||||
"moneyflow": {},
|
||||
}
|
||||
|
||||
with patch("backend.features.market.service.datetime", FixedMarketDatetime), patch(
|
||||
"backend.features.market.service.TushareClient", RealtimeClientStub
|
||||
):
|
||||
with patch("backend.features.market.service.datetime", FixedMarketDatetime):
|
||||
result = self.service._prepare_stock_detail(cached, "002141", today)
|
||||
|
||||
self.assertEqual(result["meta"]["trade_date"], FixedMarketDatetime.fixed_now.strftime("%Y-%m-%d"))
|
||||
@@ -112,9 +252,7 @@ class StockDetailRealtimeTests(unittest.TestCase):
|
||||
"stock": {"code": "002141", "price": 10, "change": 1.2},
|
||||
"prices": [{"trade_date": historical, "close": 10, "change": 1.2}],
|
||||
}
|
||||
with patch("backend.features.market.service.datetime", FixedMarketDatetime), patch(
|
||||
"backend.features.market.service.TushareClient", RealtimeClientStub
|
||||
):
|
||||
with patch("backend.features.market.service.datetime", FixedMarketDatetime):
|
||||
result = self.service._prepare_stock_detail(payload, "002141", historical)
|
||||
|
||||
self.assertEqual(result["stock"]["change"], 1.2)
|
||||
@@ -151,9 +289,7 @@ class StockDetailRealtimeTests(unittest.TestCase):
|
||||
},
|
||||
],
|
||||
}
|
||||
with patch("backend.features.market.service.datetime", FixedPreopenDatetime), patch(
|
||||
"backend.features.market.service.TushareClient", RealtimeClientStub
|
||||
):
|
||||
with patch("backend.features.market.service.datetime", FixedPreopenDatetime):
|
||||
result = self.service._prepare_stock_detail(payload, "002141", today)
|
||||
|
||||
self.assertEqual(result["meta"]["trade_date"], yesterday)
|
||||
@@ -162,6 +298,179 @@ class StockDetailRealtimeTests(unittest.TestCase):
|
||||
self.assertEqual(result["stock"]["change"], 1.2)
|
||||
self.assertEqual(RealtimeClientStub.quote_calls, 0)
|
||||
|
||||
def test_today_detail_falls_back_to_tencent_quote_when_rt_k_denied(self):
|
||||
today = FixedMarketDatetime.fixed_now.strftime("%Y%m%d")
|
||||
aggregator = FreeQuoteAggregator(_free_quote())
|
||||
self.service.realtime_aggregator = aggregator
|
||||
self.service._market_client_override = DeniedRealtimeClientStub("datahub")
|
||||
DeniedRealtimeClientStub.quote_calls = 0
|
||||
|
||||
with patch("backend.features.market.service.datetime", FixedMarketDatetime):
|
||||
result = self.service._prepare_stock_detail(_history_payload(), "002141", today)
|
||||
|
||||
bar = result["prices"][-1]
|
||||
self.assertEqual(bar["trade_date"], "2026-07-31")
|
||||
self.assertTrue(bar["realtime"])
|
||||
self.assertEqual(bar["open"], 10.2)
|
||||
self.assertEqual(bar["high"], 10.8)
|
||||
self.assertEqual(bar["low"], 10.1)
|
||||
self.assertEqual(bar["close"], 10.6)
|
||||
self.assertAlmostEqual(bar["change"], 6.0, places=4)
|
||||
self.assertEqual(bar["volume"], 2500)
|
||||
self.assertAlmostEqual(bar["amount_billion"], 0.265)
|
||||
self.assertEqual(len(result["prices"]), 2)
|
||||
self.assertEqual(result["meta"]["notice"], "")
|
||||
self.assertEqual(aggregator.tencent_calls, 1)
|
||||
self.assertEqual(DeniedRealtimeClientStub.quote_calls, 1)
|
||||
|
||||
def test_today_detail_falls_back_to_eastmoney_then_intraday(self):
|
||||
today = FixedMarketDatetime.fixed_now.strftime("%Y%m%d")
|
||||
aggregator = FreeQuoteAggregator(
|
||||
_free_quote(
|
||||
"eastmoney_stock",
|
||||
ts_code="600000.SH",
|
||||
name="浦发银行",
|
||||
net_mf_amount=12,
|
||||
),
|
||||
)
|
||||
self.service.realtime_aggregator = aggregator
|
||||
self.service._market_client_override = DeniedRealtimeClientStub("datahub")
|
||||
DeniedRealtimeClientStub.quote_calls = 0
|
||||
|
||||
with patch("backend.features.market.service.datetime", FixedMarketDatetime):
|
||||
result = self.service._prepare_stock_detail(_history_payload("600000"), "600000", today)
|
||||
|
||||
self.assertEqual(result["prices"][-1]["trade_date"], "2026-07-31")
|
||||
self.assertEqual(result["prices"][-1]["close"], 10.6)
|
||||
self.assertEqual(aggregator.tencent_calls, 1)
|
||||
self.assertEqual(aggregator.eastmoney_calls, 1)
|
||||
|
||||
aggregator = FreeQuoteAggregator(fail=True)
|
||||
self.service.realtime_aggregator = aggregator
|
||||
self.service.chart_data = IntradayChartStub(
|
||||
[
|
||||
{
|
||||
"date": "2026-07-31",
|
||||
"time": "09:30",
|
||||
"open": 10.1,
|
||||
"high": 10.2,
|
||||
"low": 10.0,
|
||||
"close": 10.15,
|
||||
"volume": 120,
|
||||
"amount": 121800,
|
||||
},
|
||||
{
|
||||
"date": "2026-07-31",
|
||||
"time": "10:05",
|
||||
"open": 10.15,
|
||||
"high": 10.5,
|
||||
"low": 9.9,
|
||||
"close": 10.4,
|
||||
"volume": 80,
|
||||
"amount": 83200,
|
||||
},
|
||||
]
|
||||
)
|
||||
with patch("backend.features.market.service.datetime", FixedMarketDatetime):
|
||||
result = self.service._prepare_stock_detail(_history_payload("300750"), "300750", today)
|
||||
|
||||
bar = result["prices"][-1]
|
||||
self.assertEqual(bar["trade_date"], "2026-07-31")
|
||||
self.assertEqual(bar["open"], 10.1)
|
||||
self.assertEqual(bar["high"], 10.5)
|
||||
self.assertEqual(bar["low"], 9.9)
|
||||
self.assertEqual(bar["close"], 10.4)
|
||||
self.assertAlmostEqual(bar["change"], 4.0, places=4)
|
||||
self.assertEqual(bar["volume"], 200)
|
||||
self.assertTrue(bar["realtime"])
|
||||
|
||||
def test_today_detail_keeps_history_when_free_sources_fail(self):
|
||||
today = FixedMarketDatetime.fixed_now.strftime("%Y%m%d")
|
||||
self.service.realtime_aggregator = FreeQuoteAggregator(fail=True)
|
||||
self.service.chart_data = IntradayChartStub([], trade_date="2026-07-30")
|
||||
self.service._market_client_override = DeniedRealtimeClientStub("datahub")
|
||||
DeniedRealtimeClientStub.quote_calls = 0
|
||||
|
||||
with patch("backend.features.market.service.datetime", FixedMarketDatetime):
|
||||
result = self.service._prepare_stock_detail(_history_payload(), "002141", today)
|
||||
|
||||
self.assertEqual(result["prices"][-1]["trade_date"], "2026-07-30")
|
||||
self.assertFalse(result["meta"].get("realtime", False))
|
||||
self.assertEqual(result["meta"]["notice"], TODAY_DAILY_UNAVAILABLE_NOTICE)
|
||||
self.assertEqual(len(result["prices"]), 1)
|
||||
|
||||
def test_lunch_keeps_morning_realtime_bar(self):
|
||||
today = FixedLunchDatetime.fixed_now.strftime("%Y%m%d")
|
||||
self.service.realtime_aggregator = FreeQuoteAggregator(
|
||||
_free_quote(quote_time_epoch=int(datetime(2026, 7, 31, 11, 30).timestamp()))
|
||||
)
|
||||
self.service._market_client_override = DeniedRealtimeClientStub("datahub")
|
||||
DeniedRealtimeClientStub.quote_calls = 0
|
||||
with patch("backend.features.market.service.datetime", FixedLunchDatetime):
|
||||
result = self.service._prepare_stock_detail(_history_payload(), "002141", today)
|
||||
|
||||
self.assertEqual(result["prices"][-1]["trade_date"], "2026-07-31")
|
||||
self.assertTrue(result["meta"]["realtime"])
|
||||
|
||||
def test_after_close_keeps_forming_bar_until_official_ready(self):
|
||||
today = FixedAfterCloseDatetime.fixed_now.strftime("%Y%m%d")
|
||||
self.service.realtime_aggregator = FreeQuoteAggregator(_free_quote())
|
||||
self.service._market_client_override = DeniedRealtimeClientStub("datahub")
|
||||
DeniedRealtimeClientStub.quote_calls = 0
|
||||
with patch("backend.features.market.service.datetime", FixedAfterCloseDatetime):
|
||||
forming = self.service._prepare_stock_detail(_history_payload(), "002141", today)
|
||||
self.assertEqual(forming["prices"][-1]["trade_date"], "2026-07-31")
|
||||
self.assertTrue(forming["prices"][-1]["realtime"])
|
||||
|
||||
official = _history_payload()
|
||||
official["prices"].append(
|
||||
{
|
||||
"trade_date": "2026-07-31",
|
||||
"open": 10.15,
|
||||
"high": 10.9,
|
||||
"low": 10.05,
|
||||
"close": 10.7,
|
||||
"change": 7.0,
|
||||
"volume": 1800,
|
||||
"amount_billion": 0.3,
|
||||
}
|
||||
)
|
||||
RealtimeClientStub.quote_calls = 0
|
||||
self.service._market_client_override = RealtimeClientStub("datahub")
|
||||
with patch("backend.features.market.service.datetime", FixedAfterCloseDatetime):
|
||||
replaced = self.service._prepare_stock_detail(official, "002141", today)
|
||||
|
||||
self.assertEqual(replaced["prices"][-1]["close"], 10.7)
|
||||
self.assertFalse(replaced["prices"][-1].get("realtime", False))
|
||||
self.assertEqual(len(replaced["prices"]), 2)
|
||||
self.assertEqual(RealtimeClientStub.quote_calls, 0)
|
||||
|
||||
def test_same_date_bar_is_replaced_not_duplicated(self):
|
||||
today = FixedMarketDatetime.fixed_now.strftime("%Y%m%d")
|
||||
payload = _history_payload()
|
||||
payload["prices"].append(
|
||||
{
|
||||
"trade_date": "2026-07-31",
|
||||
"open": 10.0,
|
||||
"high": 10.1,
|
||||
"low": 9.9,
|
||||
"close": 10.05,
|
||||
"change": 0.5,
|
||||
"volume": 10,
|
||||
"amount_billion": 0.01,
|
||||
"realtime": True,
|
||||
}
|
||||
)
|
||||
self.service.realtime_aggregator = FreeQuoteAggregator(_free_quote())
|
||||
self.service._market_client_override = DeniedRealtimeClientStub("datahub")
|
||||
DeniedRealtimeClientStub.quote_calls = 0
|
||||
with patch("backend.features.market.service.datetime", FixedMarketDatetime):
|
||||
result = self.service._prepare_stock_detail(payload, "002141", today)
|
||||
|
||||
self.assertEqual(len(result["prices"]), 2)
|
||||
self.assertEqual(result["prices"][-1]["close"], 10.6)
|
||||
self.assertEqual(result["prices"][-1]["trade_date"], "2026-07-31")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -218,11 +218,10 @@ def build() -> dict[str, Any]:
|
||||
),
|
||||
"external_data_adapters": [
|
||||
{"provider": "tushare", "path": "backend/data/providers/tushare_client.py", "runtime_role": "stable client facade for primary deterministic market data"},
|
||||
{"provider": "datahub", "path": "backend/data/datahub/client.py", "runtime_role": "optional official EOD read path behind per-dataset flags"},
|
||||
{"provider": "ifind", "path": "backend/data/providers/ifind_client.py", "runtime_role": "realtime, charts, snapshots, enrichment"},
|
||||
{"provider": "eastmoney", "path": "backend/features/market/charts.py", "runtime_role": "display chart fallback"},
|
||||
{"provider": "eastmoney", "path": "backend/data/realtime.py", "runtime_role": "isolated realtime observation"},
|
||||
{"provider": "tencent", "path": "backend/data/realtime.py", "runtime_role": "index observation fallback"},
|
||||
{"provider": "datahub", "path": "backend/data/datahub/client.py", "runtime_role": "website-only read path; official EOD, live quotes, and licensed iFinD"},
|
||||
{"provider": "ifind", "path": "xiaobai-datahub/datahub/adapters/ifind.py", "runtime_role": "licensed iFinD source inside the data hub"},
|
||||
{"provider": "eastmoney", "path": "xiaobai-datahub/datahub/adapters/eastmoney.py", "runtime_role": "free realtime quotes and shenwan inside the data hub"},
|
||||
{"provider": "tencent", "path": "xiaobai-datahub/datahub/adapters/tencent.py", "runtime_role": "free index and stock quotes inside the data hub"},
|
||||
],
|
||||
"provider_domains": [
|
||||
{"provider": "tushare", "path": "backend/data/providers/tushare_transport.py", "responsibility": "HTTP transport and provider errors"},
|
||||
@@ -240,9 +239,9 @@ def build() -> dict[str, Any]:
|
||||
{"client": "DatahubClient", "owner": "backend/data/gateway.py"},
|
||||
{"client": "DatahubBridge", "owner": "backend/data/gateway.py"},
|
||||
{"client": "DatahubAwareTushareClient", "owner": "backend/data/gateway.py"},
|
||||
{"client": "IfindHttpClient", "owner": "backend/data/gateway.py"},
|
||||
{"client": "HubIfindProxy", "owner": "backend/data/gateway.py"},
|
||||
{"client": "HubRealtimeProxy", "owner": "backend/data/gateway.py"},
|
||||
{"client": "MarketChartClient", "owner": "backend/data/gateway.py"},
|
||||
{"client": "WebRealtimeAggregator", "owner": "backend/data/gateway.py"},
|
||||
],
|
||||
"heaven_service_owners": {
|
||||
"facade": "backend/features/heaven/service.py",
|
||||
|
||||
@@ -10,4 +10,8 @@ DATAHUB_ADMIN_PASSWORD=
|
||||
# Tushare Pro token. Stored encrypted after first launch; never returned by API or admin pages.
|
||||
TUSHARE_TOKEN=
|
||||
|
||||
# Optional licensed iFinD tokens. Used only inside the hub; the website never dials 51ifind.com.
|
||||
IFIND_REFRESH_TOKEN=
|
||||
IFIND_ACCESS_TOKEN=
|
||||
|
||||
TZ=Asia/Shanghai
|
||||
|
||||
@@ -6,11 +6,12 @@
|
||||
## 做什么
|
||||
|
||||
- SQLite WAL `datahub.db`,容器名 `xiaobai-datahub`,端口 `8766`
|
||||
- Tushare 盘后正式数据:交易日历、股票主档、daily、daily_basic、adj_factor、index_daily、moneyflow、stk_auction
|
||||
- Tushare 盘后正式数据:交易日历、股票主档、daily、daily_basic、adj_factor、index_daily、moneyflow、stk_auction、limit_list_d、ths_hot/dc_hot、hm_detail、ths_daily/dc_index/sw_daily
|
||||
- 盘中观察(provisional):东财/腾讯指数报价、个股最新价、全市场快照、分时点(`/v1/quotes/latest` 不传 codes 即全市场,`/v1/indexes/quotes` `/v1/intraday/points`);永不写入 eod_* 正式表
|
||||
- 暂存 → 校验 → 整批原子发布 → 可回滚
|
||||
- `/v1` 稳定接口(`X-Datahub-Token`)
|
||||
- `/admin/` 最小管理后台(总览 / 数据源 / 调度 / 发布 / 数据集 / 审计)
|
||||
- 东财/腾讯/同花顺/选股宝/AKShare/iFinD 适配器位已预留,本阶段不拉实时源
|
||||
- 同花顺/选股宝/AKShare/iFinD 适配器位仍预留;东财/腾讯已接入盘中观察
|
||||
|
||||
## 单位口径(相对现站)
|
||||
|
||||
@@ -63,6 +64,78 @@ python -m unittest discover -s tests -v
|
||||
|
||||
不调用真实 Tushare;用内存/临时库和假适配器。
|
||||
|
||||
## 历史回补
|
||||
|
||||
交易日历默认从 `20160101` 拉到今天后 30 天;盘前 `precheck` 与手动回补都走同一 UPSERT,可重复执行。
|
||||
|
||||
网站实际使用的指数(上证、深成、创业板、沪深300)按交易日增量发布,默认覆盖 260 个交易日(大于现有 90 天窗口,并覆盖智能选股基准回看)。已发布日期默认跳过。
|
||||
|
||||
```bash
|
||||
cd xiaobai-datahub
|
||||
python -m datahub history-backfill
|
||||
# 可选:--calendar-start 20160101 --index-days 260 --force
|
||||
```
|
||||
|
||||
管理后台也可手动跑 `history_backfill` 任务,或 `POST /admin/api/backfill` 且 `dataset=history`、确认词 `history:full`。
|
||||
|
||||
区间接口在 `meta.coverage` / `meta.incomplete` 标明覆盖是否完整;网站只读接入把不完整区间视为不可用并回旧链路。个股日 K 的 90 天区间查询依赖已核实,本阶段不回补全市场历史。
|
||||
|
||||
## 估值字段级质量门
|
||||
|
||||
`hub-quality.config.json` 的 `field_gates` 按数据集配置关键字段:非空率下限(支持按字段覆盖,如 `dv_ttm` 合法高空值)、非有限值比例上限、以及相对上一已发布批次的非空率塌陷保护。字段大面积为空的批次会被拒绝发布、保留上一份正常正式数据,失败原因逐字段写入 `batches.error` / `quality_json`。被拒后数据集仍视为缺失,盘后自动重试(HEL-435 机制)会继续尝试直到成功或截止。配置对任意数据集生效,不写死单日或单字段。
|
||||
|
||||
## 整批原子发布(release group)
|
||||
|
||||
盘后发布/重发(eod_a、eod_retry、`eod-refresh`、跨数据集重发)不再逐数据集各自切换,而是走整批原子可见机制:
|
||||
|
||||
- 一致性边界:日 K、估值、资金流、竞价同属 A 组整批;指数日 K 为 B 组;当日股票主档快照随 A 组一同切换(主档 `stock_master` 的 UPSERT 与快照发布同一事务,不会出现主档先行/滞后)。
|
||||
- 流程:组内全部成员先在暂存表完成拉取、字段质量门、覆盖检查和跨数据集交叉校验(`cross_gates` 配置 ts_code 覆盖重叠率下限),全部达标后才在**一个 SQLite 事务**里复制正式表并翻转全部 `publications` 指针。
|
||||
- 任一成员失败(拉取失败、质量门拒绝、交叉校验不过、切换事务中断)→ 整批不切换,对外继续提供上一份完整正式版本,失败原因写入 `batches.error` 与 `audit_log`(`action=release-group`),等待晚间自动重试。
|
||||
- 读取侧任何时刻只会看到"旧完整版本"或"新完整版本":发布指针在单事务内统一翻转,容器重启/事务中断自动回滚,不暴露字段残缺或跨数据集混合版本。
|
||||
- 幂等:仅当一致性边界内全部成员都已发布时才整组跳过;边界内任有缺失则整组重暂存后统一切换,避免旧批次与新批次混在同一次重发中。重复执行、并发重试不会在完整边界已就绪时生成重复批次(调度器另有 EOD 互斥锁)。
|
||||
|
||||
## 股票主档每日刷新与发布
|
||||
|
||||
交易日 20:00 与 23:10(`stocks_refresh_times` 可配)自动刷新股票主档并发布版本化快照(`eod_stocks` + `publications.dataset='stocks'`),覆盖当日新上市、证券简称变化和上市首日 N/C 前缀摘除;无变化则跳过,重复执行幂等。`/v1/stocks` 从最新已发布快照提供数据并带 `batch_id` / `published_at`;`/v1/datasets/status` 同步展示 stocks 状态。
|
||||
|
||||
```bash
|
||||
cd xiaobai-datahub
|
||||
python -m datahub stocks-refresh # 手动触发;--force 无变化也重发
|
||||
```
|
||||
|
||||
## 资金流历史回补
|
||||
|
||||
网站会沿真实调用链查最近若干交易日的 moneyflow(个股详情任意日期点查 + 智能选股最近 5 个交易日),默认回补最近 60 个交易日(`moneyflow_history_trading_days` 可配,已发布日期自动跳过)。点查未覆盖的历史日期返回 `DATASET_NOT_PUBLISHED` 并附 `available_from` / `available_to`(低于下界时 `reason=history_not_backfilled`),网站据此明确回退旧链路,不会静默拿到半截数据。
|
||||
|
||||
```bash
|
||||
cd xiaobai-datahub
|
||||
python -m datahub moneyflow-backfill # --trading-days 60 --end-date --force 可选
|
||||
```
|
||||
|
||||
## 盘后补跑与强制重发
|
||||
|
||||
```bash
|
||||
cd xiaobai-datahub
|
||||
python -m datahub eod-refresh --trade-date 20260904 # 补不完整的 A/B 边界
|
||||
python -m datahub eod-refresh --trade-date 20260904 --force --dataset valuation
|
||||
# --force 按一致性边界整组重发:valuation/daily/moneyflow/auction/stocks → A 组;
|
||||
# index_daily → B 组。不可再单独切换某一个正式数据集。
|
||||
```
|
||||
|
||||
管理后台「补数」对盘后正式数据集同样走 `force_republish_boundary`,不会绕过 A/B 整批边界。
|
||||
|
||||
## 估值发布后复核与自动追补
|
||||
|
||||
Tushare `daily_basic` 会在盘后继续改当日字段。HEL-423 在 2026-09-07 观察到:中枢 17:10 发布 `003021.SZ turnover_rate=1.3565`,21:05 上游/旧链路已是 `1.3572`;其余 7 类观察对象当日一致。日 K、资金流、竞价、指数没有同类晚间修订证据,股票主档已有 20:00/23:10 刷新,因此默认只复核估值,不盲目全量重拉。
|
||||
|
||||
窗口(可配):交易日 **20:00–23:20**,每 30 分钟一次轻量比对(对齐网站 21:00 / 23:30 观察)。只拉取 `daily_basic`,按网站真实请求字段精确比较,无误差豁免。
|
||||
|
||||
- 无变化:不产生新批次,状态「已追平」。
|
||||
- 发现修订:重新走字段质量门、覆盖检查和 A 组整批原子发布;读者全程只能看到上一完整版本或新完整版本。
|
||||
- 上游空 / 接口失败 / 不完整 / 质量门拒绝:保留上一完整版本,状态「复核失败」。
|
||||
- 23:20 截止后停止当晚复核;下一自然日盘前对上一交易日再做一次安全追赶。
|
||||
- 与 `eod_a` / `eod_retry` 共用互斥锁;容器重启会在窗口内立即补一次。
|
||||
|
||||
## 备份
|
||||
|
||||
每日 00:40 任务把 `datahub.db` 备份到 `data/backups/`(保留 14 份)。也可手动:
|
||||
|
||||
@@ -104,11 +104,42 @@ async function render() {
|
||||
if (state.page === "overview") {
|
||||
const data = await api("/admin/api/overview");
|
||||
$("phase").textContent = data.session_phase;
|
||||
const eod = data.eod_status || {};
|
||||
const rev = data.revision_status || {};
|
||||
const eodLabels = {
|
||||
pending_first_attempt: "等待首次尝试",
|
||||
waiting_upstream: "等待上游",
|
||||
done: "已成功",
|
||||
cutoff_failed: "已截止失败",
|
||||
closed_day: "休市",
|
||||
};
|
||||
const revLabels = {
|
||||
waiting_review: "等待复核",
|
||||
review_failed: "复核失败",
|
||||
aligned: "已追平",
|
||||
cutoff: "已截止",
|
||||
pending_publish: "待发布",
|
||||
closed_day: "休市",
|
||||
};
|
||||
const eodExtra = [];
|
||||
if (eod.state === "waiting_upstream") {
|
||||
eodExtra.push(`已试 ${eod.attempts} 次`);
|
||||
if (eod.next_retry_at) eodExtra.push(`下次重试 ${esc(String(eod.next_retry_at).replace("T", " ").slice(11, 16))}`);
|
||||
if (eod.missing_datasets && eod.missing_datasets.length) eodExtra.push(`缺 ${esc(eod.missing_datasets.join(","))}`);
|
||||
}
|
||||
if (eod.state === "cutoff_failed" && eod.missing_datasets) {
|
||||
eodExtra.push(`缺 ${esc(eod.missing_datasets.join(","))}`);
|
||||
}
|
||||
const revExtra = [];
|
||||
if (rev.detail) revExtra.push(esc(String(rev.detail)));
|
||||
if (rev.window) revExtra.push(esc(String(rev.window)));
|
||||
page.innerHTML = `
|
||||
<div class="cards">
|
||||
<div class="card"><div class="muted">交易日</div><strong>${esc(data.trade_date)}</strong></div>
|
||||
<div class="card"><div class="muted">阶段</div><strong>${esc(data.session_phase)}</strong></div>
|
||||
<div class="card"><div class="muted">今日发布</div><strong>${data.publications.length}</strong></div>
|
||||
<div class="card"><div class="muted">盘后补跑</div><strong>${esc(eodLabels[eod.state] || eod.state || "-")}</strong><div class="muted">${eodExtra.join(" · ")}</div></div>
|
||||
<div class="card"><div class="muted">估值复核</div><strong>${esc(revLabels[rev.state] || rev.state || "-")}</strong><div class="muted">${revExtra.join(" · ")}</div></div>
|
||||
<div class="card"><div class="muted">异常批次</div><strong class="${data.anomalies.length ? "fail" : "ok"}">${data.anomalies.length}</strong></div>
|
||||
</div>
|
||||
<h2>最近调用</h2>
|
||||
@@ -251,7 +282,7 @@ function renderRelease(data) {
|
||||
|
||||
async function dangerous(kind, dataset) {
|
||||
const date = ($("rel-date") && $("rel-date").value) || "";
|
||||
const ds = dataset || prompt("数据集(daily / valuation / moneyflow / auction / index_daily / reference)", "daily");
|
||||
const ds = dataset || prompt("数据集(daily/valuation/moneyflow/auction/stocks→A组整批;index_daily→B组;或 reference)", "daily");
|
||||
if (!ds) return;
|
||||
const password = prompt("二次确认:输入管理密码");
|
||||
if (!password) return;
|
||||
|
||||
@@ -14,6 +14,8 @@ services:
|
||||
DATAHUB_TOKEN: "${DATAHUB_TOKEN:?DATAHUB_TOKEN must be set}"
|
||||
DATAHUB_ADMIN_PASSWORD: "${DATAHUB_ADMIN_PASSWORD:?DATAHUB_ADMIN_PASSWORD must be set}"
|
||||
TUSHARE_TOKEN: "${TUSHARE_TOKEN:-}"
|
||||
IFIND_REFRESH_TOKEN: "${IFIND_REFRESH_TOKEN:-}"
|
||||
IFIND_ACCESS_TOKEN: "${IFIND_ACCESS_TOKEN:-}"
|
||||
DATAHUB_DB_PATH: /app/data/datahub.db
|
||||
DATAHUB_BACKUP_DIR: /app/data/backups
|
||||
TZ: Asia/Shanghai
|
||||
|
||||
@@ -11,5 +11,54 @@
|
||||
"publication_generations": 3,
|
||||
"tushare_rate_per_minute": 300,
|
||||
"list_limit_default": 5000,
|
||||
"list_limit_max": 5000
|
||||
"list_limit_max": 5000,
|
||||
"calendar_start": "20160101",
|
||||
"index_history_trading_days": 260,
|
||||
"daily_history_trading_days": 250,
|
||||
"eod_retry_start": "15:15",
|
||||
"eod_retry_interval_minutes": 30,
|
||||
"eod_retry_cutoff": "23:30",
|
||||
"revision_review_datasets": ["valuation"],
|
||||
"revision_review_start": "20:00",
|
||||
"revision_review_interval_minutes": 30,
|
||||
"revision_review_cutoff": "23:20",
|
||||
"moneyflow_history_trading_days": 60,
|
||||
"stocks_refresh_times": [
|
||||
"20:00",
|
||||
"23:10"
|
||||
],
|
||||
"realtime_warmup_interval_seconds": 120,
|
||||
"cross_gates": [
|
||||
{
|
||||
"left": "daily",
|
||||
"right": "valuation",
|
||||
"min_key_overlap": 0.98
|
||||
},
|
||||
{
|
||||
"left": "daily",
|
||||
"right": "moneyflow",
|
||||
"min_key_overlap": 0.98
|
||||
}
|
||||
],
|
||||
"field_gates": {
|
||||
"valuation": {
|
||||
"fields": [
|
||||
"turnover_rate",
|
||||
"volume_ratio",
|
||||
"total_mv",
|
||||
"circ_mv",
|
||||
"pe_ttm",
|
||||
"pb",
|
||||
"ps_ttm",
|
||||
"dv_ttm"
|
||||
],
|
||||
"min_nonnull_rate": 0.9,
|
||||
"min_nonnull_rate_by_field": {
|
||||
"pe_ttm": 0.5,
|
||||
"dv_ttm": 0.3
|
||||
},
|
||||
"max_nonnull_drop_vs_prev": 0.15,
|
||||
"max_nonfinite_rate": 0.01
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
from datahub.cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,13 +1,13 @@
|
||||
from datahub.adapters.akshare import ADAPTER as akshare
|
||||
from datahub.adapters.eastmoney import ADAPTER as eastmoney
|
||||
from datahub.adapters.eastmoney import EastmoneyAdapter
|
||||
from datahub.adapters.ifind import ADAPTER as ifind
|
||||
from datahub.adapters.tencent import ADAPTER as tencent
|
||||
from datahub.adapters.tencent import TencentAdapter
|
||||
from datahub.adapters.ths import ADAPTER as ths
|
||||
from datahub.adapters.xgb import ADAPTER as xgb
|
||||
|
||||
RESERVED = {
|
||||
"eastmoney": eastmoney,
|
||||
"tencent": tencent,
|
||||
"eastmoney": EastmoneyAdapter(),
|
||||
"tencent": TencentAdapter(),
|
||||
"ths": ths,
|
||||
"xgb": xgb,
|
||||
"akshare": akshare,
|
||||
|
||||
@@ -1,3 +1,524 @@
|
||||
from datahub.adapters.base import ReservedAdapter
|
||||
from __future__ import annotations
|
||||
|
||||
ADAPTER = ReservedAdapter("eastmoney")
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from datahub.adapters.base import AdapterError, MarketAdapter
|
||||
from datahub.numbers import finite_number, round4
|
||||
|
||||
EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get"
|
||||
EASTMONEY_CLIST_URL = "https://push2.eastmoney.com/api/qt/clist/get"
|
||||
EASTMONEY_A_SHARE_BOARDS = (
|
||||
"m:0+t:6",
|
||||
"m:0+t:80",
|
||||
"m:1+t:2",
|
||||
"m:1+t:23",
|
||||
"m:0+t:81",
|
||||
)
|
||||
EASTMONEY_QUOTE_FIELDS = "f12,f13,f14,f2,f3,f4,f5,f6,f15,f16,f17,f18,f8,f124"
|
||||
EASTMONEY_MARKET_PAGE_SIZE = 100
|
||||
TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get"
|
||||
HIS_TRENDS_URL = "https://push2his.eastmoney.com/api/qt/stock/trends2/get"
|
||||
BROWSER_UA = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
|
||||
)
|
||||
INDEX_SECIDS = {
|
||||
"000001.SH": "1.000001",
|
||||
"399001.SZ": "0.399001",
|
||||
"399006.SZ": "0.399006",
|
||||
}
|
||||
|
||||
|
||||
class EastmoneyAdapter(MarketAdapter):
|
||||
name = "eastmoney"
|
||||
|
||||
def __init__(self, timeout: int = 8) -> None:
|
||||
self.timeout = timeout
|
||||
|
||||
def probe(self) -> dict[str, Any]:
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
rows = self.fetch_indices()
|
||||
state = "ok" if len(rows) == 3 else "empty"
|
||||
except AdapterError as exc:
|
||||
return {
|
||||
"provider": self.name,
|
||||
"configured": True,
|
||||
"state": "error",
|
||||
"message": str(exc),
|
||||
"latency_ms": round((time.perf_counter() - started) * 1000),
|
||||
}
|
||||
return {
|
||||
"provider": self.name,
|
||||
"configured": True,
|
||||
"state": state,
|
||||
"latency_ms": round((time.perf_counter() - started) * 1000),
|
||||
}
|
||||
|
||||
def fetch(self, dataset: str, params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if dataset in {"indexes_quotes", "index_quotes"}:
|
||||
return self.fetch_indices()
|
||||
if dataset in {"quotes", "quotes_latest"}:
|
||||
codes = params.get("codes") or []
|
||||
if isinstance(codes, str):
|
||||
codes = [item.strip() for item in codes.split(",") if item.strip()]
|
||||
if codes:
|
||||
return self.fetch_quotes(list(codes))
|
||||
return self.fetch_market_quotes()
|
||||
if dataset in {"quotes_market", "market_quotes"}:
|
||||
return self.fetch_market_quotes()
|
||||
raise AdapterError(f"{self.name} unsupported dataset: {dataset}")
|
||||
|
||||
def normalize(self, dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return list(rows)
|
||||
|
||||
def fetch_indices(self) -> list[dict[str, Any]]:
|
||||
payload = self._get_json(
|
||||
EASTMONEY_INDEX_URL,
|
||||
{
|
||||
"secids": "1.000001,0.399001,0.399006",
|
||||
"fltt": "2",
|
||||
"invt": "2",
|
||||
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124",
|
||||
},
|
||||
referer="https://quote.eastmoney.com/",
|
||||
)
|
||||
rows = list((payload.get("data") or {}).get("diff") or [])
|
||||
result = []
|
||||
for row in rows:
|
||||
code = str(row.get("f12") or "")
|
||||
if code not in {"000001", "399001", "399006"}:
|
||||
continue
|
||||
epoch = int(finite_number(row.get("f124")) or 0)
|
||||
ts_code = f"{code}.SH" if code.startswith("0") and code == "000001" else f"{code}.SZ"
|
||||
if code == "000001":
|
||||
ts_code = "000001.SH"
|
||||
result.append(
|
||||
{
|
||||
"ts_code": ts_code,
|
||||
"code": code,
|
||||
"name": row.get("f14") or code,
|
||||
"price": round4(finite_number(row.get("f2"))),
|
||||
"pct_chg": round4(finite_number(row.get("f3"))),
|
||||
"change_amount": round4(finite_number(row.get("f4"))),
|
||||
"open": round4(finite_number(row.get("f17"))),
|
||||
"high": round4(finite_number(row.get("f15"))),
|
||||
"low": round4(finite_number(row.get("f16"))),
|
||||
"previous_close": round4(finite_number(row.get("f18"))),
|
||||
"amount": round4(finite_number(row.get("f6"))),
|
||||
"quote_time_epoch": epoch,
|
||||
"quote_time": (
|
||||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||
if epoch
|
||||
else ""
|
||||
),
|
||||
"source": "eastmoney_push2",
|
||||
}
|
||||
)
|
||||
if len(result) != 3:
|
||||
raise AdapterError(f"Eastmoney returned {len(result)}/3 indices")
|
||||
return result
|
||||
|
||||
def fetch_quotes(self, codes: list[str]) -> list[dict[str, Any]]:
|
||||
# Eastmoney ulist.np accepts ~60 secids per request; page remaining codes.
|
||||
secids = []
|
||||
suffixes: dict[str, str] = {}
|
||||
for code in codes:
|
||||
ts = str(code or "").upper()
|
||||
symbol = ts.split(".")[0]
|
||||
if ts.endswith(".BJ") or symbol.startswith(("4", "8", "92")):
|
||||
secids.append(f"0.{symbol}")
|
||||
suffixes[symbol] = "BJ"
|
||||
elif ts.endswith(".SH") or symbol.startswith(("5", "6", "9")):
|
||||
secids.append(f"1.{symbol}")
|
||||
suffixes[symbol] = "SH"
|
||||
else:
|
||||
secids.append(f"0.{symbol}")
|
||||
suffixes[symbol] = "SZ"
|
||||
if not secids:
|
||||
return []
|
||||
result: list[dict[str, Any]] = []
|
||||
for index in range(0, len(secids), 60):
|
||||
payload = self._get_json(
|
||||
EASTMONEY_INDEX_URL,
|
||||
{
|
||||
"secids": ",".join(secids[index:index + 60]),
|
||||
"fltt": "2",
|
||||
"invt": "2",
|
||||
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f5,f6,f8,f124",
|
||||
},
|
||||
referer="https://quote.eastmoney.com/",
|
||||
)
|
||||
rows = list((payload.get("data") or {}).get("diff") or [])
|
||||
for row in rows:
|
||||
symbol = str(row.get("f12") or "")
|
||||
if not symbol:
|
||||
continue
|
||||
ts_code = f"{symbol}.{suffixes.get(symbol, 'SZ')}"
|
||||
epoch = int(finite_number(row.get("f124")) or 0)
|
||||
close = round4(finite_number(row.get("f2")))
|
||||
previous = round4(finite_number(row.get("f18")))
|
||||
quote_date = (
|
||||
datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d")
|
||||
if epoch
|
||||
else ""
|
||||
)
|
||||
result.append(
|
||||
{
|
||||
"ts_code": ts_code,
|
||||
"name": row.get("f14") or symbol,
|
||||
"price": close,
|
||||
"close": close,
|
||||
"pct_chg": round4(finite_number(row.get("f3"))),
|
||||
"change_amount": round4(finite_number(row.get("f4"))),
|
||||
"open": round4(finite_number(row.get("f17"))),
|
||||
"high": round4(finite_number(row.get("f15"))),
|
||||
"low": round4(finite_number(row.get("f16"))),
|
||||
"pre_close": previous,
|
||||
"previous_close": previous,
|
||||
"volume": round4(finite_number(row.get("f5"))),
|
||||
"vol": round4(finite_number(row.get("f5")) * 100),
|
||||
"amount": round4(finite_number(row.get("f6"))),
|
||||
"turnover_rate": round4(finite_number(row.get("f8"))),
|
||||
"quote_date": quote_date,
|
||||
"quote_time_epoch": epoch,
|
||||
"quote_time": (
|
||||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||
if epoch
|
||||
else ""
|
||||
),
|
||||
"source": "eastmoney_push2",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
def fetch_market_quotes(self) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
board_errors: list[str] = []
|
||||
for board in EASTMONEY_A_SHARE_BOARDS:
|
||||
try:
|
||||
rows.extend(self._board_quotes(board))
|
||||
except Exception as exc:
|
||||
board_errors.append(f"{board}:{exc}")
|
||||
quotes: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for row in rows:
|
||||
quote = _normalize_market_quote(row)
|
||||
ts_code = str((quote or {}).get("ts_code") or "")
|
||||
if not quote or ts_code in seen:
|
||||
continue
|
||||
seen.add(ts_code)
|
||||
quotes.append(quote)
|
||||
if len(quotes) < 200:
|
||||
detail = f";{'; '.join(board_errors)}" if board_errors else ""
|
||||
raise AdapterError(f"Eastmoney market snapshot too small: {len(quotes)}{detail}")
|
||||
return quotes
|
||||
|
||||
def _board_quotes(self, board: str) -> list[dict[str, Any]]:
|
||||
first = self._market_page(board, 1)
|
||||
data = first.get("data") or {}
|
||||
rows = list(data.get("diff") or [])
|
||||
total = int(finite_number(data.get("total")) or 0)
|
||||
page_count = 1
|
||||
if total > 0:
|
||||
page_count = max(1, (total + EASTMONEY_MARKET_PAGE_SIZE - 1) // EASTMONEY_MARKET_PAGE_SIZE)
|
||||
for page in range(2, min(page_count, 40) + 1):
|
||||
payload = self._market_page(board, page)
|
||||
rows.extend(list((payload.get("data") or {}).get("diff") or []))
|
||||
return rows
|
||||
|
||||
def _market_page(self, board: str, page: int) -> dict[str, Any]:
|
||||
return self._get_json(
|
||||
EASTMONEY_CLIST_URL,
|
||||
{
|
||||
"pn": str(page),
|
||||
"pz": str(EASTMONEY_MARKET_PAGE_SIZE),
|
||||
"po": "1",
|
||||
"np": "1",
|
||||
"fltt": "2",
|
||||
"invt": "2",
|
||||
"fid": "f12",
|
||||
"fs": board,
|
||||
"fields": EASTMONEY_QUOTE_FIELDS,
|
||||
},
|
||||
referer="https://quote.eastmoney.com/center/gridlist.html",
|
||||
)
|
||||
|
||||
def fetch_intraday(self, ts_code: str, date: str = "") -> dict[str, Any]:
|
||||
code = str(ts_code or "").upper()
|
||||
if code in INDEX_SECIDS:
|
||||
secid = INDEX_SECIDS[code]
|
||||
entity = "index"
|
||||
identifier = code
|
||||
elif code.startswith("BK") or code.endswith((".TI", ".SI")):
|
||||
symbol = code.split(".")[0]
|
||||
secid = f"90.{symbol}"
|
||||
entity = "board"
|
||||
identifier = symbol
|
||||
else:
|
||||
symbol = code.split(".")[0]
|
||||
market = "1" if symbol.startswith(("5", "6", "9")) else "0"
|
||||
secid = f"{market}.{symbol}"
|
||||
entity = "stock"
|
||||
identifier = symbol
|
||||
params = {
|
||||
"secid": secid,
|
||||
"fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13",
|
||||
"fields2": "f51,f52,f53,f54,f55,f56,f57,f58",
|
||||
"iscr": "0",
|
||||
}
|
||||
data: dict[str, Any] = {}
|
||||
points: list[dict[str, Any]] = []
|
||||
last_error: Exception | None = None
|
||||
for url, ndays in ((TRENDS_URL, "1"), (TRENDS_URL, "5"), (HIS_TRENDS_URL, "5")):
|
||||
try:
|
||||
payload = self._get_json(
|
||||
url,
|
||||
{**params, "ndays": ndays},
|
||||
referer="https://quote.eastmoney.com/",
|
||||
)
|
||||
except AdapterError as exc:
|
||||
last_error = exc
|
||||
continue
|
||||
data = payload.get("data") or {}
|
||||
parsed = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))]
|
||||
points = _preferred_session(parsed, date)
|
||||
if points:
|
||||
break
|
||||
if not points:
|
||||
raise AdapterError("No intraday chart data returned") from last_error
|
||||
return {
|
||||
"entity_type": entity,
|
||||
"identifier": identifier,
|
||||
"ts_code": code if "." in code else f"{identifier}.{'SH' if identifier.startswith(('5','6','9')) else 'SZ'}",
|
||||
"name": str(data.get("name") or ""),
|
||||
"code": str(data.get("code") or identifier),
|
||||
"trade_date": points[-1]["date"],
|
||||
"previous_close": round4(finite_number(data.get("preClose"))),
|
||||
"points": points,
|
||||
"source": "eastmoney_trends2",
|
||||
}
|
||||
|
||||
def fetch_shenwan_quote(self, ts_code: str) -> dict[str, Any]:
|
||||
code = str(ts_code or "").split(".")[0]
|
||||
if not code:
|
||||
raise AdapterError("Invalid Shenwan code")
|
||||
rows = self.fetch_shenwan_quotes([code])
|
||||
if not rows:
|
||||
raise AdapterError(f"Eastmoney Shenwan quote missing for {code}")
|
||||
return rows[0]
|
||||
|
||||
def fetch_shenwan_quotes(self, ts_codes: list[str]) -> list[dict[str, Any]]:
|
||||
codes = [str(item or "").split(".")[0] for item in ts_codes]
|
||||
codes = list(dict.fromkeys(code for code in codes if code))
|
||||
if not codes:
|
||||
return []
|
||||
payload = self._get_json(
|
||||
EASTMONEY_INDEX_URL,
|
||||
{
|
||||
"secids": ",".join(f"90.{code}" for code in codes),
|
||||
"fltt": "2",
|
||||
"invt": "2",
|
||||
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f8,f104,f105,f128,f136,f140,f124",
|
||||
},
|
||||
referer="https://quote.eastmoney.com/",
|
||||
)
|
||||
rows = list((payload.get("data") or {}).get("diff") or [])
|
||||
result = [quote for row in rows if row and (quote := self._normalize_shenwan_quote(row))]
|
||||
if not result:
|
||||
raise AdapterError(f"Eastmoney Shenwan quotes missing for {','.join(codes)}")
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _normalize_shenwan_quote(row: dict[str, Any]) -> dict[str, Any] | None:
|
||||
code = str(row.get("f12") or "")
|
||||
if not code:
|
||||
return None
|
||||
epoch = int(finite_number(row.get("f124")) or 0)
|
||||
close = round4(finite_number(row.get("f2")))
|
||||
previous = round4(finite_number(row.get("f18")))
|
||||
if close <= 0 or previous <= 0:
|
||||
return None
|
||||
quote_time = (
|
||||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||
if epoch
|
||||
else ""
|
||||
)
|
||||
return {
|
||||
"ts_code": f"{code}.SI",
|
||||
"code": f"{code}.SI",
|
||||
"name": row.get("f14") or code,
|
||||
"price": close,
|
||||
"close": close,
|
||||
"pre_close": previous,
|
||||
"previous_close": previous,
|
||||
"open": round4(finite_number(row.get("f17"))),
|
||||
"high": round4(finite_number(row.get("f15"))),
|
||||
"low": round4(finite_number(row.get("f16"))),
|
||||
"change": round4(finite_number(row.get("f3"))),
|
||||
"pct_change": round4(finite_number(row.get("f3"))),
|
||||
"pct_chg": round4(finite_number(row.get("f3"))),
|
||||
"amount": round4(finite_number(row.get("f6"))),
|
||||
"leader": row.get("f128") or "--",
|
||||
"leader_code": row.get("f140") or "",
|
||||
"leading_pct": round4(finite_number(row.get("f136"))),
|
||||
"up_count": int(finite_number(row.get("f104")) or 0),
|
||||
"down_count": int(finite_number(row.get("f105")) or 0),
|
||||
"quote_time": quote_time,
|
||||
"trade_time": quote_time,
|
||||
"quote_date": datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d") if epoch else "",
|
||||
"quote_time_epoch": epoch,
|
||||
"source": "eastmoney_sw",
|
||||
}
|
||||
|
||||
def fetch_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]]:
|
||||
day = str(trade_date or "").replace("-", "")
|
||||
rows: list[dict[str, Any]] = []
|
||||
for url, limit_type in (
|
||||
("https://push2ex.eastmoney.com/getTopicZTPool", "U"),
|
||||
("https://push2ex.eastmoney.com/getTopicZBPool", "Z"),
|
||||
):
|
||||
params = {
|
||||
"ut": "7eea3edcaed734bea9cbfc24409ed989",
|
||||
"dpt": "wz.ztzt",
|
||||
"PageIndex": "0",
|
||||
"PageSize": "200",
|
||||
"sort": "fbt:asc",
|
||||
"stat": "1",
|
||||
}
|
||||
if day:
|
||||
params["date"] = day
|
||||
try:
|
||||
payload = self._get_json(url, params, referer="https://quote.eastmoney.com/")
|
||||
except AdapterError:
|
||||
continue
|
||||
pool = ((payload.get("data") or {}).get("pool") or []) if isinstance(payload.get("data"), dict) else []
|
||||
for item in pool:
|
||||
code = str(item.get("c") or item.get("code") or "")
|
||||
if not code:
|
||||
continue
|
||||
market = str(item.get("m") or item.get("market") or "")
|
||||
suffix = "SH" if market in {"1", "SH"} or code.startswith(("5", "6", "9")) else "SZ"
|
||||
first = str(item.get("fbt") or item.get("first_time") or "")
|
||||
last = str(item.get("lbt") or item.get("last_time") or "")
|
||||
rows.append(
|
||||
{
|
||||
"ts_code": f"{code}.{suffix}",
|
||||
"limit_type": limit_type,
|
||||
"first_time": first,
|
||||
"last_time": last,
|
||||
"fd_amount": item.get("fund") or item.get("fd_amount"),
|
||||
"open_times": item.get("zbc") or item.get("open_times"),
|
||||
"limit_times": item.get("lbc") or item.get("limit_times"),
|
||||
"turnover_ratio": item.get("hs") or item.get("turnover_ratio"),
|
||||
"source": "eastmoney_zt_pool",
|
||||
}
|
||||
)
|
||||
if not rows:
|
||||
raise AdapterError("Eastmoney limit pool empty")
|
||||
return rows
|
||||
|
||||
def _get_json(self, url: str, params: dict[str, str], referer: str) -> dict[str, Any]:
|
||||
request_url = f"{url}?{urllib.parse.urlencode(params)}"
|
||||
request = urllib.request.Request(
|
||||
request_url,
|
||||
headers={
|
||||
"Accept": "application/json,text/plain,*/*",
|
||||
"User-Agent": BROWSER_UA,
|
||||
"Referer": referer,
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
except Exception as exc:
|
||||
raise AdapterError(f"eastmoney request failed: {exc}") from exc
|
||||
|
||||
|
||||
def _preferred_session(points: list[dict[str, Any]], preferred_date: str = "") -> list[dict[str, Any]]:
|
||||
if not points:
|
||||
return []
|
||||
want = ""
|
||||
digits = str(preferred_date or "").replace("-", "")[:8]
|
||||
if len(digits) == 8 and digits.isdigit():
|
||||
want = f"{digits[:4]}-{digits[4:6]}-{digits[6:8]}"
|
||||
if want:
|
||||
matched = [point for point in points if str(point.get("date") or "") == want]
|
||||
if matched:
|
||||
return matched
|
||||
latest = max(str(point.get("date") or "") for point in points)
|
||||
if not latest:
|
||||
return points
|
||||
return [point for point in points if str(point.get("date") or "") == latest]
|
||||
|
||||
|
||||
def _normalize_market_quote(row: dict[str, Any]) -> dict[str, Any] | None:
|
||||
symbol = str(row.get("f12") or "").strip()
|
||||
if not symbol.isdigit() or len(symbol) != 6:
|
||||
return None
|
||||
close = round4(finite_number(row.get("f2")))
|
||||
previous_close = round4(finite_number(row.get("f18")))
|
||||
if close <= 0 or previous_close <= 0:
|
||||
return None
|
||||
market = int(finite_number(row.get("f13")) or 0)
|
||||
if market == 0 and symbol.startswith(("4", "8", "92")):
|
||||
ts_code = f"{symbol}.BJ"
|
||||
elif market == 1 or symbol.startswith(("5", "6", "9")):
|
||||
ts_code = f"{symbol}.SH"
|
||||
else:
|
||||
ts_code = f"{symbol}.SZ"
|
||||
epoch = int(finite_number(row.get("f124")) or 0)
|
||||
quote_date = ""
|
||||
if epoch > 0:
|
||||
quote_date = datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d")
|
||||
return {
|
||||
"ts_code": ts_code,
|
||||
"name": row.get("f14") or symbol,
|
||||
"pre_close": previous_close,
|
||||
"previous_close": previous_close,
|
||||
"open": round4(finite_number(row.get("f17"))),
|
||||
"high": round4(finite_number(row.get("f15"))),
|
||||
"low": round4(finite_number(row.get("f16"))),
|
||||
"close": close,
|
||||
"price": close,
|
||||
"pct_chg": round4(finite_number(row.get("f3"))),
|
||||
"vol": round4(finite_number(row.get("f5")) * 100),
|
||||
"volume": round4(finite_number(row.get("f5")) * 100),
|
||||
"amount": round4(finite_number(row.get("f6"))),
|
||||
"quote_date": quote_date,
|
||||
"quote_time_epoch": epoch,
|
||||
"source": "eastmoney_clist",
|
||||
}
|
||||
|
||||
|
||||
def _parse_trend(raw: Any) -> dict[str, Any] | None:
|
||||
text = str(raw or "")
|
||||
parts = text.split(",")
|
||||
if len(parts) < 8:
|
||||
return None
|
||||
stamp = parts[0]
|
||||
try:
|
||||
when = datetime.strptime(stamp, "%Y-%m-%d %H:%M")
|
||||
except ValueError:
|
||||
return None
|
||||
close = round4(finite_number(parts[2]))
|
||||
if close <= 0:
|
||||
return None
|
||||
return {
|
||||
"time": when.strftime("%H:%M"),
|
||||
"date": when.strftime("%Y-%m-%d"),
|
||||
"open": round4(finite_number(parts[1])),
|
||||
"close": close,
|
||||
"high": round4(finite_number(parts[3])),
|
||||
"low": round4(finite_number(parts[4])),
|
||||
"avg_price": round4(finite_number(parts[7] if len(parts) > 7 else parts[2])),
|
||||
"volume": round4(finite_number(parts[5])),
|
||||
"amount": round4(finite_number(parts[6])),
|
||||
}
|
||||
|
||||
@@ -1,3 +1,432 @@
|
||||
from datahub.adapters.base import ReservedAdapter
|
||||
from __future__ import annotations
|
||||
|
||||
ADAPTER = ReservedAdapter("ifind")
|
||||
import copy
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Callable
|
||||
|
||||
from datahub.adapters.base import AdapterError, MarketAdapter
|
||||
|
||||
UrlOpen = Callable[..., Any]
|
||||
|
||||
|
||||
class IfindAdapter(MarketAdapter):
|
||||
"""Licensed iFinD source used only inside the data hub."""
|
||||
|
||||
name = "ifind"
|
||||
BASE_URL = "https://quantapi.51ifind.com/api/v1"
|
||||
AUTH_ENDPOINT = "get_access_token"
|
||||
AUTH_ERROR_CODES = {-1302, -1303, -1304, -4302, -4303}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
refresh_token: str = "",
|
||||
access_token: str = "",
|
||||
timeout: int = 15,
|
||||
urlopen: UrlOpen = urllib.request.urlopen,
|
||||
) -> None:
|
||||
self.timeout = max(3, int(timeout))
|
||||
self._urlopen = urlopen
|
||||
self._refresh_token = str(refresh_token or "").strip()
|
||||
self._access_token = str(access_token or "").strip()
|
||||
self._access_expires_at: datetime | None = None
|
||||
self._token_lock = threading.Lock()
|
||||
self._cache_lock = threading.Lock()
|
||||
self._cache: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self._refresh_token or self._access_token)
|
||||
|
||||
def set_credentials(self, refresh_token: str, access_token: str = "") -> None:
|
||||
refresh_token = str(refresh_token or "").strip()
|
||||
access_token = str(access_token or "").strip()
|
||||
with self._token_lock:
|
||||
refresh_changed = refresh_token != self._refresh_token
|
||||
self._refresh_token = refresh_token
|
||||
if access_token or refresh_changed:
|
||||
self._access_token = access_token
|
||||
self._access_expires_at = None
|
||||
if refresh_changed:
|
||||
with self._cache_lock:
|
||||
self._cache.clear()
|
||||
|
||||
def status(self) -> dict[str, Any]:
|
||||
return {
|
||||
"configured": self.configured,
|
||||
"access_ready": bool(self._access_token),
|
||||
"access_expires_at": (
|
||||
self._access_expires_at.isoformat(timespec="seconds")
|
||||
if self._access_expires_at
|
||||
else ""
|
||||
),
|
||||
}
|
||||
|
||||
def probe(self) -> dict[str, Any]:
|
||||
started = time.perf_counter()
|
||||
if not self.configured:
|
||||
return {
|
||||
"provider": self.name,
|
||||
"configured": False,
|
||||
"state": "unconfigured",
|
||||
"message": "iFinD token 未配置",
|
||||
"latency_ms": round((time.perf_counter() - started) * 1000),
|
||||
}
|
||||
try:
|
||||
rows = self.real_time("000001.SH", ["latest"], cache_ttl=0)
|
||||
state = "ok" if rows else "empty"
|
||||
return {
|
||||
"provider": self.name,
|
||||
"configured": True,
|
||||
"state": state,
|
||||
"latency_ms": round((time.perf_counter() - started) * 1000),
|
||||
}
|
||||
except AdapterError as exc:
|
||||
return {
|
||||
"provider": self.name,
|
||||
"configured": True,
|
||||
"state": "error",
|
||||
"message": str(exc),
|
||||
"latency_ms": round((time.perf_counter() - started) * 1000),
|
||||
}
|
||||
|
||||
def fetch(self, dataset: str, params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if dataset == "wencai":
|
||||
return self.wencai(
|
||||
str(params.get("query") or params.get("searchstring") or ""),
|
||||
str(params.get("search_type") or params.get("searchtype") or "stock"),
|
||||
int(params.get("cache_ttl") or 300),
|
||||
)
|
||||
if dataset == "snapshots":
|
||||
return self.snapshots(
|
||||
params.get("codes") or "",
|
||||
_indicators(params.get("indicators")),
|
||||
str(params.get("start_time") or ""),
|
||||
str(params.get("end_time") or ""),
|
||||
int(params.get("cache_ttl") or 8),
|
||||
)
|
||||
if dataset == "history":
|
||||
return self.history(
|
||||
params.get("codes") or "",
|
||||
_indicators(params.get("indicators") or ["close", "volume", "amount"]),
|
||||
str(params.get("start_date") or ""),
|
||||
str(params.get("end_date") or ""),
|
||||
int(params.get("cache_ttl") or 300),
|
||||
)
|
||||
if dataset == "realtime":
|
||||
return self.real_time(
|
||||
params.get("codes") or "",
|
||||
_indicators(params.get("indicators") or ["latest"]),
|
||||
int(params.get("cache_ttl") or 10),
|
||||
)
|
||||
if dataset == "intraday":
|
||||
return self.intraday(
|
||||
str(params.get("code") or params.get("codes") or ""),
|
||||
str(params.get("start_time") or ""),
|
||||
str(params.get("end_time") or ""),
|
||||
int(params.get("cache_ttl") or 20),
|
||||
)
|
||||
raise AdapterError(f"{self.name} unsupported dataset: {dataset}")
|
||||
|
||||
def normalize(self, dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return list(rows)
|
||||
|
||||
def real_time(
|
||||
self,
|
||||
codes: str | list[str],
|
||||
indicators: list[str],
|
||||
cache_ttl: int = 10,
|
||||
) -> list[dict[str, Any]]:
|
||||
code_text = self._codes(codes)
|
||||
payload = self._request(
|
||||
"real_time_quotation",
|
||||
{"codes": code_text, "indicators": ",".join(indicators)},
|
||||
cache_key=f"rq:{code_text}:{','.join(indicators)}",
|
||||
cache_ttl=cache_ttl,
|
||||
)
|
||||
return self._table_rows(payload)
|
||||
|
||||
def history(
|
||||
self,
|
||||
codes: str | list[str],
|
||||
indicators: list[str],
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
cache_ttl: int = 300,
|
||||
) -> list[dict[str, Any]]:
|
||||
code_text = self._codes(codes)
|
||||
payload = self._request(
|
||||
"cmd_history_quotation",
|
||||
{
|
||||
"codes": code_text,
|
||||
"indicators": ",".join(indicators),
|
||||
"startdate": self._display_date(start_date),
|
||||
"enddate": self._display_date(end_date),
|
||||
"functionpara": {"CPS": "forward1", "Fill": "Omit"},
|
||||
},
|
||||
cache_key=f"hq:{code_text}:{start_date}:{end_date}:{','.join(indicators)}",
|
||||
cache_ttl=cache_ttl,
|
||||
)
|
||||
return self._table_rows(payload)
|
||||
|
||||
def intraday(
|
||||
self,
|
||||
code: str,
|
||||
start_time: str,
|
||||
end_time: str,
|
||||
cache_ttl: int = 20,
|
||||
) -> list[dict[str, Any]]:
|
||||
indicators = ["open", "high", "low", "close", "volume", "amount", "avgPrice"]
|
||||
payload = self._request(
|
||||
"high_frequency",
|
||||
{
|
||||
"codes": self._codes(code),
|
||||
"indicators": ",".join(indicators),
|
||||
"starttime": start_time,
|
||||
"endtime": end_time,
|
||||
"functionpara": {
|
||||
"CPS": "forward1",
|
||||
"Fill": "Previous",
|
||||
"Timeformat": "LocalTime",
|
||||
"Interval": "1",
|
||||
"Limitstart": "09:30:00",
|
||||
"Limitend": "15:00:00",
|
||||
},
|
||||
},
|
||||
cache_key=f"hf:{code}:{start_time}:{end_time}",
|
||||
cache_ttl=cache_ttl,
|
||||
)
|
||||
return self._table_rows(payload)
|
||||
|
||||
def snapshots(
|
||||
self,
|
||||
codes: str | list[str],
|
||||
indicators: list[str],
|
||||
start_time: str,
|
||||
end_time: str,
|
||||
cache_ttl: int = 8,
|
||||
) -> list[dict[str, Any]]:
|
||||
code_text = self._codes(codes)
|
||||
payload = self._request(
|
||||
"snap_shot",
|
||||
{
|
||||
"codes": code_text,
|
||||
"indicators": ",".join(indicators),
|
||||
"starttime": start_time,
|
||||
"endtime": end_time,
|
||||
},
|
||||
cache_key=f"ss:{code_text}:{start_time}:{end_time}:{','.join(indicators)}",
|
||||
cache_ttl=cache_ttl,
|
||||
)
|
||||
return self._table_rows(payload)
|
||||
|
||||
def wencai(self, query: str, search_type: str = "stock", cache_ttl: int = 300) -> list[dict[str, Any]]:
|
||||
normalized = " ".join(str(query or "").split())
|
||||
if not normalized:
|
||||
raise AdapterError("问财查询不能为空。")
|
||||
payload = self._request(
|
||||
"smart_stock_picking",
|
||||
{"searchstring": normalized, "searchtype": search_type},
|
||||
cache_key=f"wc:{search_type}:{normalized}",
|
||||
cache_ttl=cache_ttl,
|
||||
)
|
||||
return self._table_rows(payload)
|
||||
|
||||
def _request(
|
||||
self,
|
||||
endpoint: str,
|
||||
body: dict[str, Any],
|
||||
cache_key: str = "",
|
||||
cache_ttl: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
if not self.configured:
|
||||
raise AdapterError("iFinD 尚未配置。")
|
||||
if cache_key and cache_ttl > 0:
|
||||
cached = self._cached(cache_key, cache_ttl)
|
||||
if cached is not None:
|
||||
return cached
|
||||
payload = self._post(endpoint, body, self._ensure_access_token())
|
||||
if self._is_auth_error(payload) and self._refresh_token:
|
||||
self._invalidate_access_token()
|
||||
payload = self._post(endpoint, body, self._ensure_access_token(force=True))
|
||||
self._validate_payload(payload)
|
||||
if cache_key and cache_ttl > 0:
|
||||
with self._cache_lock:
|
||||
self._cache[cache_key] = {
|
||||
"created_at": time.time(),
|
||||
"payload": copy.deepcopy(payload),
|
||||
}
|
||||
return payload
|
||||
|
||||
def _ensure_access_token(self, force: bool = False) -> str:
|
||||
with self._token_lock:
|
||||
now = datetime.now().astimezone().replace(tzinfo=None)
|
||||
token_valid = bool(self._access_token) and (
|
||||
self._access_expires_at is None
|
||||
or self._access_expires_at > now + timedelta(minutes=2)
|
||||
)
|
||||
if token_valid and not force:
|
||||
return self._access_token
|
||||
if not self._refresh_token:
|
||||
if self._access_token:
|
||||
return self._access_token
|
||||
raise AdapterError("iFinD Refresh Token 尚未配置。")
|
||||
payload = self._post(self.AUTH_ENDPOINT, {}, "", self._refresh_token)
|
||||
self._validate_payload(payload)
|
||||
data = payload.get("data") or {}
|
||||
token = str(data.get("access_token") or "").strip()
|
||||
if not token:
|
||||
raise AdapterError("iFinD 未返回 Access Token。")
|
||||
expires_at = self._parse_datetime(data.get("expired_time"))
|
||||
self._access_token = token
|
||||
self._access_expires_at = expires_at
|
||||
return token
|
||||
|
||||
def _post(
|
||||
self,
|
||||
endpoint: str,
|
||||
body: dict[str, Any],
|
||||
access_token: str,
|
||||
refresh_token: str = "",
|
||||
) -> dict[str, Any]:
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "XiaobaiDatahub/1.0",
|
||||
"ifindlang": "cn",
|
||||
}
|
||||
if access_token:
|
||||
headers["access_token"] = access_token
|
||||
if refresh_token:
|
||||
headers["refresh_token"] = refresh_token
|
||||
request = urllib.request.Request(
|
||||
f"{self.BASE_URL}/{endpoint}",
|
||||
data=json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
|
||||
headers=headers,
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with self._urlopen(request, timeout=self.timeout) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = ""
|
||||
try:
|
||||
detail_payload = json.loads(exc.read().decode("utf-8", errors="replace"))
|
||||
detail = str(detail_payload.get("errmsg") or detail_payload.get("message") or "")
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
raise AdapterError(f"iFinD HTTP {exc.code}{f':{detail[:160]}' if detail else ''}") from exc
|
||||
except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
|
||||
raise AdapterError("iFinD 数据请求失败。") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise AdapterError("iFinD 返回格式不正确。")
|
||||
return payload
|
||||
|
||||
def _cached(self, key: str, ttl: int) -> dict[str, Any] | None:
|
||||
with self._cache_lock:
|
||||
cached = self._cache.get(key)
|
||||
if not cached:
|
||||
return None
|
||||
if time.time() - float(cached.get("created_at") or 0) > ttl:
|
||||
self._cache.pop(key, None)
|
||||
return None
|
||||
return copy.deepcopy(cached["payload"])
|
||||
|
||||
def _invalidate_access_token(self) -> None:
|
||||
with self._token_lock:
|
||||
self._access_token = ""
|
||||
self._access_expires_at = None
|
||||
|
||||
@classmethod
|
||||
def _validate_payload(cls, payload: dict[str, Any]) -> None:
|
||||
try:
|
||||
error_code = int(payload.get("errorcode") or 0)
|
||||
except (TypeError, ValueError):
|
||||
error_code = -1
|
||||
if error_code != 0:
|
||||
message = str(payload.get("errmsg") or "未知错误")
|
||||
raise AdapterError(f"iFinD 返回错误:{message[:200]}")
|
||||
|
||||
@classmethod
|
||||
def _is_auth_error(cls, payload: dict[str, Any]) -> bool:
|
||||
try:
|
||||
error_code = int(payload.get("errorcode") or 0)
|
||||
except (TypeError, ValueError):
|
||||
error_code = 0
|
||||
message = str(payload.get("errmsg") or "").casefold()
|
||||
return error_code in cls.AUTH_ERROR_CODES or "token" in message or "鉴权" in message
|
||||
|
||||
@staticmethod
|
||||
def _table_rows(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
tables = payload.get("tables") or []
|
||||
if isinstance(tables, dict):
|
||||
tables = [tables]
|
||||
rows: list[dict[str, Any]] = []
|
||||
for block in tables if isinstance(tables, list) else []:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
table = block.get("table") or {}
|
||||
if not isinstance(table, dict):
|
||||
continue
|
||||
times = block.get("time") or []
|
||||
codes = block.get("thscode") or block.get("thscodes") or []
|
||||
if isinstance(codes, str):
|
||||
codes = [codes]
|
||||
lengths = [len(value) for value in table.values() if isinstance(value, list)]
|
||||
row_count = max(lengths or [len(times) if isinstance(times, list) else 0, 1 if table else 0])
|
||||
for index in range(row_count):
|
||||
row: dict[str, Any] = {}
|
||||
if isinstance(times, list) and index < len(times):
|
||||
row["time"] = times[index]
|
||||
if codes:
|
||||
row["thscode"] = codes[index] if index < len(codes) else codes[0]
|
||||
for field, values in table.items():
|
||||
if isinstance(values, list):
|
||||
row[field] = values[index] if index < len(values) else None
|
||||
elif index == 0:
|
||||
row[field] = values
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
@staticmethod
|
||||
def _codes(codes: str | list[str]) -> str:
|
||||
if isinstance(codes, list):
|
||||
values = [str(code or "").strip().upper() for code in codes]
|
||||
else:
|
||||
values = [part.strip().upper() for part in str(codes or "").split(",")]
|
||||
values = [value for value in values if value]
|
||||
if not values:
|
||||
raise AdapterError("iFinD 证券代码不能为空。")
|
||||
if len(values) > 100:
|
||||
raise AdapterError("iFinD 单次证券代码过多。")
|
||||
return ",".join(values)
|
||||
|
||||
@staticmethod
|
||||
def _display_date(value: str) -> str:
|
||||
compact = str(value or "").replace("-", "")
|
||||
if len(compact) != 8 or not compact.isdigit():
|
||||
raise AdapterError("iFinD 日期格式不正确。")
|
||||
return f"{compact[:4]}-{compact[4:6]}-{compact[6:]}"
|
||||
|
||||
@staticmethod
|
||||
def _parse_datetime(value: Any) -> datetime | None:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(text)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _indicators(value: Any) -> list[str]:
|
||||
if isinstance(value, list):
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
return [part.strip() for part in str(value or "").split(",") if part.strip()]
|
||||
|
||||
|
||||
ADAPTER = IfindAdapter()
|
||||
|
||||
@@ -1,3 +1,218 @@
|
||||
from datahub.adapters.base import ReservedAdapter
|
||||
from __future__ import annotations
|
||||
|
||||
ADAPTER = ReservedAdapter("tencent")
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from datahub.adapters.base import AdapterError, MarketAdapter
|
||||
from datahub.numbers import finite_number, round4
|
||||
|
||||
TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006"
|
||||
TENCENT_QUOTE_URL = "https://qt.gtimg.cn/q="
|
||||
TENCENT_QUOTE_BATCH = 80
|
||||
BROWSER_UA = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
|
||||
class TencentAdapter(MarketAdapter):
|
||||
name = "tencent"
|
||||
|
||||
def __init__(self, timeout: int = 8) -> None:
|
||||
self.timeout = timeout
|
||||
|
||||
def probe(self) -> dict[str, Any]:
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
rows = self.fetch_indices()
|
||||
state = "ok" if len(rows) == 3 else "empty"
|
||||
except AdapterError as exc:
|
||||
return {
|
||||
"provider": self.name,
|
||||
"configured": True,
|
||||
"state": "error",
|
||||
"message": str(exc),
|
||||
"latency_ms": round((time.perf_counter() - started) * 1000),
|
||||
}
|
||||
return {
|
||||
"provider": self.name,
|
||||
"configured": True,
|
||||
"state": state,
|
||||
"latency_ms": round((time.perf_counter() - started) * 1000),
|
||||
}
|
||||
|
||||
def fetch(self, dataset: str, params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if dataset in {"indexes_quotes", "index_quotes"}:
|
||||
return self.fetch_indices()
|
||||
if dataset in {"quotes", "quotes_latest"}:
|
||||
codes = params.get("codes") or []
|
||||
if isinstance(codes, str):
|
||||
codes = [item.strip() for item in codes.split(",") if item.strip()]
|
||||
return self.fetch_quotes(list(codes))
|
||||
if dataset in {"quotes_market", "market_quotes"}:
|
||||
codes = params.get("codes") or []
|
||||
if isinstance(codes, str):
|
||||
codes = [item.strip() for item in codes.split(",") if item.strip()]
|
||||
return self.fetch_quotes(list(codes))
|
||||
raise AdapterError(f"{self.name} unsupported dataset: {dataset}")
|
||||
|
||||
def normalize(self, dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return list(rows)
|
||||
|
||||
def fetch_indices(self) -> list[dict[str, Any]]:
|
||||
request = urllib.request.Request(
|
||||
TENCENT_INDEX_URL,
|
||||
headers={"User-Agent": BROWSER_UA, "Referer": "https://gu.qq.com/"},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||||
raw = response.read().decode("gb18030", errors="ignore")
|
||||
except Exception as exc:
|
||||
raise AdapterError(f"tencent request failed: {exc}") from exc
|
||||
result = []
|
||||
for line in raw.splitlines():
|
||||
if '="' not in line:
|
||||
continue
|
||||
fields = line.split('="', 1)[1].rsplit('";', 1)[0].split("~")
|
||||
if len(fields) < 38:
|
||||
continue
|
||||
code = fields[2]
|
||||
if code not in {"000001", "399001", "399006"}:
|
||||
continue
|
||||
try:
|
||||
quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S").astimezone()
|
||||
except ValueError as exc:
|
||||
raise AdapterError(f"Tencent invalid quote time for {code}") from exc
|
||||
ts_code = "000001.SH" if code == "000001" else f"{code}.SZ"
|
||||
result.append(
|
||||
{
|
||||
"ts_code": ts_code,
|
||||
"code": code,
|
||||
"name": fields[1] or code,
|
||||
"price": round4(finite_number(fields[3])),
|
||||
"pct_chg": round4(finite_number(fields[32])),
|
||||
"change_amount": round4(finite_number(fields[31])),
|
||||
"open": round4(finite_number(fields[5])),
|
||||
"high": round4(finite_number(fields[33])),
|
||||
"low": round4(finite_number(fields[34])),
|
||||
"previous_close": round4(finite_number(fields[4])),
|
||||
"amount": round4(finite_number(fields[37]) * 10000),
|
||||
"quote_time_epoch": int(quote_time.timestamp()),
|
||||
"quote_time": quote_time.isoformat(timespec="seconds"),
|
||||
"source": "tencent_qt",
|
||||
}
|
||||
)
|
||||
if len(result) != 3:
|
||||
raise AdapterError(f"Tencent returned {len(result)}/3 indices")
|
||||
return result
|
||||
|
||||
def fetch_quotes(self, codes: list[str]) -> list[dict[str, Any]]:
|
||||
symbols: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in codes:
|
||||
symbol = _tencent_symbol(str(raw or ""))
|
||||
if not symbol or symbol in seen:
|
||||
continue
|
||||
seen.add(symbol)
|
||||
symbols.append(symbol)
|
||||
if not symbols:
|
||||
return []
|
||||
result: list[dict[str, Any]] = []
|
||||
errors: list[str] = []
|
||||
for index in range(0, len(symbols), TENCENT_QUOTE_BATCH):
|
||||
batch = symbols[index:index + TENCENT_QUOTE_BATCH]
|
||||
try:
|
||||
raw = self._get_text(f"{TENCENT_QUOTE_URL}{','.join(batch)}")
|
||||
except AdapterError as exc:
|
||||
errors.append(str(exc))
|
||||
continue
|
||||
for line in raw.splitlines():
|
||||
quote = _parse_tencent_stock_quote(line)
|
||||
if quote:
|
||||
result.append(quote)
|
||||
if not result:
|
||||
detail = f";{'; '.join(errors[:3])}" if errors else ""
|
||||
raise AdapterError(f"Tencent quotes empty{detail}")
|
||||
return result
|
||||
|
||||
def _get_text(self, url: str) -> str:
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
headers={"User-Agent": BROWSER_UA, "Referer": "https://gu.qq.com/"},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||||
return response.read().decode("gb18030", errors="ignore")
|
||||
except Exception as exc:
|
||||
raise AdapterError(f"tencent request failed: {exc}") from exc
|
||||
|
||||
|
||||
def _tencent_symbol(code: str) -> str:
|
||||
raw = str(code or "").strip().upper()
|
||||
if not raw:
|
||||
return ""
|
||||
symbol = raw.split(".")[0]
|
||||
if not symbol.isdigit() or len(symbol) != 6:
|
||||
return ""
|
||||
if raw.endswith(".BJ") or symbol.startswith(("4", "8", "92")):
|
||||
return f"bj{symbol}"
|
||||
if raw.endswith(".SH") or symbol.startswith(("5", "6", "9")):
|
||||
return f"sh{symbol}"
|
||||
return f"sz{symbol}"
|
||||
|
||||
|
||||
def _parse_tencent_stock_quote(line: str) -> dict[str, Any] | None:
|
||||
if '="' not in line:
|
||||
return None
|
||||
prefix, payload = line.split('="', 1)
|
||||
fields = payload.rsplit('";', 1)[0].split("~")
|
||||
if len(fields) < 38:
|
||||
return None
|
||||
symbol = str(fields[2] or "")
|
||||
if not symbol.isdigit() or len(symbol) != 6:
|
||||
return None
|
||||
close = round4(finite_number(fields[3]))
|
||||
previous = round4(finite_number(fields[4]))
|
||||
if not close or not previous or close <= 0 or previous <= 0:
|
||||
return None
|
||||
marker = prefix.lower()
|
||||
if "sh" in marker:
|
||||
ts_code = f"{symbol}.SH"
|
||||
elif "bj" in marker:
|
||||
ts_code = f"{symbol}.BJ"
|
||||
else:
|
||||
ts_code = f"{symbol}.SZ"
|
||||
quote_stamp = ""
|
||||
quote_date = ""
|
||||
epoch = 0
|
||||
try:
|
||||
parsed = datetime.strptime(fields[30], "%Y%m%d%H%M%S")
|
||||
quote_date = parsed.strftime("%Y%m%d")
|
||||
epoch = int(parsed.timestamp())
|
||||
quote_stamp = parsed.astimezone().isoformat(timespec="seconds")
|
||||
except ValueError:
|
||||
pass
|
||||
return {
|
||||
"ts_code": ts_code,
|
||||
"name": fields[1] or symbol,
|
||||
"price": close,
|
||||
"close": close,
|
||||
"pct_chg": round4(finite_number(fields[32])),
|
||||
"change_amount": round4(finite_number(fields[31])),
|
||||
"open": round4(finite_number(fields[5])),
|
||||
"high": round4(finite_number(fields[33])),
|
||||
"low": round4(finite_number(fields[34])),
|
||||
"pre_close": previous,
|
||||
"previous_close": previous,
|
||||
"volume": round4(finite_number(fields[6]) * 100),
|
||||
"vol": round4(finite_number(fields[6]) * 100),
|
||||
"amount": round4(finite_number(fields[37]) * 10000),
|
||||
"quote_date": quote_date,
|
||||
"quote_time_epoch": epoch,
|
||||
"quote_time": quote_stamp,
|
||||
"source": "tencent_qt",
|
||||
}
|
||||
|
||||
@@ -11,8 +11,12 @@ from datahub.normalize import (
|
||||
normalize_auction,
|
||||
normalize_calendar,
|
||||
normalize_daily,
|
||||
normalize_dragon_tiger,
|
||||
normalize_index_daily,
|
||||
normalize_limit_event,
|
||||
normalize_moneyflow,
|
||||
normalize_popularity,
|
||||
normalize_sector_daily,
|
||||
normalize_stock,
|
||||
normalize_valuation,
|
||||
)
|
||||
@@ -31,6 +35,29 @@ TUSHARE_FIELDS = {
|
||||
"buy_lg_amount,sell_lg_amount,buy_elg_amount,sell_elg_amount,net_mf_amount"
|
||||
),
|
||||
"stk_auction": "ts_code,trade_date,vol,price,amount,pre_close,turnover_rate,volume_ratio,float_share",
|
||||
"limit_list_d": (
|
||||
"trade_date,ts_code,industry,name,close,pct_chg,amount,limit_amount,"
|
||||
"float_mv,total_mv,turnover_ratio,fd_amount,first_time,last_time,"
|
||||
"open_times,up_stat,limit_times,limit_type"
|
||||
),
|
||||
"ths_hot": "ts_code,ts_name,hot,rank,pct_change,current_price,concept,data_type,trade_date",
|
||||
"dc_hot": "ts_code,ts_name,rank,pct_change,current_price,hot,concept,data_type,trade_date",
|
||||
"hm_detail": "trade_date,ts_code,ts_name,buy_amount,sell_amount,net_amount,hm_name,hm_orgs,tag",
|
||||
"hm_list": "name,desc,orgs",
|
||||
"top_list": "trade_date,ts_code,name,pct_change,reason",
|
||||
"top_inst": "trade_date,ts_code,exalter,buy,buy_rate,sell,sell_rate,net_buy,side,reason",
|
||||
"ths_index": "ts_code,name,count,exchange,list_date,type",
|
||||
"ths_daily": "ts_code,trade_date,open,high,low,close,pre_close,pct_change,vol,turnover_rate",
|
||||
"dc_index": "ts_code,trade_date,name,open,high,low,close,pre_close,pct_change,vol,amount,turnover_rate",
|
||||
"sw_daily": "ts_code,trade_date,name,open,high,low,close,pct_change,vol,amount",
|
||||
"index_member_all": (
|
||||
"l1_code,l1_name,l2_code,l2_name,l3_code,l3_name,"
|
||||
"ts_code,name,in_date,out_date,is_new"
|
||||
),
|
||||
"stk_limit": "ts_code,trade_date,up_limit,down_limit",
|
||||
"suspend_d": "ts_code,suspend_date,resume_date,ann_date,suspend_reason,reason_type",
|
||||
"ths_member": "ts_code,con_code,con_name,in_date,out_date,is_new",
|
||||
"stk_mins": "ts_code,trade_time,open,close,high,low,vol,amount",
|
||||
}
|
||||
|
||||
DATASET_API = {
|
||||
@@ -42,9 +69,15 @@ DATASET_API = {
|
||||
"index_daily": "index_daily",
|
||||
"moneyflow": "moneyflow",
|
||||
"auction": "stk_auction",
|
||||
"limit_events": "limit_list_d",
|
||||
"popularity": "ths_hot",
|
||||
"dragon_tiger": "hm_detail",
|
||||
"sector_daily": "ths_daily",
|
||||
}
|
||||
|
||||
DEFAULT_INDEX_CODES = ("000001.SH", "399001.SZ", "399006.SZ", "000300.SH")
|
||||
WEBSITE_INDEX_CODES = ("000001.SH", "399001.SZ", "399006.SZ", "000300.SH")
|
||||
DEFAULT_INDEX_CODES = WEBSITE_INDEX_CODES
|
||||
LIMIT_TYPES = ("U", "D", "Z")
|
||||
|
||||
|
||||
class TushareAdapter(MarketAdapter):
|
||||
@@ -82,6 +115,14 @@ class TushareAdapter(MarketAdapter):
|
||||
}
|
||||
|
||||
def fetch(self, dataset: str, params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if dataset == "limit_events":
|
||||
return self.fetch_limit_events(str(params.get("trade_date") or ""))
|
||||
if dataset == "popularity":
|
||||
return self.fetch_popularity(str(params.get("trade_date") or ""))
|
||||
if dataset == "dragon_tiger":
|
||||
return self.fetch_dragon_tiger(str(params.get("trade_date") or ""))
|
||||
if dataset == "sector_daily":
|
||||
return self.fetch_sector_daily(str(params.get("trade_date") or ""))
|
||||
api_name = DATASET_API.get(dataset, dataset)
|
||||
fields = TUSHARE_FIELDS.get(api_name, "")
|
||||
query_params = dict(params)
|
||||
@@ -90,10 +131,67 @@ class TushareAdapter(MarketAdapter):
|
||||
if api_name == "trade_cal" and "exchange" not in query_params:
|
||||
query_params["exchange"] = "SSE"
|
||||
if api_name == "index_daily" and "ts_code" not in query_params:
|
||||
# Caller typically loops codes; a missing code would pull nothing useful.
|
||||
query_params.setdefault("ts_code", DEFAULT_INDEX_CODES[0])
|
||||
return self._query(api_name, query_params, fields)
|
||||
|
||||
def fetch_limit_events(self, trade_date: str) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for limit_type in LIMIT_TYPES:
|
||||
part = self._query(
|
||||
"limit_list_d",
|
||||
{"trade_date": trade_date, "limit_type": limit_type},
|
||||
TUSHARE_FIELDS["limit_list_d"],
|
||||
)
|
||||
for row in part:
|
||||
row = dict(row)
|
||||
row.setdefault("limit_type", limit_type)
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
def fetch_popularity(self, trade_date: str) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for api_name, source in (("ths_hot", "ths"), ("dc_hot", "dc")):
|
||||
for row in self._query(api_name, {"trade_date": trade_date}, TUSHARE_FIELDS[api_name]):
|
||||
item = dict(row)
|
||||
item["source"] = source
|
||||
item.setdefault("trade_date", trade_date)
|
||||
rows.append(item)
|
||||
return rows
|
||||
|
||||
def fetch_dragon_tiger(self, trade_date: str) -> list[dict[str, Any]]:
|
||||
details = self._query("hm_detail", {"trade_date": trade_date}, TUSHARE_FIELDS["hm_detail"])
|
||||
top_rows = self._query("top_list", {"trade_date": trade_date}, TUSHARE_FIELDS["top_list"])
|
||||
context = {
|
||||
str(row.get("ts_code") or ""): row
|
||||
for row in top_rows
|
||||
if str(row.get("ts_code") or "")
|
||||
}
|
||||
rows: list[dict[str, Any]] = []
|
||||
for row in details:
|
||||
item = dict(row)
|
||||
stock = context.get(str(item.get("ts_code") or ""), {})
|
||||
if item.get("pct_change") is None and stock.get("pct_change") is not None:
|
||||
item["pct_change"] = stock.get("pct_change")
|
||||
if not item.get("reason") and stock.get("reason"):
|
||||
item["reason"] = stock.get("reason")
|
||||
if not item.get("ts_name") and stock.get("name"):
|
||||
item["ts_name"] = stock.get("name")
|
||||
rows.append(item)
|
||||
return rows
|
||||
|
||||
def fetch_sector_daily(self, trade_date: str) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for api_name, family in (("ths_daily", "ths"), ("dc_index", "dc"), ("sw_daily", "sw")):
|
||||
try:
|
||||
part = self._query(api_name, {"trade_date": trade_date}, TUSHARE_FIELDS[api_name])
|
||||
except AdapterError:
|
||||
part = []
|
||||
for row in part:
|
||||
item = dict(row)
|
||||
item["family"] = family
|
||||
rows.append(item)
|
||||
return rows
|
||||
|
||||
def fetch_index_daily(self, trade_date: str, codes: tuple[str, ...] = DEFAULT_INDEX_CODES) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for ts_code in codes:
|
||||
@@ -101,6 +199,17 @@ class TushareAdapter(MarketAdapter):
|
||||
return rows
|
||||
|
||||
def normalize(self, dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
if dataset in {"limit_events", "limit_list_d"}:
|
||||
return [normalize_limit_event(row) for row in rows]
|
||||
if dataset == "popularity":
|
||||
return [normalize_popularity(row, source=str(row.get("source") or "")) for row in rows]
|
||||
if dataset == "dragon_tiger":
|
||||
return [normalize_dragon_tiger(row) for row in rows]
|
||||
if dataset == "sector_daily":
|
||||
return [
|
||||
normalize_sector_daily(row, family=str(row.get("family") or "ths"))
|
||||
for row in rows
|
||||
]
|
||||
mapping = {
|
||||
"calendar": normalize_calendar,
|
||||
"trade_cal": normalize_calendar,
|
||||
@@ -145,12 +254,14 @@ class TushareAdapter(MarketAdapter):
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||||
result = json.loads(response.read().decode("utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
raise AdapterError("Tushare returned invalid json") from None
|
||||
except (urllib.error.URLError, TimeoutError) as exc:
|
||||
raise AdapterError(f"Tushare request failed: {exc}") from exc
|
||||
if result.get("code") != 0:
|
||||
raise AdapterError(result.get("msg") or "Tushare returned an unknown error")
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
raise AdapterError(f"Tushare 请求失败: {exc}") from exc
|
||||
if result.get("code") not in (0, "0", None):
|
||||
raise AdapterError(str(result.get("msg") or f"Tushare error {result.get('code')}"))
|
||||
data = result.get("data") or {}
|
||||
columns = data.get("fields") or []
|
||||
return [dict(zip(columns, item)) for item in data.get("items") or []]
|
||||
items = data.get("items") or []
|
||||
fields_list = data.get("fields") or (fields.split(",") if fields else [])
|
||||
return [dict(zip(fields_list, item)) for item in items]
|
||||
|
||||
def query_raw(self, api_name: str, params: dict[str, Any], fields: str = "") -> list[dict[str, Any]]:
|
||||
return self._query(api_name, params, fields or TUSHARE_FIELDS.get(api_name, ""))
|
||||
|
||||
@@ -6,18 +6,19 @@ from typing import Any
|
||||
from datahub.adapters import RESERVED
|
||||
from datahub.auth import AuthService
|
||||
from datahub.db import HubDB
|
||||
from datahub.pipeline import Pipeline
|
||||
from datahub.pipeline import OFFICIAL_DATASETS, STOCKS_DATASET, Pipeline
|
||||
from datahub.scheduler import Scheduler
|
||||
from datahub.serving import ApiError
|
||||
from datahub.timeutil import isoformat, now_shanghai, session_phase, yyyymmdd
|
||||
|
||||
|
||||
class AdminAPI:
|
||||
def __init__(self, db: HubDB, pipeline: Pipeline, scheduler: Scheduler, auth: AuthService) -> None:
|
||||
def __init__(self, db: HubDB, pipeline: Pipeline, scheduler: Scheduler, auth: AuthService, ifind: Any = None) -> None:
|
||||
self.db = db
|
||||
self.pipeline = pipeline
|
||||
self.scheduler = scheduler
|
||||
self.auth = auth
|
||||
self.ifind = ifind
|
||||
|
||||
def overview(self) -> dict[str, Any]:
|
||||
today = yyyymmdd(now_shanghai())
|
||||
@@ -38,6 +39,8 @@ class AdminAPI:
|
||||
"trade_date": today,
|
||||
"session_phase": session_phase(now_shanghai(), is_open),
|
||||
"is_open_day": is_open,
|
||||
"eod_status": self.scheduler.eod_status(today),
|
||||
"revision_status": self.scheduler.revision_status(today),
|
||||
"publications": pubs,
|
||||
"anomalies": failed,
|
||||
"recent_calls": _public_calls(calls),
|
||||
@@ -55,10 +58,26 @@ class AdminAPI:
|
||||
}
|
||||
]
|
||||
for name, adapter in RESERVED.items():
|
||||
if name == "ifind":
|
||||
live = self.ifind or adapter
|
||||
cred = self.auth.credential_status("ifind_refresh_token") or {
|
||||
"configured": bool(getattr(live, "configured", False)),
|
||||
"last4": "",
|
||||
"updated_at": "",
|
||||
}
|
||||
items.append(
|
||||
{
|
||||
"provider": name,
|
||||
"role": "reserved",
|
||||
"role": "licensed",
|
||||
"health": live.probe(),
|
||||
"credential": cred,
|
||||
}
|
||||
)
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"provider": name,
|
||||
"role": "reserved" if name in {"ths", "xgb", "akshare"} else "free",
|
||||
"health": adapter.probe(),
|
||||
"credential": {"configured": False, "last4": "", "updated_at": ""},
|
||||
}
|
||||
@@ -76,6 +95,11 @@ class AdminAPI:
|
||||
def probe(self, provider: str) -> dict[str, Any]:
|
||||
if provider == "tushare":
|
||||
return self.pipeline.adapter.probe()
|
||||
if provider == "ifind":
|
||||
adapter = self.ifind or RESERVED.get("ifind")
|
||||
if adapter is None:
|
||||
raise ApiError("INVALID_ARGUMENT", "unknown provider: ifind")
|
||||
return adapter.probe()
|
||||
adapter = RESERVED.get(provider)
|
||||
if adapter is None:
|
||||
raise ApiError("INVALID_ARGUMENT", f"unknown provider: {provider}")
|
||||
@@ -83,11 +107,16 @@ class AdminAPI:
|
||||
|
||||
def jobs(self) -> dict[str, Any]:
|
||||
runs = self.db.fetchall("SELECT * FROM job_runs ORDER BY id DESC LIMIT 100")
|
||||
stocks_times = "/".join(self.pipeline.settings.stocks_refresh_times) or "20:00"
|
||||
return {
|
||||
"jobs": [
|
||||
{"id": "precheck", "at": "08:45", "title": "盘前预检"},
|
||||
{"id": "eod_a", "at": "15:05", "title": "盘后批 A daily/valuation/moneyflow/auction"},
|
||||
{"id": "eod_b", "at": "15:10", "title": "盘后批 B index_daily"},
|
||||
{"id": "eod_retry", "at": "15:15-23:30", "title": "盘后未出数自动重试(每 30 分钟,成功即停)"},
|
||||
{"id": "eod_revise", "at": "20:00-23:20", "title": "估值发布后复核(轻量比对,有修订才整组原子追补)"},
|
||||
{"id": "stocks_refresh", "at": stocks_times, "title": "股票主档刷新与正式发布(新上市/更名,无变化跳过)"},
|
||||
{"id": "history_backfill", "at": "manual", "title": "回补历史日历、个股日 K 与指数日 K"},
|
||||
{"id": "cleanup", "at": "00:30", "title": "清理 staging / 日志"},
|
||||
{"id": "backup", "at": "00:40", "title": "SQLite 备份"},
|
||||
],
|
||||
@@ -130,12 +159,31 @@ class AdminAPI:
|
||||
return result
|
||||
|
||||
def backfill(self, dataset: str, trade_date: str, password: str, confirm: str, actor: str) -> dict[str, Any]:
|
||||
self._dangerous(password, confirm, f"{dataset}:{trade_date}")
|
||||
if dataset == "reference":
|
||||
result = self.pipeline.ingest_reference(trade_date)
|
||||
day = yyyymmdd(trade_date or now_shanghai())
|
||||
if dataset == "history":
|
||||
self._dangerous(password, confirm, "history:full")
|
||||
result = self.pipeline.backfill_history(day)
|
||||
else:
|
||||
result = self.pipeline.run_dataset(dataset, trade_date)
|
||||
self.pipeline.audit(actor, "backfill", f"{dataset}:{trade_date}", json.dumps({"ok": True}))
|
||||
self._dangerous(password, confirm, f"{dataset}:{day}")
|
||||
if dataset == "reference":
|
||||
result = self.pipeline.ingest_reference(day)
|
||||
elif dataset in OFFICIAL_DATASETS or dataset == STOCKS_DATASET:
|
||||
# Manual same-day republish must rebuild the full A/B boundary.
|
||||
# Gate failures and mid-switch exceptions both surface as
|
||||
# FAILED_PRECONDITION so the admin API never leaks raw
|
||||
# transaction errors to the client.
|
||||
try:
|
||||
result = self.pipeline.force_republish_boundary(dataset, day)
|
||||
failures = self.pipeline.eod_failures(result)
|
||||
if failures:
|
||||
raise ApiError("FAILED_PRECONDITION", "; ".join(failures))
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise ApiError("FAILED_PRECONDITION", str(exc)) from exc
|
||||
else:
|
||||
raise ApiError("INVALID_ARGUMENT", f"unsupported backfill dataset: {dataset}")
|
||||
self.pipeline.audit(actor, "backfill", f"{dataset}:{day}", json.dumps({"ok": True}))
|
||||
return result
|
||||
|
||||
def _dangerous(self, password: str, confirm: str, expected: str) -> None:
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Command-line entry for one-shot datahub operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
|
||||
from datahub.hub import build_hub
|
||||
from datahub.pipeline import EOD_A_DATASETS, OFFICIAL_DATASETS, STOCKS_DATASET
|
||||
from datahub.settings import load_settings
|
||||
from datahub.timeutil import yyyymmdd
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="xiaobai-datahub CLI")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
history = sub.add_parser("history-backfill", help="回补交易日历、个股日 K(默认 250 日)和网站所用指数日 K")
|
||||
history.add_argument("--calendar-start", default=None, help="日历起点,默认配置 calendar_start")
|
||||
history.add_argument("--index-days", type=int, default=None, help="指数回补交易日数量,默认 260")
|
||||
history.add_argument("--daily-days", type=int, default=None, help="个股日 K 回补交易日数量,默认 250")
|
||||
history.add_argument("--force", action="store_true", help="覆盖已发布的个股日 K / 指数日期")
|
||||
refresh = sub.add_parser("eod-refresh", help="对指定交易日补跑盘后正式数据(跳过已完整发布的一致性边界,仍走质量门禁)")
|
||||
refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD,默认今天")
|
||||
refresh.add_argument(
|
||||
"--force", action="store_true",
|
||||
help="强制重发 --dataset 所属的完整一致性边界(A 组或 B 组),生成新批次并保留上一批次可回滚",
|
||||
)
|
||||
refresh.add_argument(
|
||||
"--dataset", default=None,
|
||||
help="配合 --force:指定边界内任一成员(如 valuation→整组 A;index_daily→整组 B)",
|
||||
)
|
||||
stocks_refresh = sub.add_parser("stocks-refresh", help="刷新股票主档并发布正式快照(幂等:无变化则跳过)")
|
||||
stocks_refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD,默认今天")
|
||||
stocks_refresh.add_argument("--force", action="store_true", help="即使快照无变化也重新发布")
|
||||
moneyflow_backfill = sub.add_parser(
|
||||
"moneyflow-backfill", help="回补资金流历史(默认覆盖网站所需的最近 N 个交易日,跳过已发布日期)",
|
||||
)
|
||||
moneyflow_backfill.add_argument("--end-date", default=None, help="截止交易日 YYYYMMDD,默认今天")
|
||||
moneyflow_backfill.add_argument("--trading-days", type=int, default=None, help="回补交易日数量,默认配置 moneyflow_history_trading_days")
|
||||
moneyflow_backfill.add_argument("--force", action="store_true", help="覆盖已发布的资金流日期")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
settings = load_settings()
|
||||
hub = build_hub(settings)
|
||||
if args.command == "history-backfill":
|
||||
result = hub.pipeline.backfill_history(
|
||||
calendar_start=args.calendar_start,
|
||||
index_days=args.index_days,
|
||||
daily_days=args.daily_days,
|
||||
force=args.force,
|
||||
)
|
||||
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, default=str)
|
||||
sys.stdout.write("\n")
|
||||
return 0 if result.get("ok") else 1
|
||||
if args.command == "eod-refresh":
|
||||
day = yyyymmdd(args.trade_date) if args.trade_date else yyyymmdd()
|
||||
if args.force:
|
||||
allowed = set(OFFICIAL_DATASETS) | {STOCKS_DATASET}
|
||||
if not args.dataset:
|
||||
parser.error("--force requires --dataset (e.g. --dataset valuation)")
|
||||
if args.dataset not in allowed:
|
||||
parser.error(f"unknown dataset: {args.dataset}")
|
||||
result = hub.pipeline.force_republish_boundary(args.dataset, day)
|
||||
boundary = "A" if args.dataset in EOD_A_DATASETS or args.dataset == STOCKS_DATASET else "B"
|
||||
else:
|
||||
result = hub.pipeline.run_eod_missing(day)
|
||||
boundary = None
|
||||
hub.pipeline.audit("cli", "eod-refresh", f"eod:{day}", json.dumps(
|
||||
{"force": bool(args.force), "dataset": args.dataset, "boundary": boundary,
|
||||
**{name: item.get("state") for name, item in result.items() if isinstance(item, dict)}},
|
||||
ensure_ascii=False,
|
||||
))
|
||||
if args.force:
|
||||
failures = hub.pipeline.eod_failures(result)
|
||||
payload = {"trade_date": day, "boundary": boundary, "datasets": result}
|
||||
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, default=str)
|
||||
sys.stdout.write("\n")
|
||||
return 0 if not failures else 1
|
||||
missing = hub.pipeline.missing_official_datasets(day)
|
||||
payload = {"trade_date": day, "datasets": result, "missing_after": missing}
|
||||
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, default=str)
|
||||
sys.stdout.write("\n")
|
||||
return 0 if not missing else 1
|
||||
if args.command == "stocks-refresh":
|
||||
day = yyyymmdd(args.trade_date) if args.trade_date else yyyymmdd()
|
||||
result = hub.pipeline.refresh_stocks(day, force=args.force)
|
||||
hub.pipeline.audit("cli", "stocks-refresh", f"stocks:{day}", json.dumps(
|
||||
{"force": bool(args.force), "state": result.get("state"), "batch_id": result.get("batch_id")},
|
||||
ensure_ascii=False,
|
||||
))
|
||||
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, default=str)
|
||||
sys.stdout.write("\n")
|
||||
return 0 if result.get("state") != "failed" else 1
|
||||
if args.command == "moneyflow-backfill":
|
||||
result = hub.pipeline.backfill_moneyflow_history(
|
||||
end_date=args.end_date,
|
||||
trading_days=args.trading_days,
|
||||
force=args.force,
|
||||
)
|
||||
hub.pipeline.audit("cli", "moneyflow-backfill", f"moneyflow:{result.get('end')}", json.dumps(
|
||||
{"published": len(result.get("published") or []), "skipped": len(result.get("skipped") or []),
|
||||
"failed": len(result.get("failed") or [])},
|
||||
ensure_ascii=False,
|
||||
))
|
||||
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, default=str)
|
||||
sys.stdout.write("\n")
|
||||
return 0 if result.get("ok") else 1
|
||||
parser.error(f"unknown command: {args.command}")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,130 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Iterable
|
||||
|
||||
from datahub.db import HubDB
|
||||
from datahub.timeutil import iter_yyyymmdd, yyyymmdd
|
||||
|
||||
MISSING_SAMPLE_LIMIT = 10
|
||||
|
||||
|
||||
def coverage_payload(
|
||||
*,
|
||||
kind: str,
|
||||
start: str,
|
||||
end: str,
|
||||
expected: Iterable[str],
|
||||
available: Iterable[str],
|
||||
extra: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
start = yyyymmdd(start)
|
||||
end = yyyymmdd(end)
|
||||
expected_list = sorted({yyyymmdd(item) for item in expected if item})
|
||||
available_set = {yyyymmdd(item) for item in available if item}
|
||||
missing = [item for item in expected_list if item not in available_set]
|
||||
payload: dict[str, Any] = {
|
||||
"kind": kind,
|
||||
"complete": not missing,
|
||||
"requested_from": start,
|
||||
"requested_to": end,
|
||||
"available_from": min(available_set) if available_set else None,
|
||||
"available_to": max(available_set) if available_set else None,
|
||||
"expected_count": len(expected_list),
|
||||
"available_count": len(available_set),
|
||||
"missing_count": len(missing),
|
||||
"missing_sample": missing[:MISSING_SAMPLE_LIMIT],
|
||||
}
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
return payload
|
||||
|
||||
|
||||
def calendar_coverage(db: HubDB, start: str, end: str, exchange: str = "SSE") -> dict[str, Any]:
|
||||
start = yyyymmdd(start)
|
||||
end = yyyymmdd(end)
|
||||
expected = list(iter_yyyymmdd(start, end))
|
||||
rows = db.fetchall(
|
||||
"SELECT cal_date FROM trade_calendar WHERE exchange = ? AND cal_date >= ? AND cal_date <= ?",
|
||||
(exchange, start, end),
|
||||
)
|
||||
return coverage_payload(
|
||||
kind="calendar",
|
||||
start=start,
|
||||
end=end,
|
||||
expected=expected,
|
||||
available=(row["cal_date"] for row in rows),
|
||||
extra={"exchange": exchange},
|
||||
)
|
||||
|
||||
|
||||
def published_range_coverage(
|
||||
db: HubDB,
|
||||
dataset: str,
|
||||
start: str,
|
||||
end: str,
|
||||
ts_code: str = "",
|
||||
table: str = "",
|
||||
) -> dict[str, Any]:
|
||||
start = yyyymmdd(start)
|
||||
end = yyyymmdd(end)
|
||||
calendar = calendar_coverage(db, start, end)
|
||||
open_rows = db.fetchall(
|
||||
"""
|
||||
SELECT cal_date FROM trade_calendar
|
||||
WHERE exchange = 'SSE' AND is_open = 1 AND cal_date >= ? AND cal_date <= ?
|
||||
ORDER BY cal_date
|
||||
""",
|
||||
(start, end),
|
||||
)
|
||||
expected_open = [row["cal_date"] for row in open_rows]
|
||||
pubs = db.fetchall(
|
||||
"""
|
||||
SELECT trade_date, active_batch FROM publications
|
||||
WHERE dataset = ? AND trade_date >= ? AND trade_date <= ?
|
||||
ORDER BY trade_date
|
||||
""",
|
||||
(dataset, start, end),
|
||||
)
|
||||
published_dates = [row["trade_date"] for row in pubs]
|
||||
available = list(published_dates)
|
||||
extra: dict[str, Any] = {
|
||||
"dataset": dataset,
|
||||
"calendar_complete": calendar["complete"],
|
||||
"calendar_missing_count": calendar["missing_count"],
|
||||
}
|
||||
if ts_code and table and pubs:
|
||||
present_code: list[str] = []
|
||||
for pub in pubs:
|
||||
hit = db.fetchone(
|
||||
f"SELECT 1 AS ok FROM {table} WHERE trade_date = ? AND batch_id = ? AND ts_code = ? LIMIT 1",
|
||||
(pub["trade_date"], pub["active_batch"], ts_code),
|
||||
)
|
||||
if hit:
|
||||
present_code.append(pub["trade_date"])
|
||||
available = present_code
|
||||
extra["code"] = ts_code
|
||||
payload = coverage_payload(
|
||||
kind="published_range",
|
||||
start=start,
|
||||
end=end,
|
||||
expected=expected_open,
|
||||
available=available,
|
||||
extra=extra,
|
||||
)
|
||||
if not calendar["complete"]:
|
||||
payload["complete"] = False
|
||||
payload["calendar_missing_sample"] = calendar["missing_sample"]
|
||||
return payload
|
||||
|
||||
|
||||
def point_coverage(trade_date: str, dataset: str = "") -> dict[str, Any]:
|
||||
day = yyyymmdd(trade_date)
|
||||
payload = coverage_payload(
|
||||
kind="point",
|
||||
start=day,
|
||||
end=day,
|
||||
expected=[day],
|
||||
available=[day],
|
||||
extra={"dataset": dataset} if dataset else None,
|
||||
)
|
||||
return payload
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Extended EOD datasets beyond the first-batch A/B release groups.
|
||||
|
||||
These publish independently (soft): a failure here must not block daily/valuation
|
||||
release. Scheduler runs them after the core EOD window.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
# Independent soft datasets (not part of A/B atomic groups).
|
||||
EXTENDED_SOFT_DATASETS = {
|
||||
"limit_events",
|
||||
"popularity",
|
||||
"dragon_tiger",
|
||||
"sector_daily",
|
||||
}
|
||||
|
||||
EXTENDED_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS eod_limit_events (
|
||||
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, limit_type TEXT NOT NULL,
|
||||
name TEXT, industry TEXT, close REAL, pct_chg REAL, amount REAL,
|
||||
limit_amount REAL, float_mv REAL, total_mv REAL, turnover_ratio REAL,
|
||||
fd_amount REAL, first_time TEXT, last_time TEXT,
|
||||
open_times INTEGER, up_stat TEXT, limit_times INTEGER,
|
||||
batch_id TEXT NOT NULL,
|
||||
PRIMARY KEY (ts_code, trade_date, limit_type, batch_id)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS staging_limit_events (
|
||||
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, limit_type TEXT NOT NULL, batch_id TEXT NOT NULL,
|
||||
name TEXT, industry TEXT, close REAL, pct_chg REAL, amount REAL,
|
||||
limit_amount REAL, float_mv REAL, total_mv REAL, turnover_ratio REAL,
|
||||
fd_amount REAL, first_time TEXT, last_time TEXT,
|
||||
open_times INTEGER, up_stat TEXT, limit_times INTEGER,
|
||||
PRIMARY KEY (batch_id, ts_code, trade_date, limit_type)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eod_popularity (
|
||||
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, source TEXT NOT NULL,
|
||||
ts_name TEXT, rank INTEGER, pct_change REAL, current_price REAL,
|
||||
hot REAL, concept TEXT, data_type TEXT,
|
||||
batch_id TEXT NOT NULL,
|
||||
PRIMARY KEY (ts_code, trade_date, source, batch_id)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS staging_popularity (
|
||||
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, source TEXT NOT NULL, batch_id TEXT NOT NULL,
|
||||
ts_name TEXT, rank INTEGER, pct_change REAL, current_price REAL,
|
||||
hot REAL, concept TEXT, data_type TEXT,
|
||||
PRIMARY KEY (batch_id, ts_code, trade_date, source)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eod_dragon_tiger (
|
||||
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, hm_name TEXT NOT NULL,
|
||||
ts_name TEXT, buy_amount REAL, sell_amount REAL, net_amount REAL,
|
||||
hm_orgs TEXT, tag TEXT, pct_change REAL, reason TEXT,
|
||||
batch_id TEXT NOT NULL,
|
||||
PRIMARY KEY (ts_code, trade_date, hm_name, batch_id)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS staging_dragon_tiger (
|
||||
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, hm_name TEXT NOT NULL, batch_id TEXT NOT NULL,
|
||||
ts_name TEXT, buy_amount REAL, sell_amount REAL, net_amount REAL,
|
||||
hm_orgs TEXT, tag TEXT, pct_change REAL, reason TEXT,
|
||||
PRIMARY KEY (batch_id, ts_code, trade_date, hm_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eod_sector_daily (
|
||||
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, family TEXT NOT NULL,
|
||||
name TEXT, open REAL, high REAL, low REAL, close REAL, pre_close REAL,
|
||||
pct_change REAL, vol REAL, turnover_rate REAL, amount REAL,
|
||||
batch_id TEXT NOT NULL,
|
||||
PRIMARY KEY (ts_code, trade_date, family, batch_id)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS staging_sector_daily (
|
||||
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, family TEXT NOT NULL, batch_id TEXT NOT NULL,
|
||||
name TEXT, open REAL, high REAL, low REAL, close REAL, pre_close REAL,
|
||||
pct_change REAL, vol REAL, turnover_rate REAL, amount REAL,
|
||||
PRIMARY KEY (batch_id, ts_code, trade_date, family)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sector_master (
|
||||
ts_code TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
family TEXT NOT NULL,
|
||||
exchange TEXT,
|
||||
list_date TEXT,
|
||||
member_count INTEGER,
|
||||
type TEXT,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_eod_limit_date ON eod_limit_events(trade_date, batch_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_eod_pop_date ON eod_popularity(trade_date, batch_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_eod_lhb_date ON eod_dragon_tiger(trade_date, batch_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_eod_sector_date ON eod_sector_daily(trade_date, family, batch_id);
|
||||
"""
|
||||
|
||||
EXTENDED_DATASET_TABLES = {
|
||||
"limit_events": ("eod_limit_events", "staging_limit_events"),
|
||||
"popularity": ("eod_popularity", "staging_popularity"),
|
||||
"dragon_tiger": ("eod_dragon_tiger", "staging_dragon_tiger"),
|
||||
"sector_daily": ("eod_sector_daily", "staging_sector_daily"),
|
||||
}
|
||||
|
||||
EXTENDED_STAGING_INSERT: dict[str, tuple[str, Any]] = {
|
||||
"limit_events": (
|
||||
"INSERT INTO staging_limit_events("
|
||||
"ts_code,trade_date,limit_type,batch_id,name,industry,close,pct_chg,amount,"
|
||||
"limit_amount,float_mv,total_mv,turnover_ratio,fd_amount,first_time,last_time,"
|
||||
"open_times,up_stat,limit_times) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
lambda r, b: (
|
||||
r["ts_code"], r["trade_date"], r["limit_type"], b,
|
||||
r.get("name"), r.get("industry"), r.get("close"), r.get("pct_chg"), r.get("amount"),
|
||||
r.get("limit_amount"), r.get("float_mv"), r.get("total_mv"), r.get("turnover_ratio"),
|
||||
r.get("fd_amount"), r.get("first_time"), r.get("last_time"),
|
||||
r.get("open_times"), r.get("up_stat"), r.get("limit_times"),
|
||||
),
|
||||
),
|
||||
"popularity": (
|
||||
"INSERT INTO staging_popularity("
|
||||
"ts_code,trade_date,source,batch_id,ts_name,rank,pct_change,current_price,hot,concept,data_type) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
lambda r, b: (
|
||||
r["ts_code"], r["trade_date"], r["source"], b,
|
||||
r.get("ts_name"), r.get("rank"), r.get("pct_change"), r.get("current_price"),
|
||||
r.get("hot"), r.get("concept"), r.get("data_type"),
|
||||
),
|
||||
),
|
||||
"dragon_tiger": (
|
||||
"INSERT INTO staging_dragon_tiger("
|
||||
"ts_code,trade_date,hm_name,batch_id,ts_name,buy_amount,sell_amount,net_amount,"
|
||||
"hm_orgs,tag,pct_change,reason) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
lambda r, b: (
|
||||
r["ts_code"], r["trade_date"], r["hm_name"], b,
|
||||
r.get("ts_name"), r.get("buy_amount"), r.get("sell_amount"), r.get("net_amount"),
|
||||
r.get("hm_orgs"), r.get("tag"), r.get("pct_change"), r.get("reason"),
|
||||
),
|
||||
),
|
||||
"sector_daily": (
|
||||
"INSERT INTO staging_sector_daily("
|
||||
"ts_code,trade_date,family,batch_id,name,open,high,low,close,pre_close,"
|
||||
"pct_change,vol,turnover_rate,amount) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
lambda r, b: (
|
||||
r["ts_code"], r["trade_date"], r["family"], b,
|
||||
r.get("name"), r.get("open"), r.get("high"), r.get("low"), r.get("close"),
|
||||
r.get("pre_close"), r.get("pct_change"), r.get("vol"), r.get("turnover_rate"),
|
||||
r.get("amount"),
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
EXTENDED_EOD_COPY = {
|
||||
"limit_events": (
|
||||
"INSERT OR REPLACE INTO eod_limit_events "
|
||||
"SELECT ts_code,trade_date,limit_type,name,industry,close,pct_chg,amount,"
|
||||
"limit_amount,float_mv,total_mv,turnover_ratio,fd_amount,first_time,last_time,"
|
||||
"open_times,up_stat,limit_times,batch_id "
|
||||
"FROM staging_limit_events WHERE batch_id = ?"
|
||||
),
|
||||
"popularity": (
|
||||
"INSERT OR REPLACE INTO eod_popularity "
|
||||
"SELECT ts_code,trade_date,source,ts_name,rank,pct_change,current_price,hot,concept,data_type,batch_id "
|
||||
"FROM staging_popularity WHERE batch_id = ?"
|
||||
),
|
||||
"dragon_tiger": (
|
||||
"INSERT OR REPLACE INTO eod_dragon_tiger "
|
||||
"SELECT ts_code,trade_date,hm_name,ts_name,buy_amount,sell_amount,net_amount,"
|
||||
"hm_orgs,tag,pct_change,reason,batch_id "
|
||||
"FROM staging_dragon_tiger WHERE batch_id = ?"
|
||||
),
|
||||
"sector_daily": (
|
||||
"INSERT OR REPLACE INTO eod_sector_daily "
|
||||
"SELECT ts_code,trade_date,family,name,open,high,low,close,pre_close,"
|
||||
"pct_change,vol,turnover_rate,amount,batch_id "
|
||||
"FROM staging_sector_daily WHERE batch_id = ?"
|
||||
),
|
||||
}
|
||||
@@ -7,9 +7,10 @@ from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from datahub.datasets_ext import EXTENDED_DATASET_TABLES, EXTENDED_SCHEMA
|
||||
from datahub.timeutil import isoformat
|
||||
|
||||
SCHEMA = """
|
||||
_BASE_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version INTEGER PRIMARY KEY,
|
||||
applied_at TEXT NOT NULL
|
||||
@@ -114,6 +115,21 @@ CREATE TABLE IF NOT EXISTS eod_index_bars (
|
||||
PRIMARY KEY (ts_code, trade_date, batch_id)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eod_stocks (
|
||||
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL,
|
||||
symbol TEXT, name TEXT, area TEXT, industry TEXT, market TEXT,
|
||||
list_status TEXT, list_date TEXT,
|
||||
batch_id TEXT NOT NULL,
|
||||
PRIMARY KEY (ts_code, trade_date, batch_id)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS staging_stocks (
|
||||
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, batch_id TEXT NOT NULL,
|
||||
symbol TEXT, name TEXT, area TEXT, industry TEXT, market TEXT,
|
||||
list_status TEXT, list_date TEXT,
|
||||
PRIMARY KEY (batch_id, ts_code, trade_date)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS staging_bars (
|
||||
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, batch_id TEXT NOT NULL,
|
||||
open REAL, high REAL, low REAL, close REAL, pct_chg REAL,
|
||||
@@ -212,6 +228,30 @@ CREATE TABLE IF NOT EXISTS job_runs (
|
||||
detail TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS eod_progress (
|
||||
trade_date TEXT PRIMARY KEY,
|
||||
state TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_attempt_at TEXT,
|
||||
next_retry_at TEXT,
|
||||
finished_at TEXT,
|
||||
detail TEXT,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS revision_progress (
|
||||
trade_date TEXT PRIMARY KEY,
|
||||
state TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_attempt_at TEXT,
|
||||
next_retry_at TEXT,
|
||||
finished_at TEXT,
|
||||
catchup_done INTEGER NOT NULL DEFAULT 0,
|
||||
last_diff TEXT,
|
||||
detail TEXT,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS audit_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
actor TEXT NOT NULL,
|
||||
@@ -256,12 +296,16 @@ CREATE INDEX IF NOT EXISTS idx_eod_bars_date ON eod_bars(trade_date, batch_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_calendar_open ON trade_calendar(is_open, cal_date);
|
||||
"""
|
||||
|
||||
SCHEMA = _BASE_SCHEMA + EXTENDED_SCHEMA
|
||||
|
||||
DATASET_TABLES = {
|
||||
"daily": ("eod_bars", "staging_bars"),
|
||||
"valuation": ("eod_valuation", "staging_valuation"),
|
||||
"moneyflow": ("eod_moneyflow", "staging_moneyflow"),
|
||||
"auction": ("eod_auction", "staging_auction"),
|
||||
"index_daily": ("eod_index_bars", "staging_index_bars"),
|
||||
"stocks": ("eod_stocks", "staging_stocks"),
|
||||
**EXTENDED_DATASET_TABLES,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
self._json({"status": "ok"}, HTTPStatus.OK)
|
||||
return
|
||||
if path.startswith("/v1/"):
|
||||
self._v1(path, parsed.query)
|
||||
self._v1(path, parsed.query, method)
|
||||
return
|
||||
if path.startswith("/admin/api/"):
|
||||
self._admin_api(method, path)
|
||||
@@ -66,11 +66,21 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
LOGGER.exception("internal error")
|
||||
self._json({"error": {"code": "INTERNAL", "message": "internal error"}}, HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
|
||||
def _v1(self, path: str, query: str) -> None:
|
||||
def _v1(self, path: str, query: str, method: str = "GET") -> None:
|
||||
token = self.headers.get("X-Datahub-Token", "")
|
||||
if not self.hub.auth.check_api_token(token):
|
||||
self.hub.pipeline.audit("anonymous", "unauthorized", path, "")
|
||||
raise ApiError("UNAUTHORIZED", "missing or invalid X-Datahub-Token")
|
||||
if path == "/v1/query" and method == "POST":
|
||||
body = self._read_json(max_bytes=1_000_000)
|
||||
payload = self.hub.api.query_api(body)
|
||||
self._json(payload, HTTPStatus.OK)
|
||||
return
|
||||
if path == "/v1/credentials/ifind" and method == "POST":
|
||||
body = self._read_json()
|
||||
payload = self.hub.put_ifind_credentials(body)
|
||||
self._json(payload, HTTPStatus.OK)
|
||||
return
|
||||
payload = self.hub.api.handle(path, parse_query(query))
|
||||
self._json(payload, HTTPStatus.OK)
|
||||
|
||||
@@ -184,11 +194,11 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
self.end_headers()
|
||||
self.wfile.write(content)
|
||||
|
||||
def _read_json(self, allow_empty: bool = False) -> dict[str, Any]:
|
||||
def _read_json(self, allow_empty: bool = False, max_bytes: int = 65536) -> dict[str, Any]:
|
||||
length = int(self.headers.get("Content-Length", "0") or 0)
|
||||
if length == 0 and allow_empty:
|
||||
return {}
|
||||
if length <= 0 or length > 65536:
|
||||
if length <= 0 or length > max_bytes:
|
||||
raise ValueError("请求内容为空或过大")
|
||||
raw = self.rfile.read(length)
|
||||
try:
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from datahub.adapters.ifind import IfindAdapter
|
||||
from datahub.adapters.tushare import TushareAdapter
|
||||
from datahub.admin_api import AdminAPI
|
||||
from datahub.auth import AuthService
|
||||
@@ -28,7 +30,16 @@ class Hub:
|
||||
if settings.tushare_token:
|
||||
self.auth.store_credential("tushare_token", settings.tushare_token)
|
||||
token = settings.tushare_token
|
||||
refresh = settings.ifind_refresh_token or self.auth.load_credential("ifind_refresh_token") or ""
|
||||
access = settings.ifind_access_token or self.auth.load_credential("ifind_access_token") or ""
|
||||
if settings.ifind_refresh_token:
|
||||
self.auth.store_credential("ifind_refresh_token", settings.ifind_refresh_token)
|
||||
refresh = settings.ifind_refresh_token
|
||||
if settings.ifind_access_token:
|
||||
self.auth.store_credential("ifind_access_token", settings.ifind_access_token)
|
||||
access = settings.ifind_access_token
|
||||
self.adapter = adapter or TushareAdapter(token)
|
||||
self.ifind = IfindAdapter(refresh, access)
|
||||
self.pipeline = Pipeline(
|
||||
self.db,
|
||||
self.adapter,
|
||||
@@ -38,10 +49,21 @@ class Hub:
|
||||
)
|
||||
self.lkg = LastKnownGood(self.db)
|
||||
self.scheduler = Scheduler(self.db, self.pipeline)
|
||||
self.api = V1API(self.db, self.pipeline, settings)
|
||||
self.admin = AdminAPI(self.db, self.pipeline, self.scheduler, self.auth)
|
||||
self.api = V1API(self.db, self.pipeline, settings, ifind=self.ifind)
|
||||
self.admin = AdminAPI(self.db, self.pipeline, self.scheduler, self.auth, ifind=self.ifind)
|
||||
self.static_dir = Path(__file__).resolve().parents[1] / "admin"
|
||||
|
||||
def put_ifind_credentials(self, body: dict[str, Any] | None) -> dict[str, Any]:
|
||||
from datahub.serving import envelope
|
||||
|
||||
payload = dict(body or {})
|
||||
refresh = str(payload.get("ifind_refresh_token") or "").strip()
|
||||
access = str(payload.get("ifind_access_token") or "").strip()
|
||||
self.auth.store_credential("ifind_refresh_token", refresh)
|
||||
self.auth.store_credential("ifind_access_token", access)
|
||||
self.ifind.set_credentials(refresh, access)
|
||||
return envelope(self.ifind.status(), {"source": "ifind"})
|
||||
|
||||
def start(self) -> None:
|
||||
if self.settings.scheduler_enabled:
|
||||
self.scheduler.start()
|
||||
|
||||
@@ -12,6 +12,7 @@ from datahub.timeutil import isoformat
|
||||
_SECRET_KEYS = (
|
||||
"token", "password", "secret", "key", "authorization", "credential",
|
||||
"tushare_token", "datahub_token", "encryption_key", "cookie",
|
||||
"ifind", "refresh_token", "access_token",
|
||||
)
|
||||
_SECRET_JSON = re.compile(
|
||||
r'(?i)("(?:' + "|".join(re.escape(key) for key in _SECRET_KEYS) + r')"\s*:\s*")([^"\\]*(?:\\.[^"\\]*)*)(")'
|
||||
|
||||
@@ -156,6 +156,95 @@ def normalize_stock(row: dict[str, Any]) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def normalize_limit_event(row: dict[str, Any]) -> dict[str, Any]:
|
||||
"""limit_list_d. float_mv/total_mv/limit_amount are 万元 → yuan; amount/fd_amount already yuan."""
|
||||
return {
|
||||
"ts_code": _code(row.get("ts_code")),
|
||||
"trade_date": _date(row.get("trade_date")),
|
||||
"limit_type": str(row.get("limit_type") or "").strip().upper() or "U",
|
||||
"name": str(row.get("name") or "").strip() or None,
|
||||
"industry": str(row.get("industry") or "").strip() or None,
|
||||
"close": round4(finite_number(row.get("close"))),
|
||||
"pct_chg": round4(finite_number(row.get("pct_chg"))),
|
||||
"amount": round4(finite_number(row.get("amount"))),
|
||||
"limit_amount": round4(_scale(row.get("limit_amount"), AMOUNT_WAN_YUAN)),
|
||||
"float_mv": round4(_scale(row.get("float_mv"), AMOUNT_WAN_YUAN)),
|
||||
"total_mv": round4(_scale(row.get("total_mv"), AMOUNT_WAN_YUAN)),
|
||||
"turnover_ratio": round4(finite_number(row.get("turnover_ratio"))),
|
||||
"fd_amount": round4(finite_number(row.get("fd_amount"))),
|
||||
"first_time": str(row.get("first_time") or "").strip() or None,
|
||||
"last_time": str(row.get("last_time") or "").strip() or None,
|
||||
"open_times": _optional_int(row.get("open_times")),
|
||||
"up_stat": str(row.get("up_stat") or "").strip() or None,
|
||||
"limit_times": _optional_int(row.get("limit_times")),
|
||||
}
|
||||
|
||||
|
||||
def normalize_popularity(row: dict[str, Any], source: str = "") -> dict[str, Any]:
|
||||
src = str(source or row.get("source") or "").strip().lower() or "ths"
|
||||
return {
|
||||
"ts_code": _code(row.get("ts_code")),
|
||||
"trade_date": _date(row.get("trade_date")),
|
||||
"source": src,
|
||||
"ts_name": str(row.get("ts_name") or row.get("name") or "").strip() or None,
|
||||
"rank": _optional_int(row.get("rank")),
|
||||
"pct_change": round4(
|
||||
finite_number(row.get("pct_change") if row.get("pct_change") is not None else row.get("pct_chg"))
|
||||
),
|
||||
"current_price": round4(finite_number(row.get("current_price") or row.get("price"))),
|
||||
"hot": round4(finite_number(row.get("hot"))),
|
||||
"concept": str(row.get("concept") or "").strip() or None,
|
||||
"data_type": str(row.get("data_type") or "").strip() or None,
|
||||
}
|
||||
|
||||
|
||||
def normalize_dragon_tiger(row: dict[str, Any]) -> dict[str, Any]:
|
||||
"""hm_detail amounts are 万元 → yuan."""
|
||||
return {
|
||||
"ts_code": _code(row.get("ts_code")),
|
||||
"trade_date": _date(row.get("trade_date")),
|
||||
"hm_name": str(row.get("hm_name") or "未命名游资").strip() or "未命名游资",
|
||||
"ts_name": str(row.get("ts_name") or row.get("name") or "").strip() or None,
|
||||
"buy_amount": round4(_scale(row.get("buy_amount"), AMOUNT_WAN_YUAN)),
|
||||
"sell_amount": round4(_scale(row.get("sell_amount"), AMOUNT_WAN_YUAN)),
|
||||
"net_amount": round4(_scale(row.get("net_amount"), AMOUNT_WAN_YUAN)),
|
||||
"hm_orgs": str(row.get("hm_orgs") or "").strip() or None,
|
||||
"tag": str(row.get("tag") or "").strip() or None,
|
||||
"pct_change": round4(finite_number(row.get("pct_change"))),
|
||||
"reason": str(row.get("reason") or "").strip() or None,
|
||||
}
|
||||
|
||||
|
||||
def normalize_sector_daily(row: dict[str, Any], family: str = "ths") -> dict[str, Any]:
|
||||
fam = str(family or row.get("family") or "ths").strip().lower()
|
||||
return {
|
||||
"ts_code": _code(row.get("ts_code")),
|
||||
"trade_date": _date(row.get("trade_date")),
|
||||
"family": fam,
|
||||
"name": str(row.get("name") or "").strip() or None,
|
||||
"open": round4(finite_number(row.get("open"))),
|
||||
"high": round4(finite_number(row.get("high"))),
|
||||
"low": round4(finite_number(row.get("low"))),
|
||||
"close": round4(finite_number(row.get("close"))),
|
||||
"pre_close": round4(finite_number(row.get("pre_close"))),
|
||||
"pct_change": round4(
|
||||
finite_number(row.get("pct_change") if row.get("pct_change") is not None else row.get("pct_chg"))
|
||||
),
|
||||
"vol": round4(finite_number(row.get("vol"))),
|
||||
"turnover_rate": round4(finite_number(row.get("turnover_rate"))),
|
||||
"amount": round4(finite_number(row.get("amount"))),
|
||||
}
|
||||
|
||||
|
||||
def _optional_int(value: Any) -> int | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
try:
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def apply_qfq(price: float | None, factor: float | None, latest_factor: float | None) -> float | None:
|
||||
if price is None:
|
||||
return None
|
||||
@@ -184,6 +273,11 @@ NORMALIZERS = {
|
||||
"calendar": normalize_calendar,
|
||||
"stock_basic": normalize_stock,
|
||||
"stocks": normalize_stock,
|
||||
"limit_events": normalize_limit_event,
|
||||
"limit_list_d": normalize_limit_event,
|
||||
"popularity": normalize_popularity,
|
||||
"dragon_tiger": normalize_dragon_tiger,
|
||||
"sector_daily": normalize_sector_daily,
|
||||
}
|
||||
|
||||
|
||||
|
||||
+1335
-90
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user