leefer/xiaobai-review
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

migration: preserve screener and tracking slice

Browse Source
This commit is contained in:
leefer
2026-07-31 03:57:07 +08:00
parent f511bf484f
commit 76b57f410d
19 changed files with 4508 additions and 4146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/application.py
+7 -418
View File
@@ -40,15 +40,8 @@ from heaven_engine import (
hexagram_from_lines,
)
from backend.data.providers.ifind_client import IfindError
from llm_strategy import LLMCompilerError, compile_strategy_with_llm, test_llm_connection
from llm_strategy import LLMCompilerError, test_llm_connection
from mentor_agent import MentorAgentError, stream_with_mentor
from screener import (
FACTOR_FIELDS,
FACTOR_GROUPS,
REGIMES,
FactorDataService,
compile_local_strategy,
)
from backend.features.accounts.http import AccountHttpMixin
from backend.features.accounts.security import SecretVault
from backend.features.accounts.service import AccountService
@@ -57,36 +50,17 @@ from backend.features.dragon_tiger import DragonTigerServiceMixin
from backend.features.pools import PoolServiceMixin
from backend.features.popularity import PopularityServiceMixin
from backend.features.rotation import RotationServiceMixin
from backend.features.screener.service import (
SCREENER_LIBRARY_VERSION,
ScreenerServiceMixin,
automatic_screener_jobs,
)
from backend.features.sentiment import SentimentServiceMixin
from backend.features.system import SystemHttpMixin
from backend.features.themes import ThemeServiceMixin
from backend.data.providers.tushare_client import TushareClient, TushareError, _sector_coverage_issue
SCREENER_LIBRARY_VERSION = 8
def automatic_screener_jobs(
strategies: list[dict[str, Any]], regime_id: str
) -> list[dict[str, Any]]:
"""Build the close-of-day jobs; only stage screening is regime-gated."""
smart_strategy = next(
(
item for item in strategies
if item.get("formula", {}).get("meta", {}).get("library") != "curated"
and regime_id in (item.get("regimes") or [])
),
None,
)
curated = [
item for item in strategies
if item.get("formula", {}).get("meta", {}).get("library") == "curated"
]
jobs = ([{"mode": "smart", "strategy": smart_strategy}] if smart_strategy else [])
jobs.extend({"mode": "curated", "strategy": item} for item in curated)
return jobs
LEGACY_SECRET_KEYS = {
"TUSHARE_TOKEN",
"IFIND_REFRESH_TOKEN",
@@ -149,6 +123,7 @@ class DashboardService(
ThemeServiceMixin,
PopularityServiceMixin,
DragonTigerServiceMixin,
ScreenerServiceMixin,
):
def __init__(self) -> None:
runtime = load_runtime_settings()
@@ -788,141 +763,6 @@ class DashboardService(
return match.group(1)
return ""
@staticmethod
def _strategy_missing_data(
strategy: dict[str, Any], factor_dates: list[str], factor_health: dict[str, Any]
) -> list[str]:
formula = strategy.get("formula") or {}
meta = formula.get("meta") or {}
used_fields = {
str(item.get("field") or "")
for item in list(formula.get("filters") or []) + list(formula.get("score") or [])
}
valuation_fields = {"pe_ttm", "pb", "ps_ttm", "dividend_yield_ttm", "total_mv_billion"}
fundamental_fields = {"roe", "roa", "roic", "gross_margin", "netprofit_yoy", "revenue_yoy", "ocf_to_opincome"}
auction_fields = {"auction_change", "auction_amount_million", "auction_turnover_rate", "auction_volume_ratio"}
missing = []
required_history = max(21, min(260, int(meta.get("history_days") or 21)))
if len(factor_dates) < required_history:
missing.append(f"历史行情（需{required_history}日）")
if used_fields & valuation_fields and not factor_health["valuation"]:
missing.append("估值数据")
if used_fields & fundamental_fields and not factor_health["fundamental"]:
missing.append("财务质量")
if meta.get("requires_valuation") and not factor_health["valuation"]:
missing.append("估值数据")
if meta.get("requires_fundamental") and not factor_health["fundamental"]:
missing.append("财务质量")
if "dividend_years" in used_fields and not factor_health["dividend_history"]:
missing.append("历年分红")
if used_fields & auction_fields and not factor_health["auction"]:
missing.append("竞价数据")
if meta.get("requires_benchmark") and not factor_health.get("benchmark"):
missing.append("沪深300基准")
if meta.get("requires_moneyflow_history") and not factor_health.get("moneyflow_history"):
missing.append("近5日资金流")
if meta.get("requires_earnings_events") and not factor_health.get("earnings_events"):
missing.append("业绩预告与快报")
if meta.get("requires_popularity") and not factor_health.get("popularity"):
missing.append("当日人气榜")
if meta.get("requires_institutions") and not factor_health.get("institutions"):
missing.append("龙虎榜机构席位")
return list(dict.fromkeys(missing))
def screener_setup(self, trade_date: str) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
regime = self.screener.detect_regime(normalized_date)
factor_dates = self.database.factor_dates(normalized_date, 300)
auction_dates = self.database.auction_factor_dates(normalized_date, 100)
factor_health = self.screener.factor_health(normalized_date)
strategies = self.database.list_screener_strategies(self.current_user_id)
for strategy in strategies:
missing = self._strategy_missing_data(strategy, factor_dates, factor_health)
strategy["data_ready"] = not missing
strategy["missing_data"] = missing
automatic_results = self.database.screener_runs_for_date(0, normalized_date)
personal_results = self.database.screener_runs_for_date(
self.current_user_id, normalized_date
)
recent_results = [
*[item for item in automatic_results if item.get("meta", {}).get("mode") in {"smart", "curated"}],
*[item for item in personal_results if item.get("meta", {}).get("mode") == "quant"],
]
latest_results: dict[str, dict[str, Any]] = {}
for result in reversed(recent_results):
mode = str(result.get("meta", {}).get("mode") or "smart")
latest_results[mode] = result
automatic_status = self.database.get_data_snapshot(
"screener_auto_v1", normalized_date
) or {}
return {
"trade_date": normalized_date,
"regime": regime,
"regimes": [{"id": key, "label": value} for key, value in REGIMES.items()],
"strategies": strategies,
"factor_fields": [{"id": key, "label": value} for key, value in FACTOR_FIELDS.items()],
"factor_groups": [
{
"name": name,
"fields": [{"id": field, "label": FACTOR_FIELDS[field]} for field in fields],
}
for name, fields in FACTOR_GROUPS.items()
],
"operators": [">", ">=", "<", "<=", "==", "between"],
"factor_data": {
"date_count": len(factor_dates),
"start_date": factor_dates[0] if factor_dates else "",
"end_date": factor_dates[-1] if factor_dates else "",
"ready": len(factor_dates) >= 21,
"auction_date_count": len(auction_dates),
"auction_ready": bool(auction_dates and auction_dates[-1] == factor_dates[-1]) if factor_dates else False,
"health": factor_health,
},
"llm": {
"configured": self.llm_configured,
"model": self.llm_primary_model if self.llm_configured else "",
"fallback_configured": self.llm_fallback_configured,
"fallback_model": self.llm_fallback_model if self.llm_fallback_configured else "",
},
"latest_results": latest_results,
"recent_results": recent_results,
"automatic_status": automatic_status,
# Kept during the client transition for compatibility with older frontends.
"latest_result": latest_results.get("smart"),
}
def screener_tracking(self, limit: int = 12) -> dict[str, Any]:
return self.strategy_tracking.list_tracking(self.current_user_id, limit)
def add_screener_tracking(self, payload: dict[str, Any]) -> dict[str, Any]:
try:
run_id = int(payload.get("run_id") or 0)
except (TypeError, ValueError) as exc:
raise ValueError("选股批次无效。") from exc
code = str(payload.get("code") or "").strip()
if run_id <= 0 or not re.fullmatch(r"\d{6}", code):
raise ValueError("选股批次或股票代码无效。")
return self.strategy_tracking.add_candidate(self.current_user_id, run_id, code)
def remove_screener_tracking(self, track_id: int) -> dict[str, Any]:
return self.strategy_tracking.remove_candidate(self.current_user_id, track_id)
def refresh_screener_tracking(self, trade_date: str) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
notice = ""
if self.configured:
try:
FactorDataService(self.database, self._tushare_client()).sync(
normalized_date, 15
)
except TushareError:
notice = "最新日线暂未补齐，已按现有数据更新跟踪。"
else:
notice = "公共行情尚未配置，已按现有数据更新跟踪。"
return {
"tracking": self.screener_tracking(),
"notice": notice,
}
def alert_center(self, status: str = "all", as_of: str = "") -> dict[str, Any]:
tracking = self.strategy_tracking.list_tracking(self.current_user_id, 12)
@@ -1126,220 +966,6 @@ class DashboardService(
},
}
def sync_screener_data(self, trade_date: str, lookback: int = 45) -> dict[str, Any]:
if not self.configured:
raise ValueError("请先配置 Tushare Token。")
normalized_date = normalize_date(trade_date)
lookback = max(25, min(260, int(lookback)))
with self.sync_lock:
return FactorDataService(self.database, self._tushare_client()).sync(
normalized_date, lookback
)
def _schedule_automatic_screeners(
self, trade_date: str, snapshot: dict[str, Any] | None = None
) -> bool:
normalized_date = normalize_date(trade_date)
now = datetime.now().astimezone()
if (
normalized_date != now.strftime("%Y%m%d")
or now.weekday() >= 5
or now.time().replace(tzinfo=None) < datetime.strptime("15:10", "%H:%M").time()
or self.auto_screener_lock.locked()
):
return False
snapshot = snapshot or self.database.get_snapshot(normalized_date) or {}
actual_date = str((snapshot.get("meta") or {}).get("trade_date") or "").replace("-", "")
if actual_date != normalized_date:
return False
marker = self.database.get_data_snapshot("screener_auto_v1", normalized_date) or {}
if (
marker.get("status") == "complete"
and int(marker.get("library_version") or 0) == SCREENER_LIBRARY_VERSION
):
return False
last_attempt = self._auto_screener_last_attempt.get(normalized_date)
if last_attempt and (now - last_attempt).total_seconds() < 600:
return False
self._auto_screener_last_attempt[normalized_date] = now
return self.jobs.submit(
"screener.automatic",
f"{normalized_date}:v{SCREENER_LIBRARY_VERSION}",
lambda: self.run_automatic_screeners(normalized_date),
{"trade_date": normalized_date, "trigger": "post-close"},
)
def run_automatic_screeners(self, trade_date: str) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
with self.auto_screener_lock:
started_at = datetime.now().astimezone().isoformat(timespec="seconds")
status: dict[str, Any] = {
"trade_date": normalized_date,
"library_version": SCREENER_LIBRARY_VERSION,
"status": "running",
"started_at": started_at,
"completed": [],
"skipped": [],
"failed": [],
}
self.database.save_data_snapshot(
"screener_auto_v1", normalized_date, "system", status
)
try:
factor_sync = FactorDataService(
self.database, self._tushare_client()
).sync(normalized_date, 260)
factor_dates = self.database.factor_dates(normalized_date, 300)
if not factor_dates or factor_dates[-1] != normalized_date:
raise ValueError("当日收盘行情尚未入库")
factor_health = self.screener.factor_health(normalized_date)
regime = self.screener.detect_regime(normalized_date)
regime_id = str(regime.get("id") or "repair")
strategies = self.database.list_screener_strategies(None)
jobs = automatic_screener_jobs(strategies, regime_id)
existing = {
(
str(item.get("meta", {}).get("mode") or "smart"),
str(item.get("meta", {}).get("strategy_name") or ""),
)
for item in self.database.screener_runs_for_date(0, normalized_date)
if int(item.get("meta", {}).get("library_version") or 0)
== SCREENER_LIBRARY_VERSION
}
required_history = max(
[
int((job["strategy"].get("formula", {}).get("meta", {}) or {}).get("history_days") or 80)
for job in jobs if job.get("strategy")
] or [80]
)
factors, actual_date = self.screener.build_factors(
normalized_date, history_days=required_history
)
if actual_date != normalized_date:
raise ValueError("当日因子尚未完成收盘定格")
for job in jobs:
strategy = job["strategy"]
mode = str(job["mode"])
name = str(strategy.get("name") or "未命名策略")
if (mode, name) in existing:
status["completed"].append({"mode": mode, "name": name, "cached": True})
continue
missing = self._strategy_missing_data(
strategy, factor_dates, factor_health
)
if missing:
status["skipped"].append(
{"mode": mode, "name": name, "reason": "、".join(missing)}
)
continue
try:
formula = copy.deepcopy(strategy.get("formula") or {})
formula.setdefault("meta", {})["library_version"] = (
SCREENER_LIBRARY_VERSION
)
result = self.screener.screen(
0,
normalized_date,
formula,
regime_id,
name,
False,
None,
mode,
factors,
actual_date,
)
status["completed"].append(
{
"mode": mode,
"name": name,
"candidate_count": len(result.get("candidates") or []),
}
)
except Exception as exc:
status["failed"].append(
{"mode": mode, "name": name, "reason": str(exc)}
)
status.update(
{
"status": "complete" if not status["failed"] else "partial",
"finished_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"factor_sync": factor_sync,
"regime": regime,
}
)
except Exception as exc:
status.update(
{
"status": "failed",
"finished_at": datetime.now().astimezone().isoformat(timespec="seconds"),
"error": str(exc),
}
)
self.database.save_data_snapshot(
"screener_auto_v1", normalized_date, "system", status
)
return status
def compile_screener_strategy(self, prompt: str, regime: str) -> dict[str, Any]:
prompt = prompt.strip()
if not prompt or len(prompt) > 3000:
raise ValueError("策略描述应为 1 至 3000 个字符。")
if regime not in REGIMES:
raise ValueError("市场阶段不支持。")
notice = ""
source = self.llm_source
if source == "platform":
try:
gateway_result = self.llm_gateway.call(
"screener",
"strategy-compiler-v1",
lambda profile: compile_strategy_with_llm(
prompt,
regime,
profile.api_key,
profile.base_url,
profile.model,
),
(LLMCompilerError,),
)
compiled = gateway_result.value
if gateway_result.role == "fallback":
compiled["compiler"] = "llm_fallback"
notice = "智能策略生成服务已自动切换。"
except LLMGatewayError as exc:
if exc.code != "unavailable":
raise
compiled = compile_local_strategy(prompt, regime)
notice = "智能策略生成暂不可用，已使用本地模板。"
else:
compiled = compile_local_strategy(prompt, regime)
notice = "智能策略生成暂不可用，已使用本地模板。"
compiled["formula"] = self.screener.validate_formula(compiled["formula"])
compiled["notice"] = notice
return compiled
def save_screener_strategy(self, payload: dict[str, Any]) -> dict[str, Any]:
name = validate_text(payload.get("name"), "策略名称", 60, required=True)
description = validate_text(payload.get("description"), "策略说明", 1000)
regimes = payload.get("regimes") or []
if not isinstance(regimes, list) or not regimes or any(item not in REGIMES for item in regimes):
raise ValueError("策略适用阶段不正确。")
formula = self.screener.validate_formula(payload.get("formula") or {})
strategy_id = self.database.save_screener_strategy(
self.current_user_id, name, description, regimes, formula
)
return {
"id": strategy_id,
"strategies": self.database.list_screener_strategies(self.current_user_id),
}
def delete_screener_strategy(self, strategy_id: int) -> dict[str, Any]:
deleted = self.database.delete_screener_strategy(self.current_user_id, strategy_id)
return {
"deleted": deleted,
"strategies": self.database.list_screener_strategies(self.current_user_id),
}
def mentor_setup(self, trade_date: str) -> dict[str, Any]:
normalized_date = normalize_date(trade_date)
@@ -3025,43 +2651,6 @@ class DashboardService(
)
return result
def run_screener(self, payload: dict[str, Any]) -> dict[str, Any]:
trade_date = normalize_date(str(payload.get("trade_date") or date.today().isoformat()))
regime = str(payload.get("regime") or "")
if regime not in REGIMES:
raise ValueError("市场阶段不支持。")
strategy_name = validate_text(payload.get("strategy_name"), "策略名称", 60, required=True)
formula = payload.get("formula") or {}
requested_mode = str(payload.get("mode") or "").strip()
if requested_mode and requested_mode not in {"smart", "curated", "quant"}:
raise ValueError("选股模式不受支持。")
if requested_mode:
mode = requested_mode
else:
meta = formula.get("meta") if isinstance(formula, dict) else {}
library = str((meta or {}).get("library") or "")
category = str((meta or {}).get("category") or "")
if library == "curated":
mode = "curated"
elif library == "quant" or (library == "custom" and category == "量化公式"):
mode = "quant"
else:
mode = "smart"
realtime_snapshot = None
dashboard = self.get_dashboard(trade_date)
if self.configured and dashboard.get("meta", {}).get("realtime"):
try:
realtime_snapshot = self._tushare_client().realtime_factor_snapshot(trade_date)
except TushareError as exc:
raise ValueError(f"实时选股行情不可用，已停止筛选：{exc}") from exc
result = self.screener.screen(
self.current_user_id, trade_date, formula, regime, strategy_name,
bool(payload.get("run_backtest", True)),
realtime_snapshot,
mode,
)
return result
SERVICE = DashboardService()