521 lines
24 KiB
Python
521 lines
24 KiB
Python
from __future__ import annotations
|
|
|
|
import copy
|
|
import re
|
|
from datetime import date, datetime
|
|
from typing import Any
|
|
|
|
from backend.bootstrap.config import normalize_date, validate_text
|
|
from backend.data.providers.tushare_client import TushareError
|
|
from backend.llm import LLMGatewayError
|
|
from backend.features.screener.compiler import (
|
|
LLMCompilerError,
|
|
compile_strategy_with_llm,
|
|
)
|
|
from backend.features.screener.catalog import FACTOR_FIELDS, FACTOR_GROUPS, REGIMES
|
|
from backend.features.screener.data_sync import FactorDataService
|
|
from backend.features.screener.formula import compile_local_strategy
|
|
from backend.features.screener.publication import resolve_published_batch
|
|
from backend.features.screener.signals import (
|
|
attach_strategy_validity,
|
|
build_candidate_archive,
|
|
)
|
|
|
|
|
|
SCREENER_LIBRARY_VERSION = 8
|
|
|
|
|
|
def automatic_screener_jobs(
|
|
strategies: list[dict[str, Any]], regime_id: str
|
|
) -> list[dict[str, Any]]:
|
|
"""Build the close-of-day jobs; only stage screening is regime-gated."""
|
|
smart_strategy = next(
|
|
(
|
|
item for item in strategies
|
|
if item.get("formula", {}).get("meta", {}).get("library") != "curated"
|
|
and regime_id in (item.get("regimes") or [])
|
|
),
|
|
None,
|
|
)
|
|
curated = [
|
|
item for item in strategies
|
|
if item.get("formula", {}).get("meta", {}).get("library") == "curated"
|
|
]
|
|
jobs = ([{"mode": "smart", "strategy": smart_strategy}] if smart_strategy else [])
|
|
jobs.extend({"mode": "curated", "strategy": item} for item in curated)
|
|
return jobs
|
|
|
|
|
|
class ScreenerServiceMixin:
|
|
@staticmethod
|
|
def _strategy_missing_data(
|
|
strategy: dict[str, Any], factor_dates: list[str], factor_health: dict[str, Any]
|
|
) -> list[str]:
|
|
formula = strategy.get("formula") or {}
|
|
meta = formula.get("meta") or {}
|
|
used_fields = {
|
|
str(item.get("field") or "")
|
|
for item in list(formula.get("filters") or []) + list(formula.get("score") or [])
|
|
}
|
|
valuation_fields = {"pe_ttm", "pb", "ps_ttm", "dividend_yield_ttm", "total_mv_billion"}
|
|
fundamental_fields = {"roe", "roa", "roic", "gross_margin", "netprofit_yoy", "revenue_yoy", "ocf_to_opincome"}
|
|
auction_fields = {"auction_change", "auction_amount_million", "auction_turnover_rate", "auction_volume_ratio"}
|
|
missing = []
|
|
required_history = max(21, min(260, int(meta.get("history_days") or 21)))
|
|
if len(factor_dates) < required_history:
|
|
missing.append(f"历史行情（需{required_history}日）")
|
|
if used_fields & valuation_fields and not factor_health["valuation"]:
|
|
missing.append("估值数据")
|
|
if used_fields & fundamental_fields and not factor_health["fundamental"]:
|
|
missing.append("财务质量")
|
|
if meta.get("requires_valuation") and not factor_health["valuation"]:
|
|
missing.append("估值数据")
|
|
if meta.get("requires_fundamental") and not factor_health["fundamental"]:
|
|
missing.append("财务质量")
|
|
if "dividend_years" in used_fields and not factor_health["dividend_history"]:
|
|
missing.append("历年分红")
|
|
if used_fields & auction_fields and not factor_health["auction"]:
|
|
missing.append("竞价数据")
|
|
if meta.get("requires_benchmark") and not factor_health.get("benchmark"):
|
|
missing.append("沪深300基准")
|
|
if meta.get("requires_moneyflow_history") and not factor_health.get("moneyflow_history"):
|
|
missing.append("近5日资金流")
|
|
if meta.get("requires_earnings_events") and not factor_health.get("earnings_events"):
|
|
missing.append("业绩预告与快报")
|
|
if meta.get("requires_popularity") and not factor_health.get("popularity"):
|
|
missing.append("当日人气榜")
|
|
if meta.get("requires_institutions") and not factor_health.get("institutions"):
|
|
missing.append("龙虎榜机构席位")
|
|
return list(dict.fromkeys(missing))
|
|
|
|
def screener_setup(self, trade_date: str) -> dict[str, Any]:
|
|
normalized_date = normalize_date(trade_date)
|
|
regime = self.screener.detect_regime(normalized_date)
|
|
factor_dates = self.database.factor_dates(normalized_date, 300)
|
|
auction_dates = self.database.auction_factor_dates(normalized_date, 100)
|
|
factor_health = self.screener.factor_health(normalized_date)
|
|
strategies = self.database.list_screener_strategies(self.current_user_id)
|
|
for strategy in strategies:
|
|
attach_strategy_validity(strategy)
|
|
missing = self._strategy_missing_data(strategy, factor_dates, factor_health)
|
|
strategy["data_ready"] = not missing
|
|
strategy["missing_data"] = missing
|
|
|
|
batch_markers = self.database.list_screener_batch_markers(normalized_date, 120)
|
|
complete_markers = [
|
|
item for item in batch_markers if item.get("status") == "complete"
|
|
][:30]
|
|
legacy_results = []
|
|
legacy_date = ""
|
|
if not batch_markers:
|
|
legacy_results = self.database.screener_runs_for_date(0, normalized_date)
|
|
if legacy_results:
|
|
legacy_date = normalized_date
|
|
published_marker, automatic_status, published_batch = resolve_published_batch(
|
|
batch_markers, normalized_date, legacy_date
|
|
)
|
|
published_date = str((published_batch or {}).get("trade_date") or "")
|
|
automatic_results = (
|
|
legacy_results
|
|
if legacy_results and published_date == normalized_date
|
|
else self.database.screener_runs_for_date(0, published_date)
|
|
if published_date
|
|
else []
|
|
)
|
|
personal_results = self.database.recent_screener_runs(
|
|
self.current_user_id, normalized_date, "quant", 40
|
|
)
|
|
recent_results = [
|
|
*[item for item in automatic_results if item.get("meta", {}).get("mode") in {"smart", "curated"}],
|
|
*personal_results,
|
|
]
|
|
latest_results: dict[str, dict[str, Any]] = {}
|
|
for result in reversed(recent_results):
|
|
mode = str(result.get("meta", {}).get("mode") or "smart")
|
|
latest_results[mode] = result
|
|
|
|
published_dates = [
|
|
str(item.get("trade_date") or "") for item in complete_markers
|
|
if item.get("trade_date")
|
|
]
|
|
if legacy_date and legacy_date not in published_dates:
|
|
published_dates.append(legacy_date)
|
|
archive_runs = self.database.screener_runs_for_dates(0, published_dates, 1800)
|
|
archive_runs.extend(personal_results)
|
|
archive_as_of_date = published_date or (factor_dates[-1] if factor_dates else "")
|
|
marker_regime = (published_marker or {}).get("regime") or {}
|
|
archive_regime = str(
|
|
(marker_regime.get("id") if isinstance(marker_regime, dict) else marker_regime)
|
|
or regime.get("id") or "repair"
|
|
)
|
|
active_signals, candidate_history = build_candidate_archive(
|
|
archive_runs,
|
|
strategies,
|
|
factor_dates,
|
|
archive_as_of_date,
|
|
archive_regime,
|
|
)
|
|
self._attach_published_strategy_status(
|
|
strategies, automatic_results, published_marker, published_date
|
|
)
|
|
return {
|
|
"trade_date": normalized_date,
|
|
"requested_trade_date": normalized_date,
|
|
"regime": regime,
|
|
"regimes": [{"id": key, "label": value} for key, value in REGIMES.items()],
|
|
"strategies": strategies,
|
|
"factor_fields": [{"id": key, "label": value} for key, value in FACTOR_FIELDS.items()],
|
|
"factor_groups": [
|
|
{
|
|
"name": name,
|
|
"fields": [{"id": field, "label": FACTOR_FIELDS[field]} for field in fields],
|
|
}
|
|
for name, fields in FACTOR_GROUPS.items()
|
|
],
|
|
"operators": [">", ">=", "<", "<=", "==", "between"],
|
|
"factor_data": {
|
|
"date_count": len(factor_dates),
|
|
"start_date": factor_dates[0] if factor_dates else "",
|
|
"end_date": factor_dates[-1] if factor_dates else "",
|
|
"ready": len(factor_dates) >= 21,
|
|
"auction_date_count": len(auction_dates),
|
|
"auction_ready": bool(auction_dates and auction_dates[-1] == factor_dates[-1]) if factor_dates else False,
|
|
"health": factor_health,
|
|
},
|
|
"llm": {
|
|
"configured": self.llm_configured,
|
|
"model": self.llm_primary_model if self.llm_configured else "",
|
|
"fallback_configured": self.llm_fallback_configured,
|
|
"fallback_model": self.llm_fallback_model if self.llm_fallback_configured else "",
|
|
},
|
|
"latest_results": latest_results,
|
|
"recent_results": recent_results,
|
|
"automatic_status": automatic_status,
|
|
"published_batch": published_batch,
|
|
"published_status": published_marker or {},
|
|
"active_signals": active_signals,
|
|
"candidate_history": candidate_history,
|
|
# Kept during the client transition for compatibility with older frontends.
|
|
"latest_result": latest_results.get("smart"),
|
|
}
|
|
|
|
@staticmethod
|
|
def _attach_published_strategy_status(
|
|
strategies: list[dict[str, Any]],
|
|
automatic_results: list[dict[str, Any]],
|
|
marker: dict[str, Any] | None,
|
|
published_date: str,
|
|
) -> None:
|
|
results_by_name = {
|
|
str((item.get("meta") or {}).get("strategy_name") or ""): item
|
|
for item in automatic_results
|
|
}
|
|
skipped_by_name = {
|
|
str(item.get("name") or ""): item
|
|
for item in (marker or {}).get("skipped") or []
|
|
}
|
|
for strategy in strategies:
|
|
name = str(strategy.get("name") or "")
|
|
result = results_by_name.get(name)
|
|
skipped = skipped_by_name.get(name)
|
|
if result is not None:
|
|
candidates = result.get("candidates") or []
|
|
status = "ready" if candidates else "no_signal"
|
|
detail = f"{len(candidates)} 只候选" if candidates else "数据完整，暂无符合条件个股"
|
|
elif skipped is not None:
|
|
status = "missing_data"
|
|
detail = str(skipped.get("reason") or "缺少策略必需数据")
|
|
else:
|
|
status = "not_run"
|
|
detail = "该成功批次未运行此策略"
|
|
strategy["published_run"] = {
|
|
"trade_date": published_date,
|
|
"status": status,
|
|
"detail": detail,
|
|
}
|
|
|
|
def screener_tracking(self, limit: int = 12) -> dict[str, Any]:
|
|
return self.strategy_tracking.list_tracking(self.current_user_id, limit)
|
|
|
|
def add_screener_tracking(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
try:
|
|
run_id = int(payload.get("run_id") or 0)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError("选股批次无效。") from exc
|
|
code = str(payload.get("code") or "").strip()
|
|
if run_id <= 0 or not re.fullmatch(r"\d{6}", code):
|
|
raise ValueError("选股批次或股票代码无效。")
|
|
return self.strategy_tracking.add_candidate(self.current_user_id, run_id, code)
|
|
|
|
def remove_screener_tracking(self, track_id: int) -> dict[str, Any]:
|
|
return self.strategy_tracking.remove_candidate(self.current_user_id, track_id)
|
|
|
|
def refresh_screener_tracking(self, trade_date: str) -> dict[str, Any]:
|
|
normalized_date = normalize_date(trade_date)
|
|
notice = ""
|
|
if self.configured:
|
|
try:
|
|
FactorDataService(self.database, self._tushare_client()).sync(
|
|
normalized_date, 15
|
|
)
|
|
except TushareError:
|
|
notice = "最新日线暂未补齐，已按现有数据更新跟踪。"
|
|
else:
|
|
notice = "公共行情尚未配置，已按现有数据更新跟踪。"
|
|
return {
|
|
"tracking": self.screener_tracking(),
|
|
"notice": notice,
|
|
}
|
|
|
|
def sync_screener_data(self, trade_date: str, lookback: int = 45) -> dict[str, Any]:
|
|
if not self.configured:
|
|
raise ValueError("请先配置 Tushare Token。")
|
|
normalized_date = normalize_date(trade_date)
|
|
lookback = max(25, min(260, int(lookback)))
|
|
with self.sync_lock:
|
|
return FactorDataService(self.database, self._tushare_client()).sync(
|
|
normalized_date, lookback
|
|
)
|
|
|
|
def _schedule_automatic_screeners(
|
|
self, trade_date: str, snapshot: dict[str, Any] | None = None
|
|
) -> bool:
|
|
normalized_date = normalize_date(trade_date)
|
|
now = datetime.now().astimezone()
|
|
if (
|
|
normalized_date != now.strftime("%Y%m%d")
|
|
or now.weekday() >= 5
|
|
or now.time().replace(tzinfo=None) < datetime.strptime("15:10", "%H:%M").time()
|
|
or self.auto_screener_lock.locked()
|
|
):
|
|
return False
|
|
snapshot = snapshot or self.database.get_snapshot(normalized_date) or {}
|
|
actual_date = str((snapshot.get("meta") or {}).get("trade_date") or "").replace("-", "")
|
|
if actual_date != normalized_date:
|
|
return False
|
|
marker = self.database.get_data_snapshot("screener_auto_v1", normalized_date) or {}
|
|
if (
|
|
marker.get("status") == "complete"
|
|
and int(marker.get("library_version") or 0) == SCREENER_LIBRARY_VERSION
|
|
):
|
|
return False
|
|
last_attempt = self._auto_screener_last_attempt.get(normalized_date)
|
|
if last_attempt and (now - last_attempt).total_seconds() < 600:
|
|
return False
|
|
self._auto_screener_last_attempt[normalized_date] = now
|
|
return self.jobs.submit(
|
|
"screener.automatic",
|
|
f"{normalized_date}:v{SCREENER_LIBRARY_VERSION}",
|
|
lambda: self.run_automatic_screeners(normalized_date),
|
|
{"trade_date": normalized_date, "trigger": "post-close"},
|
|
)
|
|
|
|
def run_automatic_screeners(self, trade_date: str) -> dict[str, Any]:
|
|
normalized_date = normalize_date(trade_date)
|
|
with self.auto_screener_lock:
|
|
started_at = datetime.now().astimezone().isoformat(timespec="seconds")
|
|
status: dict[str, Any] = {
|
|
"trade_date": normalized_date,
|
|
"library_version": SCREENER_LIBRARY_VERSION,
|
|
"status": "running",
|
|
"started_at": started_at,
|
|
"completed": [],
|
|
"skipped": [],
|
|
"failed": [],
|
|
}
|
|
self.database.save_data_snapshot(
|
|
"screener_auto_v1", normalized_date, "system", status
|
|
)
|
|
try:
|
|
factor_sync = FactorDataService(
|
|
self.database, self._tushare_client()
|
|
).sync(normalized_date, 260)
|
|
factor_dates = self.database.factor_dates(normalized_date, 300)
|
|
if not factor_dates or factor_dates[-1] != normalized_date:
|
|
raise ValueError("当日收盘行情尚未入库")
|
|
factor_health = self.screener.factor_health(normalized_date)
|
|
regime = self.screener.detect_regime(normalized_date)
|
|
regime_id = str(regime.get("id") or "repair")
|
|
strategies = self.database.list_screener_strategies(None)
|
|
jobs = automatic_screener_jobs(strategies, regime_id)
|
|
existing = {
|
|
(
|
|
str(item.get("meta", {}).get("mode") or "smart"),
|
|
str(item.get("meta", {}).get("strategy_name") or ""),
|
|
)
|
|
for item in self.database.screener_runs_for_date(0, normalized_date)
|
|
if int(item.get("meta", {}).get("library_version") or 0)
|
|
== SCREENER_LIBRARY_VERSION
|
|
}
|
|
required_history = max(
|
|
[
|
|
int((job["strategy"].get("formula", {}).get("meta", {}) or {}).get("history_days") or 80)
|
|
for job in jobs if job.get("strategy")
|
|
] or [80]
|
|
)
|
|
factors, actual_date = self.screener.build_factors(
|
|
normalized_date, history_days=required_history
|
|
)
|
|
if actual_date != normalized_date:
|
|
raise ValueError("当日因子尚未完成收盘定格")
|
|
for job in jobs:
|
|
strategy = job["strategy"]
|
|
mode = str(job["mode"])
|
|
name = str(strategy.get("name") or "未命名策略")
|
|
if (mode, name) in existing:
|
|
status["completed"].append({"mode": mode, "name": name, "cached": True})
|
|
continue
|
|
missing = self._strategy_missing_data(
|
|
strategy, factor_dates, factor_health
|
|
)
|
|
if missing:
|
|
status["skipped"].append(
|
|
{"mode": mode, "name": name, "reason": "、".join(missing)}
|
|
)
|
|
continue
|
|
try:
|
|
formula = copy.deepcopy(strategy.get("formula") or {})
|
|
formula.setdefault("meta", {})["library_version"] = (
|
|
SCREENER_LIBRARY_VERSION
|
|
)
|
|
result = self.screener.screen(
|
|
0,
|
|
normalized_date,
|
|
formula,
|
|
regime_id,
|
|
name,
|
|
False,
|
|
None,
|
|
mode,
|
|
factors,
|
|
actual_date,
|
|
)
|
|
status["completed"].append(
|
|
{
|
|
"mode": mode,
|
|
"name": name,
|
|
"candidate_count": len(result.get("candidates") or []),
|
|
}
|
|
)
|
|
except Exception as exc:
|
|
status["failed"].append(
|
|
{"mode": mode, "name": name, "reason": str(exc)}
|
|
)
|
|
status.update(
|
|
{
|
|
"status": "complete" if not status["failed"] else "partial",
|
|
"finished_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
|
"factor_sync": factor_sync,
|
|
"regime": regime,
|
|
}
|
|
)
|
|
except Exception as exc:
|
|
status.update(
|
|
{
|
|
"status": "failed",
|
|
"finished_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
|
"error": str(exc),
|
|
}
|
|
)
|
|
self.database.save_data_snapshot(
|
|
"screener_auto_v1", normalized_date, "system", status
|
|
)
|
|
return status
|
|
|
|
def compile_screener_strategy(self, prompt: str, regime: str) -> dict[str, Any]:
|
|
prompt = prompt.strip()
|
|
if not prompt or len(prompt) > 3000:
|
|
raise ValueError("策略描述应为 1 至 3000 个字符。")
|
|
if regime not in REGIMES:
|
|
raise ValueError("市场阶段不支持。")
|
|
notice = ""
|
|
source = self.llm_source
|
|
if source == "platform":
|
|
try:
|
|
gateway_result = self.llm_gateway.call(
|
|
"screener",
|
|
"strategy-compiler-v1",
|
|
lambda profile: compile_strategy_with_llm(
|
|
prompt,
|
|
regime,
|
|
profile.api_key,
|
|
profile.base_url,
|
|
profile.model,
|
|
),
|
|
(LLMCompilerError,),
|
|
)
|
|
compiled = gateway_result.value
|
|
if gateway_result.role == "fallback":
|
|
compiled["compiler"] = "llm_fallback"
|
|
notice = "智能策略生成服务已自动切换。"
|
|
except LLMGatewayError as exc:
|
|
if exc.code != "unavailable":
|
|
raise
|
|
compiled = compile_local_strategy(prompt, regime)
|
|
notice = "智能策略生成暂不可用，已使用本地模板。"
|
|
else:
|
|
compiled = compile_local_strategy(prompt, regime)
|
|
notice = "智能策略生成暂不可用，已使用本地模板。"
|
|
compiled["formula"] = self.screener.validate_formula(compiled["formula"])
|
|
compiled["notice"] = notice
|
|
return compiled
|
|
|
|
def save_screener_strategy(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
name = validate_text(payload.get("name"), "策略名称", 60, required=True)
|
|
description = validate_text(payload.get("description"), "策略说明", 1000)
|
|
regimes = payload.get("regimes") or []
|
|
if not isinstance(regimes, list) or not regimes or any(item not in REGIMES for item in regimes):
|
|
raise ValueError("策略适用阶段不正确。")
|
|
formula = self.screener.validate_formula(payload.get("formula") or {})
|
|
strategy_id = self.database.save_screener_strategy(
|
|
self.current_user_id, name, description, regimes, formula
|
|
)
|
|
return {
|
|
"id": strategy_id,
|
|
"strategies": self.database.list_screener_strategies(self.current_user_id),
|
|
}
|
|
|
|
def delete_screener_strategy(self, strategy_id: int) -> dict[str, Any]:
|
|
deleted = self.database.delete_screener_strategy(self.current_user_id, strategy_id)
|
|
return {
|
|
"deleted": deleted,
|
|
"strategies": self.database.list_screener_strategies(self.current_user_id),
|
|
}
|
|
|
|
def run_screener(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
trade_date = normalize_date(str(payload.get("trade_date") or date.today().isoformat()))
|
|
regime = str(payload.get("regime") or "")
|
|
if regime not in REGIMES:
|
|
raise ValueError("市场阶段不支持。")
|
|
strategy_name = validate_text(payload.get("strategy_name"), "策略名称", 60, required=True)
|
|
formula = payload.get("formula") or {}
|
|
requested_mode = str(payload.get("mode") or "").strip()
|
|
if requested_mode and requested_mode not in {"smart", "curated", "quant"}:
|
|
raise ValueError("选股模式不受支持。")
|
|
if requested_mode:
|
|
mode = requested_mode
|
|
else:
|
|
meta = formula.get("meta") if isinstance(formula, dict) else {}
|
|
library = str((meta or {}).get("library") or "")
|
|
category = str((meta or {}).get("category") or "")
|
|
if library == "curated":
|
|
mode = "curated"
|
|
elif library == "quant" or (library == "custom" and category == "量化公式"):
|
|
mode = "quant"
|
|
else:
|
|
mode = "smart"
|
|
realtime_snapshot = None
|
|
dashboard = self.get_dashboard(trade_date)
|
|
if self.configured and dashboard.get("meta", {}).get("realtime"):
|
|
try:
|
|
realtime_snapshot = self._tushare_client().realtime_factor_snapshot(trade_date)
|
|
except TushareError as exc:
|
|
raise ValueError(f"实时选股行情不可用，已停止筛选：{exc}") from exc
|
|
result = self.screener.screen(
|
|
self.current_user_id, trade_date, formula, regime, strategy_name,
|
|
bool(payload.get("run_backtest", True)),
|
|
realtime_snapshot,
|
|
mode,
|
|
)
|
|
return result
|