from __future__ import annotations import statistics from datetime import datetime, timedelta from typing import Any from backend.data.numbers import finite_number as _number from backend.data.providers.tushare_client import TushareClient, TushareError from backend.features.screener.indicators import _optional_number from database import ReviewDatabase def _quarter_periods(trade_date: str, count: int) -> list[str]: current = datetime.strptime(trade_date, "%Y%m%d") quarter_ends = ((3, 31), (6, 30), (9, 30), (12, 31)) periods = [] year = current.year while len(periods) < count: for month, day in reversed(quarter_ends): value = datetime(year, month, day) if value <= current: periods.append(value.strftime("%Y%m%d")) if len(periods) == count: break year -= 1 return sorted(periods) def _earnings_event_rows( forecasts: list[dict[str, Any]], expresses: list[dict[str, Any]], trade_date: str, ) -> list[dict[str, Any]]: forecast_map: dict[tuple[str, str], dict[str, Any]] = {} for row in forecasts: key = (str(row.get("ts_code") or ""), str(row.get("end_date") or "")) ann_date = str(row.get("ann_date") or "") if not all(key) or not ann_date or ann_date > trade_date: continue previous = forecast_map.get(key) if previous is None or ann_date > str(previous.get("ann_date") or ""): forecast_map[key] = row result = [] for row in expresses: ts_code = str(row.get("ts_code") or "") end_date = str(row.get("end_date") or "") ann_date = str(row.get("ann_date") or "") forecast = forecast_map.get((ts_code, end_date)) if not forecast or not ts_code or not end_date or not ann_date or ann_date > trade_date: continue lower = _optional_number(forecast.get("net_profit_min")) upper = _optional_number(forecast.get("net_profit_max")) forecast_profit = statistics.fmean( value for value in (lower, upper) if value is not None ) if lower is not None or upper is not None else None actual_profit = _optional_number(row.get("n_income")) if forecast_profit in (None, 0) or actual_profit is None: continue # forecast is reported in ten-thousand yuan while express uses yuan. if abs(actual_profit) > max(abs(forecast_profit), 1) * 100: actual_profit /= 10000 surprise_pct = (actual_profit / forecast_profit - 1) * 100 result.append( { "end_date": end_date, "ann_date": ann_date, "ts_code": ts_code, "forecast_profit": forecast_profit, "actual_profit": actual_profit, "surprise_pct": surprise_pct, "revenue_yoy": _optional_number(row.get("yoy_sales")), "netprofit_yoy": _optional_number(row.get("yoy_net_profit")), "source": "forecast+express", } ) return result def _popularity_factor_rows( trade_date: str, ths_rows: list[dict[str, Any]], dc_rows: list[dict[str, Any]], previous_ths: list[dict[str, Any]], previous_dc: list[dict[str, Any]], ) -> list[dict[str, Any]]: def ranks(rows: list[dict[str, Any]], data_type: str) -> dict[str, int]: result = {} for row in rows: if data_type and str(row.get("data_type") or "") != data_type: continue ts_code = str(row.get("ts_code") or "") rank = int(_number(row.get("rank"))) if ts_code and rank > 0: result[ts_code] = rank return result ths = ranks(ths_rows, "热股") dc = ranks(dc_rows, "A股市场") previous_ths_map = ranks(previous_ths, "热股") previous_dc_map = ranks(previous_dc, "A股市场") result = [] for ts_code in set(ths) | set(dc): ths_rank = ths.get(ts_code) dc_rank = dc.get(ts_code) current_best = min(value for value in (ths_rank, dc_rank) if value is not None) previous_candidates = [ value for value in (previous_ths_map.get(ts_code), previous_dc_map.get(ts_code)) if value is not None ] previous_best = min(previous_candidates) if previous_candidates else None score = (101 - (ths_rank or 101)) * 0.5 + (201 - (dc_rank or 201)) * 0.25 result.append( { "trade_date": trade_date, "ts_code": ts_code, "ths_rank": ths_rank, "dc_rank": dc_rank, "combined_score": round(score, 2), "rank_change": ( previous_best - current_best if previous_best is not None else min(30, max(0, 31 - current_best)) if previous_ths_map or previous_dc_map else 0 ), "dual_source": bool(ths_rank and dc_rank), } ) return result class FactorDataService: def __init__(self, database: ReviewDatabase, client: TushareClient) -> None: self.database = database self.client = client def sync(self, requested_date: str, lookback: int = 45) -> dict[str, Any]: lookback = max(25, min(260, int(lookback))) trade_date, _ = self.client.resolve_trade_context(requested_date) end = datetime.strptime(trade_date, "%Y%m%d") start = (end - timedelta(days=max(100, lookback * 2 + 20))).strftime("%Y%m%d") calendar = self.client.query( "trade_cal", {"exchange": "SSE", "start_date": start, "end_date": trade_date, "is_open": 1}, "cal_date,is_open", ) dates = sorted(row["cal_date"] for row in calendar if row.get("is_open") == 1)[-lookback:] existing = set(self.database.factor_dates(trade_date, lookback + 10)) dates_to_fetch = [value for value in dates if value not in existing or value == trade_date] auction_source_dates = dates[-min(80, len(dates)):] existing_auction = set(self.database.auction_factor_dates(trade_date, 90)) auction_dates_to_fetch = [ value for value in auction_source_dates if value not in existing_auction or value == trade_date ] long_calendar = self.client.query( "trade_cal", { "exchange": "SSE", "start_date": datetime(end.year - 5, 1, 1).strftime("%Y%m%d"), "end_date": trade_date, "is_open": 1, }, "cal_date,is_open", ) last_open_by_year: dict[str, str] = {} last_open_by_month: dict[str, str] = {} for row in long_calendar: if row.get("is_open") == 1 and row.get("cal_date"): value = str(row["cal_date"]) last_open_by_year[value[:4]] = max(last_open_by_year.get(value[:4], ""), value) last_open_by_month[value[:6]] = max(last_open_by_month.get(value[:6], ""), value) valuation_dates = set(dates[-min(80, len(dates)):]) valuation_dates.update(last_open_by_year.values()) valuation_dates.update(last_open_by_month.values()) existing_indicators = set(self.database.daily_indicator_dates(trade_date, 500)) indicator_dates_to_fetch = sorted( value for value in valuation_dates if value not in existing_indicators or value == trade_date ) master = self.client.query( "stock_basic", {"list_status": "L"}, "ts_code,name,industry,market,list_date", ) master_count = self.database.upsert_stock_master(master) bar_count = 0 for current_date in dates_to_fetch: rows = self.client.query( "daily", {"trade_date": current_date}, "ts_code,trade_date,open,high,low,close,pct_chg,vol,amount", ) bar_count += self.database.upsert_daily_bars(rows) indicator_count = 0 for current_date in indicator_dates_to_fetch: indicators = self.client.query( "daily_basic", {"trade_date": current_date}, "ts_code,trade_date,turnover_rate,volume_ratio,total_mv,circ_mv," "pe_ttm,pb,ps_ttm,dv_ttm", ) indicator_count += self.database.upsert_daily_indicators(indicators) notices = [] benchmark_count = 0 try: benchmark_rows = self.client.query( "index_daily", {"ts_code": "000300.SH", "start_date": dates[0], "end_date": trade_date}, "ts_code,trade_date,close,pct_chg", ) benchmark_count = self.database.upsert_benchmark_bars(benchmark_rows) except TushareError as exc: notices.append(f"沪深300基准暂不可用：{exc}") fundamental_count = 0 existing_periods = set(self.database.fundamental_periods()) for period in _quarter_periods(trade_date, 9): if period in existing_periods and period < trade_date[:4] + "0101": continue try: rows = self.client.query( "fina_indicator_vip", {"period": period}, "ts_code,ann_date,end_date,roe,roa,roic,grossprofit_margin," "netprofit_yoy,or_yoy,ocf_to_opincome", ) except TushareError as exc: notices.append(f"财务质量接口不可用：{exc}") break published = [ row for row in rows if not row.get("ann_date") or str(row.get("ann_date")) <= trade_date ] published.sort(key=lambda row: str(row.get("ann_date") or "")) fundamental_count += self.database.upsert_fundamental_indicators(published) auction_count = 0 auction_dates = 0 for current_date in auction_dates_to_fetch: try: auction_rows = self.client.query( "stk_auction", {"trade_date": current_date}, "ts_code,trade_date,vol,price,amount,pre_close,turnover_rate,volume_ratio,float_share", ) if auction_rows: auction_count += self.database.upsert_auction_factors(auction_rows) auction_dates += 1 except TushareError as exc: notices.append(f"竞价因子接口不可用：{exc}") break moneyflow_count = 0 moneyflow_dates = 0 for current_date in dates[-min(5, len(dates)):]: try: moneyflow = self.client.query( "moneyflow", {"trade_date": current_date}, "ts_code,trade_date,buy_sm_amount,sell_sm_amount,buy_md_amount,sell_md_amount," "buy_lg_amount,sell_lg_amount,buy_elg_amount,sell_elg_amount,net_mf_amount", ) moneyflow_count += self.database.upsert_moneyflow(moneyflow) if moneyflow: moneyflow_dates += 1 except TushareError as exc: notices.append(f"资金流接口不可用：{exc}") break earnings_count = 0 forecasts: list[dict[str, Any]] = [] expresses: list[dict[str, Any]] = [] for period in _quarter_periods(trade_date, 5): try: forecast_rows = self.client.query( "forecast_vip", {"period": period}, "ts_code,ann_date,end_date,net_profit_min,net_profit_max,last_parent_net,p_change_min,p_change_max", ) express_rows = self.client.query( "express_vip", {"period": period}, "ts_code,ann_date,end_date,n_income,yoy_net_profit,yoy_sales", ) except TushareError as exc: notices.append(f"业绩事件接口不可用：{exc}") break forecasts.extend(forecast_rows) expresses.extend(express_rows) if forecasts and expresses: earnings_count = self.database.upsert_earnings_events( _earnings_event_rows(forecasts, expresses, trade_date) ) popularity_count = 0 previous_trade_date = dates[-2] if len(dates) >= 2 else "" try: ths_rows = self.client.query("ths_hot", {"trade_date": trade_date}) dc_rows = self.client.query("dc_hot", {"trade_date": trade_date}) previous_ths = ( self.client.query("ths_hot", {"trade_date": previous_trade_date}) if previous_trade_date else [] ) previous_dc = ( self.client.query("dc_hot", {"trade_date": previous_trade_date}) if previous_trade_date else [] ) popularity_count = self.database.upsert_popularity_factors( _popularity_factor_rows( trade_date, ths_rows, dc_rows, previous_ths, previous_dc ) ) except TushareError as exc: notices.append(f"人气榜因子不可用：{exc}") institution_count = 0 try: institution_rows = self.client.query( "top_inst", {"trade_date": trade_date}, "trade_date,ts_code,exalter,buy,sell,net_buy,side,reason", ) institution_count = self.database.upsert_lhb_institutions(institution_rows) except TushareError as exc: notices.append(f"机构席位明细不可用：{exc}") return { "trade_date": trade_date, "calendar_dates": len(dates), "fetched_dates": len(dates_to_fetch), "stocks": master_count, "bars": bar_count, "benchmark_bars": benchmark_count, "indicators": indicator_count, "indicator_dates": len(indicator_dates_to_fetch), "fundamentals": fundamental_count, "moneyflow": moneyflow_count, "moneyflow_dates": moneyflow_dates, "auction_rows": auction_count, "auction_dates": auction_dates, "earnings_events": earnings_count, "popularity_rows": popularity_count, "institution_rows": institution_count, "notice": "；".join(notices), }