343 lines
14 KiB
Python
343 lines
14 KiB
Python
from __future__ import annotations
|
||
|
||
import statistics
|
||
from datetime import datetime, timedelta
|
||
from typing import Any
|
||
|
||
from backend.data.numbers import finite_number as _number
|
||
from backend.data.providers.tushare_client import TushareClient, TushareError
|
||
from backend.features.screener.indicators import _optional_number
|
||
from database import ReviewDatabase
|
||
|
||
|
||
def _quarter_periods(trade_date: str, count: int) -> list[str]:
|
||
current = datetime.strptime(trade_date, "%Y%m%d")
|
||
quarter_ends = ((3, 31), (6, 30), (9, 30), (12, 31))
|
||
periods = []
|
||
year = current.year
|
||
while len(periods) < count:
|
||
for month, day in reversed(quarter_ends):
|
||
value = datetime(year, month, day)
|
||
if value <= current:
|
||
periods.append(value.strftime("%Y%m%d"))
|
||
if len(periods) == count:
|
||
break
|
||
year -= 1
|
||
return sorted(periods)
|
||
|
||
|
||
def _earnings_event_rows(
|
||
forecasts: list[dict[str, Any]], expresses: list[dict[str, Any]], trade_date: str,
|
||
) -> list[dict[str, Any]]:
|
||
forecast_map: dict[tuple[str, str], dict[str, Any]] = {}
|
||
for row in forecasts:
|
||
key = (str(row.get("ts_code") or ""), str(row.get("end_date") or ""))
|
||
ann_date = str(row.get("ann_date") or "")
|
||
if not all(key) or not ann_date or ann_date > trade_date:
|
||
continue
|
||
previous = forecast_map.get(key)
|
||
if previous is None or ann_date > str(previous.get("ann_date") or ""):
|
||
forecast_map[key] = row
|
||
result = []
|
||
for row in expresses:
|
||
ts_code = str(row.get("ts_code") or "")
|
||
end_date = str(row.get("end_date") or "")
|
||
ann_date = str(row.get("ann_date") or "")
|
||
forecast = forecast_map.get((ts_code, end_date))
|
||
if not forecast or not ts_code or not end_date or not ann_date or ann_date > trade_date:
|
||
continue
|
||
lower = _optional_number(forecast.get("net_profit_min"))
|
||
upper = _optional_number(forecast.get("net_profit_max"))
|
||
forecast_profit = statistics.fmean(
|
||
value for value in (lower, upper) if value is not None
|
||
) if lower is not None or upper is not None else None
|
||
actual_profit = _optional_number(row.get("n_income"))
|
||
if forecast_profit in (None, 0) or actual_profit is None:
|
||
continue
|
||
# forecast is reported in ten-thousand yuan while express uses yuan.
|
||
if abs(actual_profit) > max(abs(forecast_profit), 1) * 100:
|
||
actual_profit /= 10000
|
||
surprise_pct = (actual_profit / forecast_profit - 1) * 100
|
||
result.append(
|
||
{
|
||
"end_date": end_date,
|
||
"ann_date": ann_date,
|
||
"ts_code": ts_code,
|
||
"forecast_profit": forecast_profit,
|
||
"actual_profit": actual_profit,
|
||
"surprise_pct": surprise_pct,
|
||
"revenue_yoy": _optional_number(row.get("yoy_sales")),
|
||
"netprofit_yoy": _optional_number(row.get("yoy_net_profit")),
|
||
"source": "forecast+express",
|
||
}
|
||
)
|
||
return result
|
||
|
||
|
||
def _popularity_factor_rows(
|
||
trade_date: str,
|
||
ths_rows: list[dict[str, Any]],
|
||
dc_rows: list[dict[str, Any]],
|
||
previous_ths: list[dict[str, Any]],
|
||
previous_dc: list[dict[str, Any]],
|
||
) -> list[dict[str, Any]]:
|
||
def ranks(rows: list[dict[str, Any]], data_type: str) -> dict[str, int]:
|
||
result = {}
|
||
for row in rows:
|
||
if data_type and str(row.get("data_type") or "") != data_type:
|
||
continue
|
||
ts_code = str(row.get("ts_code") or "")
|
||
rank = int(_number(row.get("rank")))
|
||
if ts_code and rank > 0:
|
||
result[ts_code] = rank
|
||
return result
|
||
|
||
ths = ranks(ths_rows, "热股")
|
||
dc = ranks(dc_rows, "A股市场")
|
||
previous_ths_map = ranks(previous_ths, "热股")
|
||
previous_dc_map = ranks(previous_dc, "A股市场")
|
||
result = []
|
||
for ts_code in set(ths) | set(dc):
|
||
ths_rank = ths.get(ts_code)
|
||
dc_rank = dc.get(ts_code)
|
||
current_best = min(value for value in (ths_rank, dc_rank) if value is not None)
|
||
previous_candidates = [
|
||
value for value in (previous_ths_map.get(ts_code), previous_dc_map.get(ts_code))
|
||
if value is not None
|
||
]
|
||
previous_best = min(previous_candidates) if previous_candidates else None
|
||
score = (101 - (ths_rank or 101)) * 0.5 + (201 - (dc_rank or 201)) * 0.25
|
||
result.append(
|
||
{
|
||
"trade_date": trade_date,
|
||
"ts_code": ts_code,
|
||
"ths_rank": ths_rank,
|
||
"dc_rank": dc_rank,
|
||
"combined_score": round(score, 2),
|
||
"rank_change": (
|
||
previous_best - current_best
|
||
if previous_best is not None
|
||
else min(30, max(0, 31 - current_best))
|
||
if previous_ths_map or previous_dc_map else 0
|
||
),
|
||
"dual_source": bool(ths_rank and dc_rank),
|
||
}
|
||
)
|
||
return result
|
||
|
||
|
||
class FactorDataService:
|
||
def __init__(self, database: ReviewDatabase, client: TushareClient) -> None:
|
||
self.database = database
|
||
self.client = client
|
||
|
||
def sync(self, requested_date: str, lookback: int = 45) -> dict[str, Any]:
|
||
lookback = max(25, min(260, int(lookback)))
|
||
trade_date, _ = self.client.resolve_trade_context(requested_date)
|
||
end = datetime.strptime(trade_date, "%Y%m%d")
|
||
start = (end - timedelta(days=max(100, lookback * 2 + 20))).strftime("%Y%m%d")
|
||
calendar = self.client.query(
|
||
"trade_cal",
|
||
{"exchange": "SSE", "start_date": start, "end_date": trade_date, "is_open": 1},
|
||
"cal_date,is_open",
|
||
)
|
||
dates = sorted(row["cal_date"] for row in calendar if row.get("is_open") == 1)[-lookback:]
|
||
existing = set(self.database.factor_dates(trade_date, lookback + 10))
|
||
dates_to_fetch = [value for value in dates if value not in existing or value == trade_date]
|
||
auction_source_dates = dates[-min(80, len(dates)):]
|
||
existing_auction = set(self.database.auction_factor_dates(trade_date, 90))
|
||
auction_dates_to_fetch = [
|
||
value for value in auction_source_dates
|
||
if value not in existing_auction or value == trade_date
|
||
]
|
||
long_calendar = self.client.query(
|
||
"trade_cal",
|
||
{
|
||
"exchange": "SSE",
|
||
"start_date": datetime(end.year - 5, 1, 1).strftime("%Y%m%d"),
|
||
"end_date": trade_date,
|
||
"is_open": 1,
|
||
},
|
||
"cal_date,is_open",
|
||
)
|
||
last_open_by_year: dict[str, str] = {}
|
||
last_open_by_month: dict[str, str] = {}
|
||
for row in long_calendar:
|
||
if row.get("is_open") == 1 and row.get("cal_date"):
|
||
value = str(row["cal_date"])
|
||
last_open_by_year[value[:4]] = max(last_open_by_year.get(value[:4], ""), value)
|
||
last_open_by_month[value[:6]] = max(last_open_by_month.get(value[:6], ""), value)
|
||
valuation_dates = set(dates[-min(80, len(dates)):])
|
||
valuation_dates.update(last_open_by_year.values())
|
||
valuation_dates.update(last_open_by_month.values())
|
||
existing_indicators = set(self.database.daily_indicator_dates(trade_date, 500))
|
||
indicator_dates_to_fetch = sorted(
|
||
value for value in valuation_dates if value not in existing_indicators or value == trade_date
|
||
)
|
||
|
||
master = self.client.query(
|
||
"stock_basic",
|
||
{"list_status": "L"},
|
||
"ts_code,name,industry,market,list_date",
|
||
)
|
||
master_count = self.database.upsert_stock_master(master)
|
||
bar_count = 0
|
||
for current_date in dates_to_fetch:
|
||
rows = self.client.query(
|
||
"daily",
|
||
{"trade_date": current_date},
|
||
"ts_code,trade_date,open,high,low,close,pct_chg,vol,amount",
|
||
)
|
||
bar_count += self.database.upsert_daily_bars(rows)
|
||
|
||
indicator_count = 0
|
||
for current_date in indicator_dates_to_fetch:
|
||
indicators = self.client.query(
|
||
"daily_basic",
|
||
{"trade_date": current_date},
|
||
"ts_code,trade_date,turnover_rate,volume_ratio,total_mv,circ_mv,"
|
||
"pe_ttm,pb,ps_ttm,dv_ttm",
|
||
)
|
||
indicator_count += self.database.upsert_daily_indicators(indicators)
|
||
|
||
notices = []
|
||
benchmark_count = 0
|
||
try:
|
||
benchmark_rows = self.client.query(
|
||
"index_daily",
|
||
{"ts_code": "000300.SH", "start_date": dates[0], "end_date": trade_date},
|
||
"ts_code,trade_date,close,pct_chg",
|
||
)
|
||
benchmark_count = self.database.upsert_benchmark_bars(benchmark_rows)
|
||
except TushareError as exc:
|
||
notices.append(f"沪深300基准暂不可用：{exc}")
|
||
fundamental_count = 0
|
||
existing_periods = set(self.database.fundamental_periods())
|
||
for period in _quarter_periods(trade_date, 9):
|
||
if period in existing_periods and period < trade_date[:4] + "0101":
|
||
continue
|
||
try:
|
||
rows = self.client.query(
|
||
"fina_indicator_vip",
|
||
{"period": period},
|
||
"ts_code,ann_date,end_date,roe,roa,roic,grossprofit_margin,"
|
||
"netprofit_yoy,or_yoy,ocf_to_opincome",
|
||
)
|
||
except TushareError as exc:
|
||
notices.append(f"财务质量接口不可用：{exc}")
|
||
break
|
||
published = [
|
||
row for row in rows
|
||
if not row.get("ann_date") or str(row.get("ann_date")) <= trade_date
|
||
]
|
||
published.sort(key=lambda row: str(row.get("ann_date") or ""))
|
||
fundamental_count += self.database.upsert_fundamental_indicators(published)
|
||
auction_count = 0
|
||
auction_dates = 0
|
||
for current_date in auction_dates_to_fetch:
|
||
try:
|
||
auction_rows = self.client.query(
|
||
"stk_auction",
|
||
{"trade_date": current_date},
|
||
"ts_code,trade_date,vol,price,amount,pre_close,turnover_rate,volume_ratio,float_share",
|
||
)
|
||
if auction_rows:
|
||
auction_count += self.database.upsert_auction_factors(auction_rows)
|
||
auction_dates += 1
|
||
except TushareError as exc:
|
||
notices.append(f"竞价因子接口不可用：{exc}")
|
||
break
|
||
moneyflow_count = 0
|
||
moneyflow_dates = 0
|
||
for current_date in dates[-min(5, len(dates)):]:
|
||
try:
|
||
moneyflow = self.client.query(
|
||
"moneyflow",
|
||
{"trade_date": current_date},
|
||
"ts_code,trade_date,buy_sm_amount,sell_sm_amount,buy_md_amount,sell_md_amount,"
|
||
"buy_lg_amount,sell_lg_amount,buy_elg_amount,sell_elg_amount,net_mf_amount",
|
||
)
|
||
moneyflow_count += self.database.upsert_moneyflow(moneyflow)
|
||
if moneyflow:
|
||
moneyflow_dates += 1
|
||
except TushareError as exc:
|
||
notices.append(f"资金流接口不可用：{exc}")
|
||
break
|
||
|
||
earnings_count = 0
|
||
forecasts: list[dict[str, Any]] = []
|
||
expresses: list[dict[str, Any]] = []
|
||
for period in _quarter_periods(trade_date, 5):
|
||
try:
|
||
forecast_rows = self.client.query(
|
||
"forecast_vip",
|
||
{"period": period},
|
||
"ts_code,ann_date,end_date,net_profit_min,net_profit_max,last_parent_net,p_change_min,p_change_max",
|
||
)
|
||
express_rows = self.client.query(
|
||
"express_vip",
|
||
{"period": period},
|
||
"ts_code,ann_date,end_date,n_income,yoy_net_profit,yoy_sales",
|
||
)
|
||
except TushareError as exc:
|
||
notices.append(f"业绩事件接口不可用：{exc}")
|
||
break
|
||
forecasts.extend(forecast_rows)
|
||
expresses.extend(express_rows)
|
||
if forecasts and expresses:
|
||
earnings_count = self.database.upsert_earnings_events(
|
||
_earnings_event_rows(forecasts, expresses, trade_date)
|
||
)
|
||
|
||
popularity_count = 0
|
||
previous_trade_date = dates[-2] if len(dates) >= 2 else ""
|
||
try:
|
||
ths_rows = self.client.query("ths_hot", {"trade_date": trade_date})
|
||
dc_rows = self.client.query("dc_hot", {"trade_date": trade_date})
|
||
previous_ths = (
|
||
self.client.query("ths_hot", {"trade_date": previous_trade_date})
|
||
if previous_trade_date else []
|
||
)
|
||
previous_dc = (
|
||
self.client.query("dc_hot", {"trade_date": previous_trade_date})
|
||
if previous_trade_date else []
|
||
)
|
||
popularity_count = self.database.upsert_popularity_factors(
|
||
_popularity_factor_rows(
|
||
trade_date, ths_rows, dc_rows, previous_ths, previous_dc
|
||
)
|
||
)
|
||
except TushareError as exc:
|
||
notices.append(f"人气榜因子不可用：{exc}")
|
||
|
||
institution_count = 0
|
||
try:
|
||
institution_rows = self.client.query(
|
||
"top_inst",
|
||
{"trade_date": trade_date},
|
||
"trade_date,ts_code,exalter,buy,sell,net_buy,side,reason",
|
||
)
|
||
institution_count = self.database.upsert_lhb_institutions(institution_rows)
|
||
except TushareError as exc:
|
||
notices.append(f"机构席位明细不可用：{exc}")
|
||
|
||
return {
|
||
"trade_date": trade_date,
|
||
"calendar_dates": len(dates),
|
||
"fetched_dates": len(dates_to_fetch),
|
||
"stocks": master_count,
|
||
"bars": bar_count,
|
||
"benchmark_bars": benchmark_count,
|
||
"indicators": indicator_count,
|
||
"indicator_dates": len(indicator_dates_to_fetch),
|
||
"fundamentals": fundamental_count,
|
||
"moneyflow": moneyflow_count,
|
||
"moneyflow_dates": moneyflow_dates,
|
||
"auction_rows": auction_count,
|
||
"auction_dates": auction_dates,
|
||
"earnings_events": earnings_count,
|
||
"popularity_rows": popularity_count,
|
||
"institution_rows": institution_count,
|
||
"notice": "；".join(notices),
|
||
}
|