leefer/xiaobaifupan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rebuild(stage-9): deliver deterministic intelligent screening

Browse Source
This commit is contained in:
leefer
2026-07-30 05:15:17 +08:00
parent 6cb52e864a
commit 158257ebb8
46 changed files with 7322 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
next/backend/data/gateway.py
+17 -3
View File
@@ -20,6 +20,7 @@ from backend.data.policy import DataSourcePolicy
from backend.data.providers.base import MarketDataProvider, ProviderError
from backend.data.quality import DataQualityError, require_quality
from backend.data.repository import MarketRepository
from backend.data.screener_gateway import assemble_screener_inputs
from backend.database.connection import Database
SHANGHAI = ZoneInfo("Asia/Shanghai")
@@ -108,9 +109,7 @@ class DataGateway:
row = self._repository.latest_summary(connection, context.actual_date)
return {"context": context, "values": json.loads(str(row["payload_json"])) if row else None}
def snapshot_inputs(
self, trade_date: str, previous_trade_date: str
) -> dict[str, Any]:
def snapshot_inputs(self, trade_date: str, previous_trade_date: str) -> dict[str, Any]:
provider = self._provider(DataSource.TUSHARE)
self._policy.assert_allowed(provider.source, DataUsage.CALCULATION)
return provider.snapshot_inputs(trade_date, previous_trade_date)
@@ -136,6 +135,21 @@ class DataGateway:
self._policy.assert_allowed(provider.source, DataUsage.CALCULATION)
return provider.market_insight(kind, trade_date, previous_trade_date, identifier)
def screener_inputs(
self, trade_date: str, history_days: int = 260
) -> tuple[dict[str, Any], dict[str, float], list[str]]:
dates = self.trading_dates(trade_date, history_days)
if len(dates) < 21:
raise MarketDataUnavailable("历史交易日不足21日，无法生成选股因子")
chronological = tuple(reversed(dates))
provider = self._provider(DataSource.TUSHARE)
self._policy.assert_allowed(provider.source, DataUsage.CALCULATION)
raw = provider.screener_inputs(chronological)
inputs, coverage = assemble_screener_inputs(
self._database, self._repository, trade_date, chronological, raw
)
return inputs, coverage, [provider.source.value, DataSource.LOCAL.value]
def dynamic_auction(
self, identifiers: tuple[str, ...], start_time: str, end_time: str
) -> ProviderResult:
next/backend/data/policy.py
+1
View File
@@ -32,5 +32,6 @@ class DataSourcePolicy:
("realtime_quote", DataUsage.CALCULATION): (DataSource.IFIND, DataSource.TUSHARE),
("market_insight", DataUsage.CALCULATION): (DataSource.TUSHARE,),
("dynamic_auction", DataUsage.CALCULATION): (DataSource.IFIND,),
("screener_factors", DataUsage.CALCULATION): (DataSource.TUSHARE,),
}
return routes.get((dataset, usage), ())
next/backend/data/providers/base.py
+2
View File
@@ -40,3 +40,5 @@ class MarketDataProvider(Protocol):
def realtime_snapshots(
self, identifiers: tuple[str, ...], start_time: str, end_time: str
) -> ProviderResult: ...
def screener_inputs(self, trade_dates: tuple[str, ...]) -> dict[str, ProviderResult | None]: ...
next/backend/data/providers/eastmoney.py
+3
View File
@@ -116,6 +116,9 @@ class EastmoneyProvider:
) -> ProviderResult:
raise ProviderError("The display provider cannot supply calculation snapshots")
def screener_inputs(self, trade_dates: tuple[str, ...]) -> dict[str, ProviderResult | None]:
raise ProviderError("The display provider cannot supply screener factors")
@staticmethod
def _secid(entity_type: str, identifier: str) -> str:
if entity_type == "index" and identifier in INDEX_CODES:
next/backend/data/providers/ifind.py
+8 -7
View File
@@ -120,9 +120,7 @@ class IfindProvider:
},
)
rows.extend(_result(payload, "mixed", "not_applicable", SnapshotState.REALTIME).rows)
covered = {
str(row.get("thscode") or "") for row in rows if row.get("thscode")
}
covered = {str(row.get("thscode") or "") for row in rows if row.get("thscode")}
return ProviderResult(
tuple(rows),
ObservationMetadata(
@@ -137,6 +135,9 @@ class IfindProvider:
),
)
def screener_inputs(self, trade_dates: tuple[str, ...]) -> dict[str, ProviderResult | None]:
raise ProviderError("iFinD尚未批准用于盘后因子批量计算")
def _request(self, endpoint: str, body: dict[str, Any]) -> dict[str, Any]:
if not self.configured:
raise ProviderError("实时行情服务尚未配置")
@@ -183,9 +184,7 @@ class IfindProvider:
def _auth_error(self, payload: dict[str, Any]) -> bool:
message = str(payload.get("errmsg") or payload.get("message") or "").casefold()
return (
_error_code(payload) in self.auth_error_codes
or "token" in message
or "鉴权" in message
_error_code(payload) in self.auth_error_codes or "token" in message or "鉴权" in message
)
def _refresh(self) -> str:
@@ -246,7 +245,9 @@ def _result(
row = {
key: values[index]
if isinstance(values, list) and index < len(values)
else values if index == 0 else None
else values
if index == 0
else None
for key, values in columns.items()
}
if isinstance(times, list) and index < len(times):
next/backend/data/providers/tushare.py
+161 -12
View File
@@ -157,8 +157,7 @@ class TushareProvider:
code = str(row.get("ts_code") or "")
current = deduplicated.get(code)
if code and (
current is None
or str(row.get("in_date") or "") > str(current.get("in_date") or "")
current is None or str(row.get("in_date") or "") > str(current.get("in_date") or "")
):
deduplicated[code] = row
if not deduplicated:
@@ -222,8 +221,7 @@ class TushareProvider:
"daily": self._optional_query(
"ths_daily",
{"trade_date": current},
"ts_code,trade_date,open,high,low,close,pre_close,pct_change,"
"vol,turnover_rate",
"ts_code,trade_date,open,high,low,close,pre_close,pct_change,vol,turnover_rate",
),
"hot": self._optional_query("ths_hot", {"trade_date": current}, ""),
}
@@ -244,12 +242,8 @@ class TushareProvider:
return {
"ths": self._optional_query("ths_hot", {"trade_date": current}, ""),
"dc": self._optional_query("dc_hot", {"trade_date": current}, ""),
"previous_ths": self._optional_query(
"ths_hot", {"trade_date": previous}, ""
),
"previous_dc": self._optional_query(
"dc_hot", {"trade_date": previous}, ""
),
"previous_ths": self._optional_query("ths_hot", {"trade_date": previous}, ""),
"previous_dc": self._optional_query("dc_hot", {"trade_date": previous}, ""),
}
if kind == "dragon-list":
return {
@@ -278,6 +272,147 @@ class TushareProvider:
) -> ProviderResult:
raise ProviderError("Tushare不提供动态竞价快照")
def screener_inputs(self, trade_dates: tuple[str, ...]) -> dict[str, ProviderResult | None]:
if len(trade_dates) < 21:
raise ProviderError("选股因子至少需要21个交易日")
compact_dates = tuple(_compact(value) for value in trade_dates)
current = compact_dates[-1]
quarters = _quarter_periods(current, 5)
years = tuple(f"{int(current[:4]) - offset}1231" for offset in range(1, 6))
return {
"directory": self._optional_query(
"stock_basic",
{"exchange": "", "list_status": "L"},
"ts_code,symbol,name,industry,market,list_date,list_status",
),
"industry": self._optional_query(
"index_member_all",
{"is_new": "Y"},
"l1_code,l1_name,l2_code,l2_name,ts_code,name,in_date,out_date,is_new",
),
"daily": self._series_query(
"daily",
compact_dates,
"ts_code,trade_date,open,high,low,close,pre_close,pct_chg,vol,amount",
),
"daily_basic": self._series_query(
"daily_basic",
compact_dates[-5:],
"ts_code,trade_date,turnover_rate,volume_ratio,pe_ttm,pb,ps_ttm,dv_ttm,"
"total_mv,circ_mv",
),
"moneyflow": self._series_query(
"moneyflow",
compact_dates[-5:],
"ts_code,trade_date,buy_lg_amount,sell_lg_amount,buy_elg_amount,"
"sell_elg_amount,net_mf_amount",
),
"benchmark": self._optional_query(
"index_daily",
{
"ts_code": "000300.SH",
"start_date": compact_dates[0],
"end_date": current,
},
"ts_code,trade_date,close,pct_chg",
),
"fundamentals": self._period_query(
"fina_indicator",
quarters,
"ts_code,ann_date,end_date,roe,roa,roic,grossprofit_margin,"
"netprofit_yoy,or_yoy,ocf_to_or",
),
"dividends": self._period_query(
"dividend",
years,
"ts_code,end_date,ann_date,div_proc,cash_div_tax,ex_date",
parameter="end_date",
),
"auction": self._optional_query(
"stk_auction",
{"trade_date": current},
"ts_code,trade_date,price,pre_close,amount,turnover_rate,volume_ratio",
),
"limit_events": self._series_query(
"limit_list_d",
compact_dates[-80:],
"trade_date,ts_code,name,limit_type,limit_times",
empty_is_complete=True,
),
"forecast": self._period_query(
"forecast",
quarters,
"ts_code,ann_date,end_date,type,p_change_min,p_change_max,"
"net_profit_min,net_profit_max,last_parent_net",
),
"express": self._period_query(
"express",
quarters,
"ts_code,ann_date,end_date,revenue,operate_profit,total_profit,n_income,"
"total_assets,diluted_roe,yoy_net_profit",
),
}
def _series_query(
self,
api_name: str,
dates: tuple[str, ...],
fields: str,
*,
empty_is_complete: bool = False,
) -> ProviderResult | None:
rows: list[dict[str, Any]] = []
completed = 0
for trade_date in dates:
try:
result = self._query(
api_name,
{"trade_date": trade_date},
fields,
unit="mixed",
empty_is_complete=empty_is_complete,
)
except ProviderError:
continue
rows.extend(result.rows)
completed += 1
if completed == 0:
return None
return ProviderResult(
tuple(rows),
_metadata(self.source, "mixed", completed / len(dates)),
)
def _period_query(
self,
api_name: str,
periods: tuple[str, ...],
fields: str,
*,
parameter: str = "period",
) -> ProviderResult | None:
rows: list[dict[str, Any]] = []
completed = 0
for period in periods:
try:
result = self._query(
api_name,
{parameter: period},
fields,
unit="mixed",
empty_is_complete=True,
)
except ProviderError:
continue
rows.extend(result.rows)
completed += 1
if completed == 0:
return None
return ProviderResult(
tuple(rows),
_metadata(self.source, "mixed", completed / len(periods)),
)
def _optional_query(
self, api_name: str, params: dict[str, Any], fields: str
) -> ProviderResult | None:
@@ -295,8 +430,7 @@ class TushareProvider:
def _membership_rows(self, params: dict[str, str]) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
fields = (
"l1_code,l1_name,l2_code,l2_name,l3_code,l3_name,"
"ts_code,name,in_date,out_date,is_new"
"l1_code,l1_name,l2_code,l2_name,l3_code,l3_name,ts_code,name,in_date,out_date,is_new"
)
for is_new in ("Y", "N"):
result = self._query(
@@ -378,6 +512,21 @@ def _display(value: str) -> str:
return f"{compact[:4]}-{compact[4:6]}-{compact[6:]}"
def _quarter_periods(through: str, count: int) -> tuple[str, ...]:
year = int(through[:4])
quarter = (int(through[4:6]) - 1) // 3
periods = []
for offset in range(count + 4):
index = year * 4 + quarter - offset
period_year, period_quarter = divmod(index, 4)
period = f"{period_year}{('0331', '0630', '0930', '1231')[period_quarter]}"
if period <= through:
periods.append(period)
if len(periods) == count:
break
return tuple(reversed(periods))
def _active_on(row: dict[str, Any], trade_date: str) -> bool:
start = str(row.get("in_date") or "")
end = str(row.get("out_date") or "")
next/backend/data/screener_gateway.py
+272
View File
@@ -0,0 +1,272 @@
from __future__ import annotations
import json
from typing import Any
from backend.data.contracts import ProviderResult
from backend.data.repository import MarketRepository
from backend.database.connection import Database
def assemble_screener_inputs(
database: Database,
repository: MarketRepository,
trade_date: str,
trade_dates: tuple[str, ...],
raw: dict[str, ProviderResult | None],
) -> tuple[dict[str, Any], dict[str, float]]:
inputs = {
key: [_normalize_row(row) for row in value.rows]
for key, value in raw.items()
if isinstance(value, ProviderResult)
}
inputs["earnings"] = _earnings_events(
inputs.pop("forecast", []), inputs.pop("express", []), trade_date
)
popularity, popularity_coverage = _popularity(database, repository, trade_date)
institutions, institution_coverage = _institutions(database, repository, trade_date)
local_limits, local_limit_coverage = _local_limit_events(
database, repository, trade_dates[-80:]
)
inputs["popularity"] = popularity
inputs["institutions"] = institutions
inputs["limit_events"] = _merge_events(inputs.get("limit_events", []), local_limits)
directory_count = len(inputs.get("directory", []))
daily_current = _on_date(inputs.get("daily", []), trade_date)
current_count = len({str(row.get("ts_code") or "") for row in daily_current})
expected = max(directory_count, current_count, 1)
basic_current = _on_date(inputs.get("daily_basic", []), trade_date)
fundamentals = _latest_announced(inputs.get("fundamentals", []), trade_date)
moneyflow_counts = _code_date_counts(inputs.get("moneyflow", []))
industry_codes = {
str(row.get("ts_code") or "")
for row in inputs.get("industry", [])
if _active_member(row, trade_date)
}
auction_count = len({str(row.get("ts_code") or "") for row in inputs.get("auction", [])})
limit_result = raw.get("limit_events")
provider_limit_coverage = (
limit_result.metadata.coverage if isinstance(limit_result, ProviderResult) else 0
)
forecast = raw.get("forecast")
express = raw.get("express")
earnings_coverage = min(
forecast.metadata.coverage if isinstance(forecast, ProviderResult) else 0,
express.metadata.coverage if isinstance(express, ProviderResult) else 0,
)
coverage = {
"market": min(current_count / expected, 1),
"valuation": min(
len({str(row.get("ts_code") or "") for row in basic_current}) / expected, 1
),
"financial": min(len(fundamentals) / expected, 1),
"moneyflow": min(
sum(count >= min(5, len(trade_dates)) for count in moneyflow_counts.values())
/ expected,
1,
),
"industry": min(len(industry_codes) / expected, 1),
"auction": min(auction_count / expected, 1),
"popularity": popularity_coverage,
"institutions": institution_coverage,
"earnings": earnings_coverage,
"limit_events": max(provider_limit_coverage, local_limit_coverage),
}
return inputs, coverage
def _popularity(
database: Database, repository: MarketRepository, trade_date: str
) -> tuple[list[dict[str, Any]], float]:
with database.read() as connection:
row = repository.insight_snapshot(connection, "popularity", trade_date)
if row is None:
return [], 0.0
payload = json.loads(str(row["payload_json"]))
return [
{
"ts_code": str(item.get("identifier") or ""),
"combined_score": item.get("score"),
"rank_change": item.get("rank_change"),
"dual_source": bool(item.get("dual_source")),
}
for item in payload.get("combined") or []
if item.get("identifier")
], float(row["coverage"])
def _institutions(
database: Database, repository: MarketRepository, trade_date: str
) -> tuple[list[dict[str, Any]], float]:
with database.read() as connection:
row = repository.insight_snapshot(connection, "dragon-list", trade_date)
if row is None:
return [], 0.0
payload = json.loads(str(row["payload_json"]))
seats = payload.get("seats")
if seats is None:
return [], 0.0
grouped: dict[str, dict[str, float]] = {}
for item in seats:
identifier = str(item.get("ts_code") or "")
if not identifier or "机构" not in str(item.get("exalter") or ""):
continue
target = grouped.setdefault(identifier, {"net": 0.0, "count": 0})
target["net"] += float(item.get("net_buy") or 0) / 1_000_000
target["count"] += 1
return [
{
"ts_code": identifier,
"net_buy_million": values["net"],
"seat_count": int(values["count"]),
}
for identifier, values in grouped.items()
], float(row["coverage"])
def _local_limit_events(
database: Database,
repository: MarketRepository,
trade_dates: tuple[str, ...],
) -> tuple[list[dict[str, Any]], float]:
if not trade_dates:
return [], 0.0
with database.read() as connection:
rows = repository.summaries(connection, trade_dates[-1], len(trade_dates))
expected = set(trade_dates)
covered: set[str] = set()
events = []
for row in rows:
trade_date = str(row["trade_date"])
if trade_date not in expected:
continue
covered.add(trade_date)
payload = json.loads(str(row["payload_json"]))
for key, event in (("limits", "U"), ("broken", "Z"), ("down_limits", "D")):
events.extend(
{
"trade_date": trade_date,
"ts_code": str(item["identifier"]),
"limit_type": event,
}
for item in payload.get(key) or []
if item.get("identifier")
)
return events, len(covered) / len(expected)
def _normalize_row(row: dict[str, Any]) -> dict[str, Any]:
result = dict(row)
for field in ("trade_date", "ann_date", "end_date", "ex_date", "in_date", "out_date"):
value = str(result.get(field) or "")
if len(value) == 8 and value.isdigit():
result[field] = f"{value[:4]}-{value[4:6]}-{value[6:]}"
if result.get("price") is not None and result.get("pre_close") is not None:
price = _number(result.get("price"))
previous = _number(result.get("pre_close"))
result["change"] = (price / previous - 1) * 100 if price is not None and previous else None
amount = _number(result.get("amount"))
result["amount_million"] = amount / 1_000_000 if amount is not None else None
return result
def _earnings_events(
forecasts: list[dict[str, Any]], expresses: list[dict[str, Any]], through: str
) -> list[dict[str, Any]]:
forecast_map: dict[tuple[str, str], dict[str, Any]] = {}
for row in forecasts:
key = (str(row.get("ts_code") or ""), str(row.get("end_date") or ""))
announced = str(row.get("ann_date") or "")
current = forecast_map.get(key)
if (
all(key)
and announced
and announced <= through
and (current is None or announced > str(current.get("ann_date") or ""))
):
forecast_map[key] = row
result = []
for row in expresses:
key = (str(row.get("ts_code") or ""), str(row.get("end_date") or ""))
announced = str(row.get("ann_date") or "")
forecast = forecast_map.get(key)
if forecast is None or not announced or announced > through:
continue
values = [
value
for value in (
_number(forecast.get("net_profit_min")),
_number(forecast.get("net_profit_max")),
)
if value is not None
]
expected = sum(values) / len(values) if values else None
actual = _number(row.get("n_income"))
if expected in (None, 0) or actual is None:
continue
if abs(actual) > max(abs(expected), 1) * 100:
actual /= 10000
result.append(
{
"ts_code": key[0],
"end_date": key[1],
"ann_date": announced,
"surprise_pct": (actual / expected - 1) * 100,
}
)
return result
def _on_date(rows: list[dict[str, Any]], trade_date: str) -> list[dict[str, Any]]:
return [row for row in rows if str(row.get("trade_date") or "") == trade_date]
def _latest_announced(rows: list[dict[str, Any]], through: str) -> dict[str, dict[str, Any]]:
result: dict[str, dict[str, Any]] = {}
for row in rows:
identifier = str(row.get("ts_code") or "")
announced = str(row.get("ann_date") or "")
current = result.get(identifier)
if (
identifier
and announced
and announced <= through
and (current is None or announced > str(current.get("ann_date") or ""))
):
result[identifier] = row
return result
def _code_date_counts(rows: list[dict[str, Any]]) -> dict[str, int]:
values: dict[str, set[str]] = {}
for row in rows:
identifier = str(row.get("ts_code") or "")
if identifier:
values.setdefault(identifier, set()).add(str(row.get("trade_date") or ""))
return {identifier: len(dates) for identifier, dates in values.items()}
def _active_member(row: dict[str, Any], trade_date: str) -> bool:
start = str(row.get("in_date") or "")
end = str(row.get("out_date") or "")
return (not start or start <= trade_date) and (not end or end > trade_date)
def _merge_events(
provider: list[dict[str, Any]], local: list[dict[str, Any]]
) -> list[dict[str, Any]]:
merged = {
(str(row.get("trade_date") or ""), str(row.get("ts_code") or "")): row for row in local
}
for row in provider:
merged[(str(row.get("trade_date") or ""), str(row.get("ts_code") or ""))] = row
return list(merged.values())
def _number(value: Any) -> float | None:
try:
result = float(value)
return result if result == result else None
except (TypeError, ValueError):
return None