Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
89b8d33de7 | ||
|
|
d9ee725744 | ||
|
|
1cb2745867 | ||
|
|
f27471238a | ||
|
|
6d7a839202 | ||
|
|
fc1e5b89e4 | ||
|
|
cf206c7de9 | ||
|
|
09a935aac4 | ||
|
|
8e94c7b429 | ||
|
|
7ad445bc9f |
@@ -9,6 +9,7 @@ __pycache__/
|
||||
*.log
|
||||
runtime/
|
||||
data/cache/
|
||||
data/backups/
|
||||
data/private-mentor-skills/
|
||||
data/*.db
|
||||
data/*.db-shm
|
||||
|
||||
@@ -41,13 +41,16 @@ class DashboardMixin:
|
||||
raise TushareError(f"No daily data returned for {trade_date}")
|
||||
|
||||
notices: list[str] = []
|
||||
limit_data_source = "official"
|
||||
try:
|
||||
limit_rows = self._load_limit_lists(trade_date)
|
||||
previous_limit_rows = self._load_limit_type(previous_trade_date, "U")
|
||||
if not limit_rows:
|
||||
limit_data_source = "derived"
|
||||
notices.append("涨跌停高级接口当日数据尚未更新,已使用日线数据推算。")
|
||||
limit_rows = self._derive_limits(trade_date, daily)
|
||||
except TushareError as exc:
|
||||
limit_data_source = "derived"
|
||||
notices.append(f"涨跌停高级接口不可用,已使用日线数据推算:{exc}")
|
||||
limit_rows = self._derive_limits(trade_date, daily)
|
||||
previous_daily = self._load_daily(previous_trade_date)
|
||||
@@ -79,6 +82,7 @@ class DashboardMixin:
|
||||
"trade_date": _display_date(trade_date),
|
||||
"previous_trade_date": _display_date(previous_trade_date),
|
||||
"source": "tushare",
|
||||
"limit_data_source": limit_data_source,
|
||||
"updated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
"notice": ";".join(notices),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
"""Auditable recent-trading-day snapshot backfill helpers.
|
||||
|
||||
Planning and backup stay free of provider imports so feature boundary tests remain green.
|
||||
The service layer supplies open trading dates from the live calendar and executes sync.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
MAX_RANGE_TRADING_DAYS = 15
|
||||
MAX_RECENT_TRADING_DAYS = 60
|
||||
DEFAULT_RECENT_TRADING_DAYS = 60
|
||||
|
||||
# Tables touched by a successful historical dashboard sync. User / token / model
|
||||
# tables must never appear here.
|
||||
SNAPSHOT_BACKFILL_WRITE_TABLES = frozenset(
|
||||
{
|
||||
"dashboard_snapshots",
|
||||
"data_snapshots",
|
||||
"sync_runs",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def clamp_recent_lookback(lookback: int) -> int:
|
||||
value = int(lookback)
|
||||
if value < 1:
|
||||
raise ValueError("回补交易日数量至少为 1。")
|
||||
if value > MAX_RECENT_TRADING_DAYS:
|
||||
raise ValueError(f"单次最多回补最近 {MAX_RECENT_TRADING_DAYS} 个交易日。")
|
||||
return value
|
||||
|
||||
|
||||
def calendar_window_start(end_date: str, lookback: int) -> str:
|
||||
"""Natural-day lower bound large enough to cover lookback open sessions."""
|
||||
end = datetime.strptime(end_date, "%Y%m%d").date()
|
||||
span = max(40, int(lookback * 2) + 20)
|
||||
return (end - timedelta(days=span)).strftime("%Y%m%d")
|
||||
|
||||
|
||||
def select_open_trade_dates(
|
||||
calendar_rows: Iterable[dict[str, Any]],
|
||||
end_date: str,
|
||||
lookback: int,
|
||||
) -> list[str]:
|
||||
"""Pick the last ``lookback`` open SSE sessions on or before ``end_date``."""
|
||||
lookback = clamp_recent_lookback(lookback)
|
||||
end = normalize_compact_date(end_date)
|
||||
open_dates = sorted(
|
||||
{
|
||||
normalize_compact_date(str(row.get("cal_date") or ""))
|
||||
for row in calendar_rows
|
||||
if int(row.get("is_open") or 0) == 1 and row.get("cal_date")
|
||||
}
|
||||
)
|
||||
open_dates = [item for item in open_dates if item <= end]
|
||||
if not open_dates:
|
||||
raise ValueError("交易日历未返回可用交易日,请检查行情 Token。")
|
||||
return open_dates[-lookback:]
|
||||
|
||||
|
||||
def select_open_trade_dates_in_range(
|
||||
calendar_rows: Iterable[dict[str, Any]],
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
*,
|
||||
maximum: int = MAX_RANGE_TRADING_DAYS,
|
||||
) -> tuple[list[str], list[str]]:
|
||||
"""Return (open_dates, skipped_non_trading_days) inside an inclusive range."""
|
||||
start = normalize_compact_date(start_date)
|
||||
end = normalize_compact_date(end_date)
|
||||
if start > end:
|
||||
raise ValueError("开始日期不能晚于结束日期。")
|
||||
open_set = {
|
||||
normalize_compact_date(str(row.get("cal_date") or ""))
|
||||
for row in calendar_rows
|
||||
if int(row.get("is_open") or 0) == 1 and row.get("cal_date")
|
||||
}
|
||||
open_dates: list[str] = []
|
||||
skipped: list[str] = []
|
||||
cursor = datetime.strptime(start, "%Y%m%d").date()
|
||||
last = datetime.strptime(end, "%Y%m%d").date()
|
||||
while cursor <= last:
|
||||
compact = cursor.strftime("%Y%m%d")
|
||||
if compact in open_set:
|
||||
open_dates.append(compact)
|
||||
else:
|
||||
skipped.append(compact)
|
||||
cursor += timedelta(days=1)
|
||||
if len(open_dates) > maximum:
|
||||
raise ValueError(f"单次最多回补 {maximum} 个交易日。")
|
||||
return open_dates, skipped
|
||||
|
||||
|
||||
def classify_snapshot_coverage(
|
||||
trade_dates: list[str],
|
||||
existing_dates: Iterable[str],
|
||||
) -> dict[str, Any]:
|
||||
present_set = {
|
||||
normalize_compact_date(item)
|
||||
for item in existing_dates
|
||||
if item
|
||||
}
|
||||
present = [item for item in trade_dates if item in present_set]
|
||||
missing = [item for item in trade_dates if item not in present_set]
|
||||
return {
|
||||
"trade_dates": list(trade_dates),
|
||||
"present": present,
|
||||
"missing": missing,
|
||||
"present_count": len(present),
|
||||
"missing_count": len(missing),
|
||||
}
|
||||
|
||||
|
||||
def create_sqlite_backup(
|
||||
source_path: Path,
|
||||
backup_dir: Path,
|
||||
*,
|
||||
label: str = "pre-backfill",
|
||||
stamped_at: datetime | None = None,
|
||||
) -> Path:
|
||||
"""Create a timestamped SQLite backup via the native backup API."""
|
||||
source = Path(source_path)
|
||||
if not source.exists():
|
||||
raise FileNotFoundError(f"数据库不存在:{source}")
|
||||
stamp = (stamped_at or datetime.now().astimezone()).strftime("%Y%m%d-%H%M%S")
|
||||
safe_label = "".join(ch if ch.isalnum() or ch in "-_" else "-" for ch in label).strip("-") or "backup"
|
||||
backup_dir = Path(backup_dir)
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
target = backup_dir / f"review-{safe_label}-{stamp}.db"
|
||||
source_conn = sqlite3.connect(f"file:{source}?mode=ro", uri=True)
|
||||
try:
|
||||
target_conn = sqlite3.connect(target)
|
||||
try:
|
||||
source_conn.backup(target_conn)
|
||||
target_conn.commit()
|
||||
finally:
|
||||
target_conn.close()
|
||||
finally:
|
||||
source_conn.close()
|
||||
return target
|
||||
|
||||
|
||||
def display_date(compact: str) -> str:
|
||||
value = normalize_compact_date(compact)
|
||||
return f"{value[:4]}-{value[4:6]}-{value[6:8]}"
|
||||
|
||||
|
||||
def normalize_compact_date(value: str) -> str:
|
||||
compact = str(value or "").replace("-", "").strip()
|
||||
if len(compact) != 8 or not compact.isdigit():
|
||||
raise ValueError("日期格式应为 YYYY-MM-DD。")
|
||||
datetime.strptime(compact, "%Y%m%d")
|
||||
return compact
|
||||
|
||||
|
||||
def build_backfill_audit(
|
||||
*,
|
||||
mode: str,
|
||||
end_date: str,
|
||||
lookback: int | None,
|
||||
coverage: dict[str, Any],
|
||||
skipped_non_trading_days: list[str] | None = None,
|
||||
backup_path: str | None = None,
|
||||
dry_run: bool = False,
|
||||
results: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
results = list(results or [])
|
||||
succeeded = [row for row in results if row.get("status") == "success"]
|
||||
skipped = [row for row in results if row.get("status") == "skipped"]
|
||||
failed = [row for row in results if row.get("status") == "failed"]
|
||||
return {
|
||||
"ok": not failed,
|
||||
"mode": mode,
|
||||
"dry_run": dry_run,
|
||||
"end_date": display_date(end_date),
|
||||
"lookback": lookback,
|
||||
"backup_path": backup_path,
|
||||
"write_tables": sorted(SNAPSHOT_BACKFILL_WRITE_TABLES),
|
||||
"trade_dates": [display_date(item) for item in coverage.get("trade_dates") or []],
|
||||
"present": [display_date(item) for item in coverage.get("present") or []],
|
||||
"missing": [display_date(item) for item in coverage.get("missing") or []],
|
||||
"skipped_non_trading_days": [
|
||||
display_date(item) for item in (skipped_non_trading_days or [])
|
||||
],
|
||||
"present_count": int(coverage.get("present_count") or 0),
|
||||
"missing_count": int(coverage.get("missing_count") or 0),
|
||||
"results": results,
|
||||
"succeeded_count": len(succeeded),
|
||||
"skipped_count": len(skipped),
|
||||
"failed_count": len(failed),
|
||||
"created_dates": [
|
||||
str(row.get("trade_date") or "")
|
||||
for row in succeeded
|
||||
if row.get("action") == "created"
|
||||
],
|
||||
}
|
||||
@@ -227,6 +227,31 @@ class MarketRepositoryMixin:
|
||||
result.append(payload)
|
||||
return result
|
||||
|
||||
def list_snapshot_trade_dates(
|
||||
self,
|
||||
start_date: str = "",
|
||||
end_date: str = "",
|
||||
) -> list[str]:
|
||||
clauses: list[str] = []
|
||||
parameters: list[Any] = []
|
||||
if start_date:
|
||||
clauses.append("trade_date >= ?")
|
||||
parameters.append(start_date)
|
||||
if end_date:
|
||||
clauses.append("trade_date <= ?")
|
||||
parameters.append(end_date)
|
||||
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
||||
with self.connect() as connection:
|
||||
rows = connection.execute(
|
||||
f"""
|
||||
SELECT trade_date FROM dashboard_snapshots
|
||||
{where}
|
||||
ORDER BY trade_date
|
||||
""",
|
||||
parameters,
|
||||
).fetchall()
|
||||
return [str(row["trade_date"]) for row in rows]
|
||||
|
||||
def start_sync(self, trade_date: str, source: str) -> int:
|
||||
started_at = datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
with self.connect() as connection:
|
||||
|
||||
@@ -3,9 +3,11 @@ from __future__ import annotations
|
||||
import copy
|
||||
import re
|
||||
from datetime import date, datetime, time as dt_time, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.bootstrap.config import (
|
||||
DATA_DIR,
|
||||
normalize_date,
|
||||
tushare_code,
|
||||
validate_stock_code,
|
||||
@@ -13,6 +15,17 @@ from backend.bootstrap.config import (
|
||||
)
|
||||
from backend.data.providers.ifind_client import IfindError
|
||||
from backend.data.providers.tushare_client import TushareClient, TushareError
|
||||
from backend.features.market.backfill_history import (
|
||||
DEFAULT_RECENT_TRADING_DAYS,
|
||||
MAX_RANGE_TRADING_DAYS,
|
||||
build_backfill_audit,
|
||||
calendar_window_start,
|
||||
classify_snapshot_coverage,
|
||||
create_sqlite_backup,
|
||||
display_date,
|
||||
select_open_trade_dates,
|
||||
select_open_trade_dates_in_range,
|
||||
)
|
||||
from backend.features.market.charts import ChartDataError
|
||||
from backend.features.market.insights import MarketInsightsService
|
||||
from backend.features.sentiment.engine import SENTIMENT_ENGINE_VERSION
|
||||
@@ -185,6 +198,11 @@ class MarketServiceMixin:
|
||||
raise TushareError("公共行情尚未配置")
|
||||
dashboard = self._tushare_client().dashboard(normalized_date)
|
||||
|
||||
if (dashboard.get("meta") or {}).get("limit_data_source") == "derived":
|
||||
raise TushareError(
|
||||
str((dashboard.get("meta") or {}).get("notice") or "官方涨跌停数据尚未返回")
|
||||
)
|
||||
|
||||
dashboard["meta"]["source"] = source
|
||||
dashboard["meta"]["requested_date"] = self._display_compact_date(normalized_date)
|
||||
dashboard = self._enrich_dashboard_sentiment(dashboard, normalized_date)
|
||||
@@ -890,31 +908,226 @@ class MarketServiceMixin:
|
||||
"intraday": intraday_points,
|
||||
}
|
||||
|
||||
def backfill(self, start_date: str, end_date: str) -> list[dict[str, Any]]:
|
||||
start = datetime.strptime(normalize_date(start_date), "%Y%m%d").date()
|
||||
end = datetime.strptime(normalize_date(end_date), "%Y%m%d").date()
|
||||
if start > end:
|
||||
raise ValueError("开始日期不能晚于结束日期。")
|
||||
weekdays = []
|
||||
current = start
|
||||
while current <= end:
|
||||
if current.weekday() < 5:
|
||||
weekdays.append(current)
|
||||
current += timedelta(days=1)
|
||||
if len(weekdays) > 15:
|
||||
raise ValueError("单次最多回补 15 个工作日。")
|
||||
results = []
|
||||
for day in weekdays:
|
||||
dashboard = self.sync_dashboard(day.strftime("%Y%m%d"))
|
||||
def backfill(
|
||||
self,
|
||||
start_date: str = "",
|
||||
end_date: str = "",
|
||||
*,
|
||||
lookback: int | None = None,
|
||||
dry_run: bool = False,
|
||||
force: bool = False,
|
||||
create_backup: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Backfill dashboard snapshots for real trading days only.
|
||||
|
||||
- Date-range mode keeps the admin UI contract (max 15 open sessions).
|
||||
- Recent mode fills the last N open sessions (default/max 60).
|
||||
Weekends and holidays are reported as skipped non-trading days, not errors.
|
||||
"""
|
||||
if not self.configured:
|
||||
raise ValueError("公共行情尚未配置,无法回补历史快照。")
|
||||
normalized_end = normalize_date(end_date or date.today().isoformat())
|
||||
if lookback is not None or not (start_date and end_date):
|
||||
target_lookback = (
|
||||
DEFAULT_RECENT_TRADING_DAYS if lookback is None else int(lookback)
|
||||
)
|
||||
return self.backfill_recent_trading_days(
|
||||
end_date=normalized_end,
|
||||
lookback=target_lookback,
|
||||
dry_run=dry_run,
|
||||
force=force,
|
||||
create_backup=create_backup,
|
||||
)
|
||||
return self._backfill_date_range(
|
||||
start_date=normalize_date(start_date),
|
||||
end_date=normalized_end,
|
||||
dry_run=dry_run,
|
||||
force=force,
|
||||
create_backup=create_backup,
|
||||
)
|
||||
|
||||
def backfill_recent_trading_days(
|
||||
self,
|
||||
end_date: str = "",
|
||||
lookback: int = DEFAULT_RECENT_TRADING_DAYS,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
force: bool = False,
|
||||
create_backup: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
normalized_end = normalize_date(end_date or date.today().isoformat())
|
||||
trade_dates = self._load_recent_open_trade_dates(normalized_end, lookback)
|
||||
existing = self.database.list_snapshot_trade_dates(
|
||||
trade_dates[0], trade_dates[-1]
|
||||
)
|
||||
coverage = classify_snapshot_coverage(trade_dates, existing)
|
||||
return self._execute_snapshot_backfill(
|
||||
mode="recent",
|
||||
end_date=normalized_end,
|
||||
lookback=lookback,
|
||||
coverage=coverage,
|
||||
skipped_non_trading_days=[],
|
||||
dry_run=dry_run,
|
||||
force=force,
|
||||
create_backup=create_backup,
|
||||
)
|
||||
|
||||
def _backfill_date_range(
|
||||
self,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
force: bool = False,
|
||||
create_backup: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
window_start = calendar_window_start(end_date, MAX_RANGE_TRADING_DAYS)
|
||||
calendar_rows = self._tushare_client().query(
|
||||
"trade_cal",
|
||||
{
|
||||
"exchange": "SSE",
|
||||
"start_date": min(window_start, start_date),
|
||||
"end_date": end_date,
|
||||
},
|
||||
"cal_date,is_open,pretrade_date",
|
||||
)
|
||||
trade_dates, skipped = select_open_trade_dates_in_range(
|
||||
calendar_rows,
|
||||
start_date,
|
||||
end_date,
|
||||
maximum=MAX_RANGE_TRADING_DAYS,
|
||||
)
|
||||
if not trade_dates:
|
||||
raise ValueError("选定区间内没有交易日,周末或节假日无需回补。")
|
||||
existing = self.database.list_snapshot_trade_dates(trade_dates[0], trade_dates[-1])
|
||||
coverage = classify_snapshot_coverage(trade_dates, existing)
|
||||
return self._execute_snapshot_backfill(
|
||||
mode="range",
|
||||
end_date=end_date,
|
||||
lookback=None,
|
||||
coverage=coverage,
|
||||
skipped_non_trading_days=skipped,
|
||||
dry_run=dry_run,
|
||||
force=force,
|
||||
create_backup=create_backup,
|
||||
)
|
||||
|
||||
def _load_recent_open_trade_dates(self, end_date: str, lookback: int) -> list[str]:
|
||||
start_date = calendar_window_start(end_date, lookback)
|
||||
calendar_rows = self._tushare_client().query(
|
||||
"trade_cal",
|
||||
{
|
||||
"exchange": "SSE",
|
||||
"start_date": start_date,
|
||||
"end_date": end_date,
|
||||
},
|
||||
"cal_date,is_open,pretrade_date",
|
||||
)
|
||||
return select_open_trade_dates(calendar_rows, end_date, lookback)
|
||||
|
||||
def _execute_snapshot_backfill(
|
||||
self,
|
||||
*,
|
||||
mode: str,
|
||||
end_date: str,
|
||||
lookback: int | None,
|
||||
coverage: dict[str, Any],
|
||||
skipped_non_trading_days: list[str],
|
||||
dry_run: bool,
|
||||
force: bool,
|
||||
create_backup: bool,
|
||||
) -> dict[str, Any]:
|
||||
targets = list(coverage["trade_dates"] if force else coverage["missing"])
|
||||
backup_path: str | None = None
|
||||
if create_backup and not dry_run and targets:
|
||||
backup = create_sqlite_backup(
|
||||
Path(self.database.path),
|
||||
DATA_DIR / "backups",
|
||||
label=f"pre-{mode}-backfill",
|
||||
)
|
||||
backup_path = str(backup)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
if dry_run:
|
||||
for trade_date in coverage["trade_dates"]:
|
||||
exists = trade_date in coverage["present"]
|
||||
if exists and not force:
|
||||
status = "skipped"
|
||||
action = "exists"
|
||||
else:
|
||||
status = "planned"
|
||||
action = "refresh" if exists else "create"
|
||||
results.append(
|
||||
{
|
||||
"requested_date": display_date(trade_date),
|
||||
"trade_date": display_date(trade_date),
|
||||
"status": status,
|
||||
"action": action,
|
||||
}
|
||||
)
|
||||
return build_backfill_audit(
|
||||
mode=mode,
|
||||
end_date=end_date,
|
||||
lookback=lookback,
|
||||
coverage=coverage,
|
||||
skipped_non_trading_days=skipped_non_trading_days,
|
||||
backup_path=backup_path,
|
||||
dry_run=True,
|
||||
results=results,
|
||||
)
|
||||
|
||||
present_before = set(coverage["present"])
|
||||
for trade_date in targets:
|
||||
existed = trade_date in present_before
|
||||
try:
|
||||
dashboard = self.sync_dashboard(trade_date)
|
||||
actual = normalize_date(
|
||||
str(dashboard.get("meta", {}).get("trade_date") or trade_date)
|
||||
)
|
||||
results.append(
|
||||
{
|
||||
"requested_date": display_date(trade_date),
|
||||
"trade_date": display_date(actual),
|
||||
"status": "success",
|
||||
"action": "refreshed" if existed else "created",
|
||||
"source": dashboard.get("meta", {}).get("source"),
|
||||
"records": self._record_count(dashboard),
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
results.append(
|
||||
{
|
||||
"requested_date": display_date(trade_date),
|
||||
"trade_date": display_date(trade_date),
|
||||
"status": "failed",
|
||||
"action": "refresh" if existed else "create",
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
|
||||
for trade_date in coverage["present"]:
|
||||
if force:
|
||||
continue
|
||||
results.append(
|
||||
{
|
||||
"requested_date": day.isoformat(),
|
||||
"trade_date": dashboard["meta"]["trade_date"],
|
||||
"source": dashboard["meta"]["source"],
|
||||
"records": self._record_count(dashboard),
|
||||
"requested_date": display_date(trade_date),
|
||||
"trade_date": display_date(trade_date),
|
||||
"status": "skipped",
|
||||
"action": "exists",
|
||||
}
|
||||
)
|
||||
return results
|
||||
|
||||
results.sort(key=lambda row: str(row.get("requested_date") or ""))
|
||||
return build_backfill_audit(
|
||||
mode=mode,
|
||||
end_date=end_date,
|
||||
lookback=lookback,
|
||||
coverage=coverage,
|
||||
skipped_non_trading_days=skipped_non_trading_days,
|
||||
backup_path=backup_path,
|
||||
dry_run=False,
|
||||
results=results,
|
||||
)
|
||||
|
||||
def _stock_identity(self, code: str, trade_date: str) -> tuple[str, str]:
|
||||
snapshot = self.database.get_snapshot(trade_date) or {}
|
||||
@@ -955,4 +1168,3 @@ class MarketServiceMixin:
|
||||
len(dashboard.get(key) or [])
|
||||
for key in ("limits", "broken", "down_limits", "yesterday_limits")
|
||||
)
|
||||
|
||||
|
||||
@@ -26,13 +26,15 @@ class SystemHttpMixin:
|
||||
def start_background_refresh(self) -> None:
|
||||
try:
|
||||
body = self.read_json_body(allow_empty=True)
|
||||
started = self.application_service.request_background_sync(
|
||||
refresh = self.application_service.request_background_sync(
|
||||
str(body.get("trade_date") or date.today().isoformat())
|
||||
)
|
||||
started = bool(refresh.get("started"))
|
||||
self.send_json(
|
||||
{
|
||||
"ok": True,
|
||||
"started": started,
|
||||
"job_key": str(refresh.get("job_key") or ""),
|
||||
"message": "后台刷新已开始" if started else "已有后台刷新任务正在运行",
|
||||
},
|
||||
HTTPStatus.ACCEPTED,
|
||||
|
||||
@@ -29,11 +29,17 @@ class SystemRoutesMixin:
|
||||
def backfill_data(self) -> None:
|
||||
try:
|
||||
body = self.read_json_body()
|
||||
results = self.application_service.backfill(
|
||||
lookback_raw = body.get("lookback")
|
||||
lookback = int(lookback_raw) if lookback_raw not in (None, "") else None
|
||||
audit = self.application_service.backfill(
|
||||
str(body.get("start_date") or ""),
|
||||
str(body.get("end_date") or ""),
|
||||
lookback=lookback,
|
||||
dry_run=bool(body.get("dry_run")),
|
||||
force=bool(body.get("force")),
|
||||
create_backup=body.get("create_backup", True) is not False,
|
||||
)
|
||||
self.send_json({"ok": True, "results": results})
|
||||
self.send_json({"ok": True, **audit, "results": audit.get("results") or []})
|
||||
except ValueError as exc:
|
||||
self.send_json({"error": str(exc)}, HTTPStatus.BAD_REQUEST)
|
||||
except Exception as exc:
|
||||
|
||||
+14
-3
@@ -7,6 +7,16 @@ from datetime import date
|
||||
from backend.bootstrap.config import normalize_date
|
||||
|
||||
|
||||
def _verified_dashboard_result(dashboard: dict[str, object]) -> dict[str, object]:
|
||||
meta = dashboard.get("meta") or {}
|
||||
if isinstance(meta, dict) and meta.get("carried_forward"):
|
||||
return {
|
||||
"status": "failed",
|
||||
"error": str(meta.get("notice") or "未获取到所选日期的最新行情"),
|
||||
}
|
||||
return dashboard
|
||||
|
||||
|
||||
class JobServiceMixin:
|
||||
def start_background_jobs(self) -> threading.Thread:
|
||||
return self.jobs.start_scheduler(
|
||||
@@ -20,15 +30,16 @@ class JobServiceMixin:
|
||||
workers_stopped = self.jobs.wait_for_idle(timeout_seconds)
|
||||
return scheduler_stopped and workers_stopped
|
||||
|
||||
def request_background_sync(self, trade_date: str) -> bool:
|
||||
def request_background_sync(self, trade_date: str) -> dict[str, object]:
|
||||
normalized = normalize_date(trade_date)
|
||||
key = f"manual:{normalized}:{time.time_ns()}"
|
||||
return self.jobs.submit(
|
||||
started = self.jobs.submit(
|
||||
"market.refresh",
|
||||
key,
|
||||
lambda: self.sync_dashboard(normalized),
|
||||
lambda: _verified_dashboard_result(self.sync_dashboard(normalized)),
|
||||
{"trade_date": normalized, "trigger": "administrator"},
|
||||
)
|
||||
return {"started": started, "job_key": key if started else ""}
|
||||
|
||||
def _background_refresh_tick(self) -> None:
|
||||
if not (
|
||||
|
||||
@@ -461,8 +461,8 @@
|
||||
},
|
||||
{
|
||||
"path": "frontend/index.html",
|
||||
"bytes": 47891,
|
||||
"lines": 661
|
||||
"bytes": 48077,
|
||||
"lines": 662
|
||||
},
|
||||
{
|
||||
"path": "backend/features/screener/catalog.py",
|
||||
@@ -486,8 +486,8 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_dashboard.py",
|
||||
"bytes": 28051,
|
||||
"lines": 644
|
||||
"bytes": 28234,
|
||||
"lines": 648
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_industries.py",
|
||||
@@ -541,8 +541,8 @@
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/admin.js",
|
||||
"bytes": 14145,
|
||||
"lines": 261
|
||||
"bytes": 14410,
|
||||
"lines": 268
|
||||
},
|
||||
{
|
||||
"path": "backend/features/heaven/market_context.py",
|
||||
@@ -554,6 +554,11 @@
|
||||
"bytes": 13176,
|
||||
"lines": 293
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/dashboard.js",
|
||||
"bytes": 12894,
|
||||
"lines": 274
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/insights_auction_data.py",
|
||||
"bytes": 12829,
|
||||
@@ -574,11 +579,6 @@
|
||||
"bytes": 10539,
|
||||
"lines": 244
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/dashboard.js",
|
||||
"bytes": 9993,
|
||||
"lines": 220
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_sectors.py",
|
||||
"bytes": 9876,
|
||||
@@ -769,6 +769,11 @@
|
||||
"bytes": 2299,
|
||||
"lines": 57
|
||||
},
|
||||
{
|
||||
"path": "backend/jobs/service.py",
|
||||
"bytes": 2219,
|
||||
"lines": 60
|
||||
},
|
||||
{
|
||||
"path": "backend/features/screener/regime.py",
|
||||
"bytes": 2202,
|
||||
@@ -800,9 +805,9 @@
|
||||
"lines": 45
|
||||
},
|
||||
{
|
||||
"path": "backend/jobs/service.py",
|
||||
"bytes": 1746,
|
||||
"lines": 49
|
||||
"path": "backend/features/system/routes.py",
|
||||
"bytes": 1791,
|
||||
"lines": 46
|
||||
},
|
||||
{
|
||||
"path": "backend/features/alerts/routes.py",
|
||||
@@ -829,11 +834,6 @@
|
||||
"bytes": 1455,
|
||||
"lines": 48
|
||||
},
|
||||
{
|
||||
"path": "backend/features/system/routes.py",
|
||||
"bytes": 1423,
|
||||
"lines": 40
|
||||
},
|
||||
{
|
||||
"path": "backend/features/themes/routes.py",
|
||||
"bytes": 1337,
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
# 行情历史补档(最近 60 个交易日)
|
||||
|
||||
用于修复 `dashboard_snapshots` 断档导致情绪周期 / 主题轮动 / 智能选股只剩当天的问题。
|
||||
保留 `latest_contiguous_history` 连续性规则;通过真实交易日历回补缺失交易日快照。
|
||||
|
||||
## 适用场景
|
||||
|
||||
- 库中已有稀疏历史快照,但最近一个真实交易日缺失,接口 `available_days=1`。
|
||||
- 需要可重复执行、可审计、可回退的补档,而不是迁库或放宽算法。
|
||||
|
||||
## 前置
|
||||
|
||||
1. 使用与线上一致的代码分支。
|
||||
2. 管理员账号已配置可用的公共 Tushare Token。
|
||||
3. 只操作目标环境自己的 `data/review.db`;禁止 `.36` 与 `.11` 互拷。
|
||||
|
||||
## 上线步骤(总工执行)
|
||||
|
||||
在目标环境容器内执行(应用根目录;宿主机也可直接跑,脚本已自带仓库根 `sys.path` 引导):
|
||||
|
||||
```bash
|
||||
# 1) 只读规划:区分已有、真正缺档;不会写入
|
||||
docker compose exec xiaobai-review python tools/backfill_recent_snapshots.py --account <管理员账号> --lookback 60 --dry-run --json
|
||||
|
||||
# 2) 正式补档:先走 SQLite backup API 写 data/backups/review-pre-recent-backfill-*.db
|
||||
# 再对缺失交易日调用现有 sync_dashboard
|
||||
docker compose exec xiaobai-review python tools/backfill_recent_snapshots.py --account <管理员账号> --lookback 60 --json
|
||||
|
||||
# 3) 验证
|
||||
# GET /api/sentiment/history?trade_date=YYYY-MM-DD&limit=60
|
||||
# 期望 available_days >= 20,且不再只有 1 天
|
||||
```
|
||||
|
||||
管理端日期区间回补(`/api/backfill`)已改为只处理交易日历中的开市日,周末/节假日会进入
|
||||
`skipped_non_trading_days`,不再当成错误;单次仍限制 15 个交易日。最近 60 日请用本工具。
|
||||
|
||||
## 写入边界
|
||||
|
||||
只会通过现有同步路径写入:
|
||||
|
||||
- `dashboard_snapshots`
|
||||
- 同步审计表 `sync_runs`
|
||||
- 必要时的 `data_snapshots`(仅当请求日被解析到其他交易日)
|
||||
|
||||
不得改动用户、Token、模型绑定或系统配置表。
|
||||
|
||||
## 回滚
|
||||
|
||||
1. 优先按审计结果的 `created_dates` 精确删除新增行:
|
||||
|
||||
```sql
|
||||
DELETE FROM dashboard_snapshots WHERE trade_date IN ('YYYYMMDD', ...);
|
||||
```
|
||||
|
||||
2. 若需整库回退,停止写入后用补档前备份覆盖:
|
||||
|
||||
```bash
|
||||
# 示例:把 data/backups/review-pre-recent-backfill-YYYYMMDD-HHMMSS.db
|
||||
# 复制回 data/review.db 后重启容器
|
||||
```
|
||||
|
||||
3. 代码回退:对该提交执行 Git revert 后重新部署镜像。
|
||||
|
||||
## 验收要点
|
||||
|
||||
- dry-run 与正式执行可重复跑;已有交易日默认跳过。
|
||||
- 周末、节假日出现在 `skipped_non_trading_days`,不计入失败。
|
||||
- 部分交易日同步失败时,其他日期仍会继续,并在审计结果中标 `failed`。
|
||||
- 情绪周期、主题轮动 9 列、智能选股置信度随连续交易日恢复。
|
||||
@@ -609,6 +609,7 @@
|
||||
<label class="form-field"><span>iFinD Refresh Token</span><input id="systemIfindTokenInput" type="password" autocomplete="off" maxlength="2048" placeholder="留空保留现有 Token"></label>
|
||||
<label class="switch-control"><input id="systemBackgroundRefresh" type="checkbox"><span>启用交易时段后台刷新</span></label>
|
||||
<p class="form-hint">所有用户读取同一份后台快照,页面不会随后台任务自动重绘。</p>
|
||||
<div id="adminRefreshStatus" class="admin-refresh-status" data-tone="idle" role="status" aria-live="polite"><i data-lucide="circle-dot"></i><span>尚未手动刷新</span></div>
|
||||
<div class="dialog-actions admin-inline-actions"><button id="adminRefreshButton" class="button" type="button"><i data-lucide="refresh-cw"></i>立即后台刷新</button><button class="button primary" type="submit">保存行情配置</button></div>
|
||||
</form>
|
||||
<section class="settings-section">
|
||||
|
||||
@@ -7,7 +7,14 @@ async function backfillData() {
|
||||
start_date: document.querySelector("#backfillStart").value,
|
||||
end_date: document.querySelector("#backfillEnd").value,
|
||||
});
|
||||
showToast(`历史回补完成,共处理 ${payload.results.length} 个工作日`);
|
||||
const failed = (payload.failed_count || 0);
|
||||
const skipped = (payload.skipped_non_trading_days || []).length;
|
||||
const suffix = failed
|
||||
? `,失败 ${failed} 个`
|
||||
: skipped
|
||||
? `,跳过 ${skipped} 个非交易日`
|
||||
: "";
|
||||
showToast(`历史回补完成,共处理 ${payload.results.length} 个交易日${suffix}`);
|
||||
state.sentimentHistory = null;
|
||||
state.sentimentHistoryKey = "";
|
||||
if (state.activeView === "sentimentCycleView") {
|
||||
|
||||
@@ -471,6 +471,32 @@
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.admin-refresh-status {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 8px;
|
||||
margin-top: 12px;
|
||||
padding: 10px 12px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 10px;
|
||||
background: var(--surface-muted);
|
||||
color: var(--text-muted);
|
||||
font-size: 12px;
|
||||
line-height: 1.55;
|
||||
}
|
||||
|
||||
.admin-refresh-status svg {
|
||||
flex: 0 0 auto;
|
||||
width: 15px;
|
||||
height: 15px;
|
||||
margin-top: 2px;
|
||||
}
|
||||
|
||||
.admin-refresh-status[data-tone="running"] { color: var(--primary); }
|
||||
.admin-refresh-status[data-tone="success"] { color: var(--down); }
|
||||
.admin-refresh-status[data-tone="warning"] { color: var(--warning); }
|
||||
.admin-refresh-status[data-tone="failure"] { color: var(--up); }
|
||||
|
||||
.account-button > span {
|
||||
flex: 0 0 auto;
|
||||
|
||||
|
||||
@@ -40,17 +40,71 @@ async function loadDashboard(force = false, background = false, showOverlay = tr
|
||||
async function startAdminRefresh() {
|
||||
const buttons = [document.querySelector("#syncButton"), document.querySelector("#adminRefreshButton")].filter(Boolean);
|
||||
buttons.forEach((button) => { button.disabled = true; });
|
||||
const requestedDate = elements.tradeDate.value;
|
||||
setAdminRefreshStatus("running", `正在刷新 ${requestedDate} 的行情,请稍候…`, "loader-circle");
|
||||
try {
|
||||
const payload = await apiRequest("/api/admin/refresh", "POST", { trade_date: elements.tradeDate.value });
|
||||
showToast(payload.message || "后台刷新已提交");
|
||||
setStatus("后台刷新运行中,当前页面保持不变");
|
||||
const payload = await apiRequest("/api/admin/refresh", "POST", { trade_date: requestedDate });
|
||||
if (!payload.started || !payload.job_key) {
|
||||
setAdminRefreshStatus("warning", "已有刷新任务正在运行,请稍后再试。", "clock-3");
|
||||
showToast(payload.message || "已有后台刷新任务正在运行");
|
||||
return;
|
||||
}
|
||||
setStatus(`正在刷新 ${requestedDate} 的行情`);
|
||||
const job = await waitForAdminRefresh(payload.job_key);
|
||||
if (job.status === "failed") {
|
||||
const reason = job.message || job.error_code || "数据源未返回结果";
|
||||
setAdminRefreshStatus("failure", `刷新失败:${reason}`, "circle-x");
|
||||
setStatus("后台刷新失败");
|
||||
showToast("后台刷新失败");
|
||||
return;
|
||||
}
|
||||
const query = new URLSearchParams({ trade_date: requestedDate });
|
||||
const dashboard = await apiRequest(`/api/dashboard?${query}`);
|
||||
applyDashboard(dashboard);
|
||||
const meta = dashboard.meta || {};
|
||||
const actualDate = String(meta.trade_date || "").slice(0, 10);
|
||||
const requestedCompact = requestedDate.replaceAll("-", "");
|
||||
const actualCompact = actualDate.replaceAll("-", "");
|
||||
const updated = formatTimestamp(meta.updated_at);
|
||||
if (actualCompact !== requestedCompact || meta.carried_forward) {
|
||||
const reason = meta.notice ? `;${meta.notice}` : "";
|
||||
setAdminRefreshStatus("warning", `刷新已完成,但没有获取到 ${requestedDate} 的最新行情;当前仍是 ${actualDate || "未知日期"}${reason}`, "triangle-alert");
|
||||
showToast("刷新完成,但未获取到所选日期的最新行情");
|
||||
} else if (meta.notice) {
|
||||
setAdminRefreshStatus("warning", `已刷新到 ${actualDate}(${updated}),但数据源提示:${meta.notice}`, "triangle-alert");
|
||||
showToast(`已刷新到 ${actualDate},请留意数据源提示`);
|
||||
} else {
|
||||
setAdminRefreshStatus("success", `刷新成功:已获取 ${actualDate} 的最新行情,更新时间 ${updated}`, "circle-check");
|
||||
showToast(`刷新成功:已获取 ${actualDate} 的最新行情`);
|
||||
}
|
||||
} catch (error) {
|
||||
showToast(error.message || "后台刷新启动失败");
|
||||
const message = error.message || "后台刷新失败";
|
||||
setAdminRefreshStatus("failure", `刷新失败:${message}`, "circle-x");
|
||||
setStatus("后台刷新失败");
|
||||
showToast(message);
|
||||
} finally {
|
||||
buttons.forEach((button) => { button.disabled = false; });
|
||||
}
|
||||
}
|
||||
|
||||
function setAdminRefreshStatus(tone, message, icon = "circle-dot") {
|
||||
const status = document.querySelector("#adminRefreshStatus");
|
||||
if (!status) return;
|
||||
status.dataset.tone = tone;
|
||||
status.innerHTML = `<i data-lucide="${icon}"></i><span>${escapeHtml(message)}</span>`;
|
||||
refreshIcons();
|
||||
}
|
||||
|
||||
async function waitForAdminRefresh(jobKey) {
|
||||
for (let attempt = 0; attempt < 120; attempt += 1) {
|
||||
const payload = await apiRequest("/api/admin/settings");
|
||||
const job = (payload.data?.jobs || []).find((item) => item.idempotency_key === jobKey);
|
||||
if (job && ["success", "failed"].includes(job.status)) return job;
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
throw new Error("刷新等待超时,请稍后重试");
|
||||
}
|
||||
|
||||
function applyDashboard(payload, background = false) {
|
||||
state.dashboard = payload;
|
||||
const selectedDate = payload.meta.requested_date || payload.meta.trade_date;
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from backend.jobs.service import _verified_dashboard_result
|
||||
|
||||
|
||||
class AdminRefreshStatusTests(unittest.TestCase):
|
||||
def test_carried_snapshot_is_reported_as_failed_job(self):
|
||||
result = _verified_dashboard_result(
|
||||
{"meta": {"carried_forward": True, "notice": "官方涨跌停数据尚未返回"}}
|
||||
)
|
||||
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(result["error"], "官方涨跌停数据尚未返回")
|
||||
|
||||
def test_current_snapshot_is_reported_as_successful_job(self):
|
||||
dashboard = {"meta": {"trade_date": "2026-08-28", "carried_forward": False}}
|
||||
|
||||
self.assertIs(_verified_dashboard_result(dashboard), dashboard)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -111,6 +111,25 @@ class RealtimeDashboardTests(unittest.TestCase):
|
||||
self.assertEqual(quote["amount_billion"], 3.0)
|
||||
self.assertAlmostEqual(quote["turnover_rate"], 0.01)
|
||||
|
||||
def test_close_dashboard_marks_official_limit_data(self):
|
||||
dashboard = self.client.dashboard("20260720")
|
||||
|
||||
self.assertEqual(dashboard["meta"]["limit_data_source"], "official")
|
||||
|
||||
def test_close_dashboard_marks_derived_limit_data_as_incomplete(self):
|
||||
original_query = self.client.query
|
||||
|
||||
def query(api_name, params=None, fields=""):
|
||||
if api_name == "limit_list_d":
|
||||
return []
|
||||
return original_query(api_name, params, fields)
|
||||
|
||||
self.client.query = query
|
||||
dashboard = self.client.dashboard("20260720")
|
||||
|
||||
self.assertEqual(dashboard["meta"]["limit_data_source"], "derived")
|
||||
self.assertIn("日线数据推算", dashboard["meta"]["notice"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,317 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
from backend.features.market.backfill_history import (
|
||||
build_backfill_audit,
|
||||
classify_snapshot_coverage,
|
||||
create_sqlite_backup,
|
||||
select_open_trade_dates,
|
||||
select_open_trade_dates_in_range,
|
||||
)
|
||||
from backend.features.market.service import MarketServiceMixin
|
||||
from backend.features.sentiment.engine import (
|
||||
build_sentiment_history,
|
||||
latest_contiguous_history,
|
||||
)
|
||||
from backend.features.sentiment.service import SentimentServiceMixin
|
||||
from database import ReviewDatabase
|
||||
|
||||
|
||||
def _snapshot(trade_date: str, previous_trade_date: str) -> dict[str, Any]:
|
||||
display = f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:8]}"
|
||||
previous_display = (
|
||||
f"{previous_trade_date[:4]}-{previous_trade_date[4:6]}-{previous_trade_date[6:8]}"
|
||||
if previous_trade_date
|
||||
else ""
|
||||
)
|
||||
return {
|
||||
"meta": {
|
||||
"trade_date": display,
|
||||
"previous_trade_date": previous_display,
|
||||
"source": "tushare",
|
||||
},
|
||||
"overview": {
|
||||
"up_count": 2500,
|
||||
"down_count": 2000,
|
||||
"flat_count": 100,
|
||||
"amount_billion": 12000,
|
||||
"limit_up_count": 40,
|
||||
"limit_down_count": 5,
|
||||
"broken_count": 10,
|
||||
"seal_rate": 70,
|
||||
"max_height": 3,
|
||||
"second_board_count": 8,
|
||||
"three_plus_count": 4,
|
||||
"previous_limit_count": 35,
|
||||
"previous_positive_rate": 55,
|
||||
"average_previous_change": 1.2,
|
||||
"median_previous_change": 0.8,
|
||||
"advance_rate": 20,
|
||||
"severe_loss_rate": 5,
|
||||
"previous_down_count": 3,
|
||||
"ladder_completeness": 60,
|
||||
"limit_amount_billion": 300,
|
||||
},
|
||||
"limits": [{"code": "000001"}],
|
||||
"broken": [],
|
||||
"down_limits": [],
|
||||
"yesterday_limits": [],
|
||||
}
|
||||
|
||||
|
||||
class _BackfillHarness(MarketServiceMixin, SentimentServiceMixin):
|
||||
def __init__(self, database: ReviewDatabase) -> None:
|
||||
self.database = database
|
||||
self.sync_lock = threading.Lock()
|
||||
self.configured = True
|
||||
self.token = "test-token"
|
||||
self.current_user_id = 1
|
||||
self._calendar_rows: list[dict[str, Any]] = []
|
||||
self._fail_dates: set[str] = set()
|
||||
self.sync_calls: list[str] = []
|
||||
|
||||
def _tushare_client(self): # type: ignore[override]
|
||||
harness = self
|
||||
|
||||
class _Client:
|
||||
def query(self, api_name, params, fields=""):
|
||||
assert api_name == "trade_cal"
|
||||
start = str(params["start_date"])
|
||||
end = str(params["end_date"])
|
||||
return [
|
||||
row
|
||||
for row in harness._calendar_rows
|
||||
if start <= str(row["cal_date"]) <= end
|
||||
]
|
||||
|
||||
return _Client()
|
||||
|
||||
def sync_dashboard(self, trade_date: str) -> dict[str, Any]: # type: ignore[override]
|
||||
compact = trade_date.replace("-", "")
|
||||
self.sync_calls.append(compact)
|
||||
if compact in self._fail_dates:
|
||||
raise ValueError(f"simulated failure for {compact}")
|
||||
previous = ""
|
||||
for row in self._calendar_rows:
|
||||
if str(row["cal_date"]) == compact:
|
||||
previous = str(row.get("pretrade_date") or "")
|
||||
break
|
||||
payload = _snapshot(compact, previous)
|
||||
self.database.save_snapshot(compact, "tushare", payload)
|
||||
return payload
|
||||
|
||||
def _apply_reason_overrides(self, dashboard: dict[str, Any]) -> dict[str, Any]:
|
||||
return dashboard
|
||||
|
||||
def _with_storage(self, dashboard: dict[str, Any], cached: bool) -> dict[str, Any]:
|
||||
return dashboard
|
||||
|
||||
|
||||
class BackfillHistoryHelperTests(unittest.TestCase):
|
||||
def test_select_open_trade_dates_skips_weekends_and_holidays(self) -> None:
|
||||
rows = [
|
||||
{"cal_date": "20260821", "is_open": 1, "pretrade_date": "20260820"},
|
||||
{"cal_date": "20260822", "is_open": 0, "pretrade_date": "20260821"}, # Sat
|
||||
{"cal_date": "20260823", "is_open": 0, "pretrade_date": "20260821"}, # Sun
|
||||
{"cal_date": "20260824", "is_open": 1, "pretrade_date": "20260821"},
|
||||
{"cal_date": "20260825", "is_open": 1, "pretrade_date": "20260824"},
|
||||
{"cal_date": "20260826", "is_open": 1, "pretrade_date": "20260825"},
|
||||
{"cal_date": "20260827", "is_open": 1, "pretrade_date": "20260826"},
|
||||
]
|
||||
selected = select_open_trade_dates(rows, "20260827", 4)
|
||||
self.assertEqual(selected, ["20260824", "20260825", "20260826", "20260827"])
|
||||
|
||||
def test_range_mode_reports_non_trading_days_separately(self) -> None:
|
||||
rows = [
|
||||
{"cal_date": "20260821", "is_open": 1},
|
||||
{"cal_date": "20260824", "is_open": 1},
|
||||
]
|
||||
open_dates, skipped = select_open_trade_dates_in_range(
|
||||
rows, "20260821", "20260824"
|
||||
)
|
||||
self.assertEqual(open_dates, ["20260821", "20260824"])
|
||||
self.assertEqual(skipped, ["20260822", "20260823"])
|
||||
|
||||
def test_classify_snapshot_coverage_finds_real_gaps(self) -> None:
|
||||
coverage = classify_snapshot_coverage(
|
||||
["20260824", "20260825", "20260826", "20260827"],
|
||||
["20260824", "20260827"],
|
||||
)
|
||||
self.assertEqual(coverage["missing"], ["20260825", "20260826"])
|
||||
self.assertEqual(coverage["present"], ["20260824", "20260827"])
|
||||
|
||||
|
||||
class ContiguousHistoryGapTests(unittest.TestCase):
|
||||
def test_missing_previous_trade_day_collapses_to_today(self) -> None:
|
||||
payloads = [
|
||||
_snapshot("20260824", "20260821"),
|
||||
_snapshot("20260827", "20260826"), # gap: 20260826 missing
|
||||
]
|
||||
series = latest_contiguous_history(build_sentiment_history(payloads))
|
||||
self.assertEqual([row["trade_date"] for row in series], ["20260827"])
|
||||
|
||||
def test_continuous_history_keeps_full_tail(self) -> None:
|
||||
payloads = [
|
||||
_snapshot("20260825", "20260824"),
|
||||
_snapshot("20260826", "20260825"),
|
||||
_snapshot("20260827", "20260826"),
|
||||
]
|
||||
series = latest_contiguous_history(build_sentiment_history(payloads))
|
||||
self.assertEqual(
|
||||
[row["trade_date"] for row in series],
|
||||
["20260825", "20260826", "20260827"],
|
||||
)
|
||||
|
||||
|
||||
class SnapshotBackfillServiceTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temporary = tempfile.TemporaryDirectory()
|
||||
self.db_path = Path(self.temporary.name) / "review.db"
|
||||
self.database = ReviewDatabase(self.db_path)
|
||||
self.service = _BackfillHarness(self.database)
|
||||
self.service._calendar_rows = [
|
||||
{"cal_date": "20260820", "is_open": 1, "pretrade_date": "20260819"},
|
||||
{"cal_date": "20260821", "is_open": 1, "pretrade_date": "20260820"},
|
||||
{"cal_date": "20260822", "is_open": 0, "pretrade_date": "20260821"},
|
||||
{"cal_date": "20260823", "is_open": 0, "pretrade_date": "20260821"},
|
||||
{"cal_date": "20260824", "is_open": 1, "pretrade_date": "20260821"},
|
||||
{"cal_date": "20260825", "is_open": 1, "pretrade_date": "20260824"},
|
||||
{"cal_date": "20260826", "is_open": 1, "pretrade_date": "20260825"},
|
||||
{"cal_date": "20260827", "is_open": 1, "pretrade_date": "20260826"},
|
||||
]
|
||||
# Sparse history mimicking .11: keep 0824 and today, miss 0825/0826.
|
||||
self.database.save_snapshot("20260824", "tushare", _snapshot("20260824", "20260821"))
|
||||
self.database.save_snapshot("20260827", "tushare", _snapshot("20260827", "20260826"))
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.temporary.cleanup()
|
||||
|
||||
def test_recent_backfill_fills_gap_and_restores_history(self) -> None:
|
||||
before = self.service.sentiment_history("20260827", 20)
|
||||
self.assertEqual(before["available_days"], 1)
|
||||
|
||||
with patch(
|
||||
"backend.features.market.service.create_sqlite_backup",
|
||||
return_value=Path(self.temporary.name) / "fake-backup.db",
|
||||
) as backup:
|
||||
audit = self.service.backfill_recent_trading_days(
|
||||
end_date="20260827",
|
||||
lookback=4,
|
||||
dry_run=False,
|
||||
create_backup=True,
|
||||
)
|
||||
|
||||
backup.assert_called_once()
|
||||
self.assertEqual(sorted(self.service.sync_calls), ["20260825", "20260826"])
|
||||
self.assertEqual(audit["missing"], ["2026-08-25", "2026-08-26"])
|
||||
self.assertEqual(sorted(audit["created_dates"]), ["2026-08-25", "2026-08-26"])
|
||||
after = self.service.sentiment_history("20260827", 20)
|
||||
self.assertGreaterEqual(after["available_days"], 4)
|
||||
self.assertEqual(
|
||||
[row["trade_date"] for row in after["rows"]],
|
||||
["20260824", "20260825", "20260826", "20260827"],
|
||||
)
|
||||
|
||||
def test_dry_run_does_not_write_snapshots(self) -> None:
|
||||
audit = self.service.backfill_recent_trading_days(
|
||||
end_date="20260827",
|
||||
lookback=4,
|
||||
dry_run=True,
|
||||
create_backup=True,
|
||||
)
|
||||
self.assertTrue(audit["dry_run"])
|
||||
self.assertEqual(self.service.sync_calls, [])
|
||||
self.assertIsNone(audit["backup_path"])
|
||||
self.assertEqual(
|
||||
self.database.list_snapshot_trade_dates("20260824", "20260827"),
|
||||
["20260824", "20260827"],
|
||||
)
|
||||
|
||||
def test_repeat_execution_skips_existing_days(self) -> None:
|
||||
with patch(
|
||||
"backend.features.market.service.create_sqlite_backup",
|
||||
return_value=Path(self.temporary.name) / "fake-backup.db",
|
||||
):
|
||||
first = self.service.backfill_recent_trading_days(
|
||||
end_date="20260827", lookback=4
|
||||
)
|
||||
self.service.sync_calls.clear()
|
||||
second = self.service.backfill_recent_trading_days(
|
||||
end_date="20260827", lookback=4
|
||||
)
|
||||
self.assertEqual(first["succeeded_count"], 2)
|
||||
self.assertEqual(self.service.sync_calls, [])
|
||||
self.assertEqual(second["missing_count"], 0)
|
||||
self.assertEqual(second["skipped_count"], 4)
|
||||
self.assertIsNone(second["backup_path"])
|
||||
|
||||
def test_partial_failure_continues_remaining_days(self) -> None:
|
||||
self.service._fail_dates.add("20260825")
|
||||
with patch(
|
||||
"backend.features.market.service.create_sqlite_backup",
|
||||
return_value=Path(self.temporary.name) / "fake-backup.db",
|
||||
):
|
||||
audit = self.service.backfill_recent_trading_days(
|
||||
end_date="20260827", lookback=4
|
||||
)
|
||||
self.assertFalse(audit["ok"])
|
||||
self.assertEqual(audit["failed_count"], 1)
|
||||
self.assertEqual(audit["succeeded_count"], 1)
|
||||
self.assertIn("20260826", self.database.list_snapshot_trade_dates())
|
||||
self.assertNotIn("20260825", self.database.list_snapshot_trade_dates())
|
||||
|
||||
def test_range_backfill_skips_weekend_without_treating_as_error(self) -> None:
|
||||
with patch(
|
||||
"backend.features.market.service.create_sqlite_backup",
|
||||
return_value=Path(self.temporary.name) / "fake-backup.db",
|
||||
):
|
||||
audit = self.service.backfill(
|
||||
start_date="2026-08-21",
|
||||
end_date="2026-08-24",
|
||||
)
|
||||
self.assertEqual(audit["mode"], "range")
|
||||
self.assertEqual(audit["skipped_non_trading_days"], ["2026-08-22", "2026-08-23"])
|
||||
self.assertEqual(sorted(self.service.sync_calls), ["20260821"])
|
||||
self.assertTrue(audit["ok"])
|
||||
|
||||
def test_sqlite_backup_api_creates_restorable_copy(self) -> None:
|
||||
backup_dir = Path(self.temporary.name) / "backups"
|
||||
backup = create_sqlite_backup(
|
||||
self.db_path,
|
||||
backup_dir,
|
||||
label="pre-recent-backfill",
|
||||
stamped_at=datetime(2026, 8, 27, 15, 30, 0),
|
||||
)
|
||||
self.assertTrue(backup.exists())
|
||||
self.assertIn("pre-recent-backfill-20260827-153000", backup.name)
|
||||
restored = ReviewDatabase(backup)
|
||||
self.assertEqual(
|
||||
restored.list_snapshot_trade_dates(),
|
||||
["20260824", "20260827"],
|
||||
)
|
||||
|
||||
def test_audit_lists_only_snapshot_related_write_tables(self) -> None:
|
||||
audit = build_backfill_audit(
|
||||
mode="recent",
|
||||
end_date="20260827",
|
||||
lookback=60,
|
||||
coverage={"trade_dates": [], "present": [], "missing": [], "present_count": 0, "missing_count": 0},
|
||||
)
|
||||
self.assertEqual(
|
||||
audit["write_tables"],
|
||||
["dashboard_snapshots", "data_snapshots", "sync_runs"],
|
||||
)
|
||||
self.assertNotIn("users", audit["write_tables"])
|
||||
self.assertNotIn("system_settings", audit["write_tables"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -17,6 +17,9 @@ registry, and verification tools.
|
||||
`backend/features/*/routes.py` owners.
|
||||
- `python tools/build_architecture_inventory.py [--check]`: generate or verify
|
||||
`config/architecture-inventory.json` from the current source tree.
|
||||
- `python tools/backfill_recent_snapshots.py --account <admin> [--lookback 60] [--dry-run]`:
|
||||
auditable recent trading-day dashboard snapshot backfill. See
|
||||
`docs/maintenance/行情历史补档.md`.
|
||||
|
||||
`verify_baseline.py` does not inspect a parent checkout or skip tests according to files outside
|
||||
this application. Historical comparison scripts were retired after final standalone acceptance;
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Auditable recent trading-day dashboard snapshot backfill.
|
||||
|
||||
Examples:
|
||||
|
||||
python tools/backfill_recent_snapshots.py --account admin --dry-run
|
||||
python tools/backfill_recent_snapshots.py --account admin --lookback 60
|
||||
python tools/backfill_recent_snapshots.py --account admin --end-date 2026-08-27 --force
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from backend.application import SERVICE
|
||||
from backend.bootstrap.config import normalize_date
|
||||
from backend.features.market.backfill_history import DEFAULT_RECENT_TRADING_DAYS
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Backfill the latest N real trading-day dashboard snapshots"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--account",
|
||||
required=True,
|
||||
help="Account that can resolve the shared Tushare token",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--end-date",
|
||||
default=date.today().isoformat(),
|
||||
help="Inclusive end date YYYY-MM-DD (default: today)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lookback",
|
||||
type=int,
|
||||
default=DEFAULT_RECENT_TRADING_DAYS,
|
||||
help=f"Number of open trading days to cover (default {DEFAULT_RECENT_TRADING_DAYS}, max 60)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Plan only: classify missing gaps without writing",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Re-sync days that already have snapshots",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-backup",
|
||||
action="store_true",
|
||||
help="Skip the SQLite backup API step (not recommended)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="Print the full audit payload as JSON",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
user = SERVICE.database.user_by_username(args.account.strip())
|
||||
if not user:
|
||||
raise SystemExit("account not found")
|
||||
SERVICE.bind_user(int(user["id"]))
|
||||
|
||||
end_date = normalize_date(args.end_date)
|
||||
audit = SERVICE.backfill_recent_trading_days(
|
||||
end_date=end_date,
|
||||
lookback=args.lookback,
|
||||
dry_run=args.dry_run,
|
||||
force=args.force,
|
||||
create_backup=not args.no_backup,
|
||||
)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(audit, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if audit.get("ok") else 1)
|
||||
|
||||
print(
|
||||
f"mode={audit['mode']} end={audit['end_date']} lookback={audit['lookback']} "
|
||||
f"dry_run={audit['dry_run']}"
|
||||
)
|
||||
print(
|
||||
f"present={audit['present_count']} missing={audit['missing_count']} "
|
||||
f"succeeded={audit['succeeded_count']} skipped={audit['skipped_count']} "
|
||||
f"failed={audit['failed_count']}"
|
||||
)
|
||||
if audit.get("backup_path"):
|
||||
print(f"backup={audit['backup_path']}")
|
||||
if audit.get("missing"):
|
||||
print("missing_dates=" + ",".join(audit["missing"]))
|
||||
if audit.get("created_dates"):
|
||||
print("created_dates=" + ",".join(audit["created_dates"]))
|
||||
failed = [row for row in audit.get("results") or [] if row.get("status") == "failed"]
|
||||
for row in failed:
|
||||
print(f"failed {row.get('requested_date')}: {row.get('error')}")
|
||||
if not audit.get("ok"):
|
||||
raise SystemExit(1)
|
||||
print("backfill complete")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user