fix(migration): normalize legacy screener archives
This commit is contained in:
leefer
@@ -15,6 +15,7 @@ from cryptography.fernet import Fernet
|
||||
|
||||
from backend.data.sentiment import calculate_sentiment
|
||||
from backend.database import MIGRATIONS, Database, MigrationRunner
|
||||
from backend.features.screener.catalog import strategy_catalog
|
||||
|
||||
ARCHIVE_VERSION = "legacy-archive-v1"
|
||||
|
||||
@@ -109,6 +110,17 @@ def _dump(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def _formula_fingerprint(value: Any) -> str:
|
||||
formula = _json(value, {})
|
||||
if not isinstance(formula, dict):
|
||||
return ""
|
||||
normalized = json.loads(json.dumps(formula, ensure_ascii=False))
|
||||
metadata = normalized.get("meta")
|
||||
if isinstance(metadata, dict):
|
||||
metadata.pop("library_version", None)
|
||||
return json.dumps(normalized, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _hash_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
@@ -286,6 +298,7 @@ class LegacyMigrator:
|
||||
self.row_skips: dict[str, dict[str, Any]] = {}
|
||||
self.user_ids: set[int] = set()
|
||||
self.run_ids: set[int] = set()
|
||||
self.run_id_map: dict[int, int] = {}
|
||||
self.stock_ids: dict[str, str] = {}
|
||||
self.admin_id: int | None = None
|
||||
self.dashboard_events: dict[str, dict[str, str]] = {}
|
||||
@@ -847,9 +860,73 @@ class LegacyMigrator:
|
||||
self.counts["review_assistant_messages"] += 1
|
||||
|
||||
def _screener(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
snapshots: dict[str, int] = {}
|
||||
builtin_by_name = {
|
||||
str(item["name"]): item
|
||||
for item in strategy_catalog()
|
||||
if item.get("kind") in {"stage", "curated"}
|
||||
}
|
||||
builtin_by_formula = {
|
||||
(str(item["kind"]), _formula_fingerprint(item["formula"])): item
|
||||
for item in builtin_by_name.values()
|
||||
}
|
||||
prepared: list[dict[str, Any]] = []
|
||||
selected_by_key: dict[tuple[Any, ...], dict[str, Any]] = {}
|
||||
for row in source.execute("SELECT * FROM screener_runs ORDER BY id"):
|
||||
trade_date = _iso_date(row["trade_date"])
|
||||
mode = {"smart": "stage", "curated": "curated", "quant": "custom"}.get(
|
||||
str(row["mode"]), "custom"
|
||||
)
|
||||
formula = _json(row["formula"], {})
|
||||
builtin = builtin_by_name.get(str(row["strategy_name"] or "")) or (
|
||||
builtin_by_formula.get((mode, _formula_fingerprint(formula)))
|
||||
if formula
|
||||
else None
|
||||
)
|
||||
if builtin is not None and builtin["kind"] == mode:
|
||||
strategy_id = str(builtin["id"])
|
||||
strategy_name = str(builtin.get("display_name") or builtin["name"])
|
||||
strategy_version = int(builtin["version"])
|
||||
owner_user_id = None
|
||||
else:
|
||||
strategy_id = str(formula.get("id") or f"legacy-{row['id']}")
|
||||
strategy_name = str(row["strategy_name"] or strategy_id)
|
||||
raw_version = formula.get("version", 1)
|
||||
strategy_version = (
|
||||
int(raw_version)
|
||||
if isinstance(raw_version, int) and not isinstance(raw_version, bool)
|
||||
and raw_version > 0
|
||||
else 1
|
||||
)
|
||||
owner_user_id = row["user_id"]
|
||||
item = {
|
||||
"row": row,
|
||||
"mode": mode,
|
||||
"formula": formula,
|
||||
"strategy_id": strategy_id,
|
||||
"strategy_name": strategy_name,
|
||||
"strategy_version": strategy_version,
|
||||
"owner_user_id": owner_user_id,
|
||||
"trade_date": _iso_date(row["trade_date"]),
|
||||
}
|
||||
key = (
|
||||
owner_user_id,
|
||||
mode,
|
||||
strategy_id,
|
||||
item["trade_date"],
|
||||
strategy_version,
|
||||
)
|
||||
item["key"] = key
|
||||
prepared.append(item)
|
||||
selected_by_key[key] = item
|
||||
|
||||
for item in prepared:
|
||||
source_id = int(item["row"]["id"])
|
||||
self.run_id_map[source_id] = int(selected_by_key[item["key"]]["row"]["id"])
|
||||
|
||||
snapshots: dict[str, int] = {}
|
||||
selected = sorted(selected_by_key.values(), key=lambda item: int(item["row"]["id"]))
|
||||
for item in selected:
|
||||
row = item["row"]
|
||||
trade_date = item["trade_date"]
|
||||
if trade_date not in snapshots:
|
||||
target.execute(
|
||||
"""INSERT OR IGNORE INTO screener_factor_snapshots
|
||||
@@ -871,23 +948,19 @@ class LegacyMigrator:
|
||||
candidate["identifier"] = (
|
||||
candidate.get("ts_code") or candidate.get("code") or ""
|
||||
)
|
||||
mode = {"smart": "stage", "curated": "curated", "quant": "custom"}.get(
|
||||
str(row["mode"]), "custom"
|
||||
)
|
||||
formula = _json(row["formula"], {})
|
||||
strategy_id = str(formula.get("id") or f"legacy-{row['id']}")
|
||||
target.execute(
|
||||
"""INSERT OR REPLACE INTO screener_runs
|
||||
(id,owner_user_id,mode,strategy_id,strategy_name,strategy_version,
|
||||
selection_date,factor_snapshot_id,status,started_at,completed_at,coverage,
|
||||
missing_fields_json,result_json,error_message)
|
||||
VALUES (?,?,?,?,?,1,?,?,?, ?,?,0,'[]',?,'')""",
|
||||
VALUES (?,?,?,?,?,?,?,?,?, ?,?,0,'[]',?,'')""",
|
||||
(
|
||||
row["id"],
|
||||
row["user_id"],
|
||||
mode,
|
||||
strategy_id,
|
||||
row["strategy_name"],
|
||||
item["owner_user_id"],
|
||||
item["mode"],
|
||||
item["strategy_id"],
|
||||
item["strategy_name"],
|
||||
item["strategy_version"],
|
||||
trade_date,
|
||||
snapshots[trade_date],
|
||||
"completed" if candidates else "no_signal",
|
||||
@@ -924,7 +997,8 @@ class LegacyMigrator:
|
||||
)
|
||||
self.counts["custom_screener_strategies"] += 1
|
||||
for row in source.execute("SELECT * FROM strategy_tracks"):
|
||||
if int(row["run_id"]) not in self.run_ids:
|
||||
mapped_run_id = self.run_id_map.get(int(row["run_id"]))
|
||||
if mapped_run_id not in self.run_ids:
|
||||
continue
|
||||
target.execute(
|
||||
"""INSERT OR REPLACE INTO strategy_tracks
|
||||
@@ -933,7 +1007,7 @@ class LegacyMigrator:
|
||||
(
|
||||
row["id"],
|
||||
row["user_id"],
|
||||
row["run_id"],
|
||||
mapped_run_id,
|
||||
row["ts_code"],
|
||||
row["code"],
|
||||
row["name"],
|
||||
|
||||
Reference in New Issue
Block a user