rebuild(stage-14): deliver migration and recovery tooling
This commit is contained in:
leefer
@@ -0,0 +1,629 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
from collections import defaultdict
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
from backend.database import MIGRATIONS, Database, MigrationRunner
|
||||
|
||||
ARCHIVE_VERSION = "legacy-archive-v1"
|
||||
|
||||
|
||||
def _iso_date(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
compact = text[:10].replace("-", "")
|
||||
return f"{compact[:4]}-{compact[4:6]}-{compact[6:8]}" if len(compact) >= 8 else ""
|
||||
|
||||
|
||||
def _json(value: Any, fallback: Any) -> Any:
|
||||
if isinstance(value, (dict, list)):
|
||||
return value
|
||||
try:
|
||||
return json.loads(str(value))
|
||||
except (TypeError, ValueError, json.JSONDecodeError):
|
||||
return fallback
|
||||
|
||||
|
||||
def _dump(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def _hash_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _table_exists(connection: sqlite3.Connection, table: str) -> bool:
|
||||
return (
|
||||
connection.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
||||
).fetchone()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
class LegacyMigrator:
|
||||
def __init__(self, source: Path, target: Path, encryption_key: str | None) -> None:
|
||||
self.source_path = source.resolve()
|
||||
self.target_path = target.resolve()
|
||||
self.key = encryption_key
|
||||
self.counts: dict[str, int] = defaultdict(int)
|
||||
self.skipped: dict[str, str] = {
|
||||
"sessions": "sessions are intentionally invalidated during cutover",
|
||||
"user_credentials": "per-user LLM configuration was removed from the product",
|
||||
"raw_factor_tables": "reproducible provider inputs are rebuilt by governed sync jobs",
|
||||
"benchmark_bars": "legacy rows lack OHLC values required by the chart contract",
|
||||
}
|
||||
self.user_ids: set[int] = set()
|
||||
self.run_ids: set[int] = set()
|
||||
|
||||
def run(self) -> dict[str, Any]:
|
||||
if self.source_path == self.target_path:
|
||||
raise ValueError("source and target database paths must differ")
|
||||
if not self.source_path.is_file():
|
||||
raise FileNotFoundError(self.source_path)
|
||||
database = Database(self.target_path)
|
||||
MigrationRunner(database).upgrade(MIGRATIONS)
|
||||
source = sqlite3.connect(f"file:{self.source_path.as_posix()}?mode=ro", uri=True)
|
||||
source.row_factory = sqlite3.Row
|
||||
try:
|
||||
with database.transaction() as target:
|
||||
self._accounts(source, target)
|
||||
self._system_settings(source, target)
|
||||
self._market(source, target)
|
||||
self._private_data(source, target)
|
||||
self._screener(source, target)
|
||||
self._insights(source, target)
|
||||
with database.read() as target:
|
||||
integrity = str(target.execute("PRAGMA integrity_check").fetchone()[0])
|
||||
foreign_keys = list(target.execute("PRAGMA foreign_key_check"))
|
||||
target_counts = {
|
||||
table: int(target.execute(f"SELECT count(*) FROM {table}").fetchone()[0])
|
||||
for table in (
|
||||
"users", "memberships", "market_entities", "market_summaries",
|
||||
"chart_series", "watchlist_entries", "review_notes", "trade_entries",
|
||||
"alerts", "mentor_messages", "heaven_readings", "screener_runs",
|
||||
"custom_screener_strategies", "strategy_tracks",
|
||||
)
|
||||
}
|
||||
finally:
|
||||
source.close()
|
||||
if integrity != "ok" or foreign_keys:
|
||||
raise RuntimeError("migrated database failed integrity validation")
|
||||
return {
|
||||
"source": str(self.source_path),
|
||||
"target": str(self.target_path),
|
||||
"source_sha256": _hash_file(self.source_path),
|
||||
"target_sha256": _hash_file(self.target_path),
|
||||
"integrity": integrity,
|
||||
"foreign_key_violations": 0,
|
||||
"migrated": dict(sorted(self.counts.items())),
|
||||
"target_counts": target_counts,
|
||||
"intentionally_skipped": self.skipped,
|
||||
}
|
||||
|
||||
def _accounts(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
users = source.execute("SELECT * FROM users ORDER BY id").fetchall()
|
||||
for row in users:
|
||||
user_id = int(row["id"])
|
||||
self.user_ids.add(user_id)
|
||||
password = f"scrypt$16384$8$1${row['password_salt']}${row['password_hash']}"
|
||||
target.execute(
|
||||
"""INSERT INTO users
|
||||
(id,username,username_key,password_hash,is_admin,status,created_at,updated_at)
|
||||
VALUES (?,?,?,?,?,'active',?,?) ON CONFLICT(id) DO UPDATE SET
|
||||
username=excluded.username, username_key=excluded.username_key,
|
||||
password_hash=excluded.password_hash, is_admin=excluded.is_admin,
|
||||
status=excluded.status, updated_at=excluded.updated_at""",
|
||||
(
|
||||
user_id,
|
||||
row["username"],
|
||||
str(row["username"]).casefold(),
|
||||
password,
|
||||
int(str(row["role"]) == "admin"),
|
||||
row["created_at"],
|
||||
row["updated_at"],
|
||||
),
|
||||
)
|
||||
permanent = str(row["membership_plan"]) == "永久"
|
||||
state = "active" if row["membership_status"] == "active" else "inactive"
|
||||
target.execute(
|
||||
"""INSERT INTO memberships
|
||||
(user_id,state,expires_at,is_permanent,daily_llm_limit,updated_at,updated_by)
|
||||
VALUES (?,?,?,?,50,?,NULL) ON CONFLICT(user_id) DO UPDATE SET
|
||||
state=excluded.state,expires_at=excluded.expires_at,
|
||||
is_permanent=excluded.is_permanent,updated_at=excluded.updated_at""",
|
||||
(user_id, state, row["membership_expires_at"], int(permanent), row["updated_at"]),
|
||||
)
|
||||
self.counts["users"] = len(users)
|
||||
self.counts["memberships"] = len(users)
|
||||
if _table_exists(source, "user_birth_profiles"):
|
||||
for row in source.execute("SELECT * FROM user_birth_profiles"):
|
||||
target.execute(
|
||||
"""INSERT INTO birth_profiles
|
||||
(user_id,encrypted_payload,created_at,updated_at) VALUES (?,?,?,?)
|
||||
ON CONFLICT(user_id) DO UPDATE SET
|
||||
encrypted_payload=excluded.encrypted_payload,updated_at=excluded.updated_at""",
|
||||
(
|
||||
row["user_id"],
|
||||
row["encrypted_payload"],
|
||||
row["updated_at"],
|
||||
row["updated_at"],
|
||||
),
|
||||
)
|
||||
self.counts["birth_profiles"] += 1
|
||||
if _table_exists(source, "llm_usage"):
|
||||
for row in source.execute(
|
||||
"""SELECT user_id,substr(created_at,1,10) usage_date,count(*) calls,
|
||||
max(created_at) updated_at FROM llm_usage WHERE status='success'
|
||||
GROUP BY user_id,substr(created_at,1,10)"""
|
||||
):
|
||||
target.execute(
|
||||
"""INSERT INTO llm_usage_daily VALUES (?,?,?,?)
|
||||
ON CONFLICT(user_id,usage_date) DO UPDATE SET
|
||||
successful_calls=excluded.successful_calls,updated_at=excluded.updated_at""",
|
||||
(row["user_id"], row["usage_date"], row["calls"], row["updated_at"]),
|
||||
)
|
||||
self.counts["llm_usage_daily"] += 1
|
||||
|
||||
def _system_settings(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
row = source.execute(
|
||||
"""SELECT encrypted_payload,updated_at FROM system_settings
|
||||
WHERE setting_key='credentials'"""
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return
|
||||
if not self.key:
|
||||
raise ValueError("APP_ENCRYPTION_KEY is required to migrate encrypted settings")
|
||||
fernet = Fernet(self.key.encode("ascii"))
|
||||
credentials = json.loads(fernet.decrypt(str(row["encrypted_payload"]).encode("ascii")))
|
||||
admin_id = min(self.user_ids)
|
||||
limit = max(1, min(int(credentials.get("member_daily_limit") or 50), 1000))
|
||||
target.execute("UPDATE memberships SET daily_llm_limit=?", (limit,))
|
||||
for name in ("tushare_token", "ifind_refresh_token", "ifind_access_token"):
|
||||
value = str(credentials.get(name) or "").strip()
|
||||
if value:
|
||||
target.execute(
|
||||
"""INSERT INTO system_credentials VALUES (?,?,?,?)
|
||||
ON CONFLICT(name) DO UPDATE SET encrypted_value=excluded.encrypted_value,
|
||||
updated_at=excluded.updated_at,updated_by=excluded.updated_by""",
|
||||
(name, fernet.encrypt(value.encode()).decode(), row["updated_at"], admin_id),
|
||||
)
|
||||
self.counts["system_credentials"] += 1
|
||||
model_map: dict[str, int] = {}
|
||||
for model in credentials.get("llm_models") or []:
|
||||
key = str(model.get("id") or model.get("name") or "")
|
||||
display = str(model.get("name") or model.get("model") or "model").strip()
|
||||
api_key = str(model.get("api_key") or "")
|
||||
target.execute(
|
||||
"""INSERT INTO llm_models
|
||||
(display_name,display_name_key,base_url,model_identifier,encrypted_api_key,
|
||||
created_at,updated_at,updated_by) VALUES (?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(display_name_key) DO UPDATE SET base_url=excluded.base_url,
|
||||
model_identifier=excluded.model_identifier,encrypted_api_key=excluded.encrypted_api_key,
|
||||
updated_at=excluded.updated_at,updated_by=excluded.updated_by""",
|
||||
(
|
||||
display,
|
||||
display.casefold(),
|
||||
str(model.get("base_url") or ""),
|
||||
str(model.get("model") or ""),
|
||||
fernet.encrypt(api_key.encode()).decode(),
|
||||
row["updated_at"],
|
||||
row["updated_at"],
|
||||
admin_id,
|
||||
),
|
||||
)
|
||||
model_id = int(
|
||||
target.execute(
|
||||
"SELECT id FROM llm_models WHERE display_name_key=?", (display.casefold(),)
|
||||
).fetchone()[0]
|
||||
)
|
||||
model_map[key] = model_id
|
||||
self.counts["llm_models"] += 1
|
||||
primary = model_map.get(str(credentials.get("primary_model_id") or ""))
|
||||
fallback = model_map.get(str(credentials.get("fallback_model_id") or ""))
|
||||
if fallback == primary:
|
||||
fallback = None
|
||||
target.execute(
|
||||
"""UPDATE llm_configuration SET primary_model_id=?,fallback_model_id=?,
|
||||
updated_at=?,updated_by=? WHERE id=1""",
|
||||
(primary, fallback, row["updated_at"], admin_id),
|
||||
)
|
||||
|
||||
def _market(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
observed = datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
stocks = source.execute("SELECT * FROM stock_master ORDER BY ts_code").fetchall()
|
||||
for row in stocks:
|
||||
target.execute(
|
||||
"""INSERT INTO market_entities VALUES ('stock',?,?,?,?,?,1,'legacy',?)
|
||||
ON CONFLICT(entity_type,identifier) DO UPDATE SET code=excluded.code,
|
||||
name=excluded.name,search_key=excluded.search_key,sector=excluded.sector,
|
||||
active=1,source='legacy',observed_at=excluded.observed_at""",
|
||||
(
|
||||
row["ts_code"],
|
||||
row["code"],
|
||||
row["name"],
|
||||
f"{row['code']} {row['ts_code']} {row['name']} {row['industry']}".casefold(),
|
||||
row["industry"] or None,
|
||||
row["updated_at"] or observed,
|
||||
),
|
||||
)
|
||||
self.counts["market_entities"] = len(stocks)
|
||||
dates = [
|
||||
row[0]
|
||||
for row in source.execute(
|
||||
"SELECT DISTINCT trade_date FROM daily_bars ORDER BY trade_date"
|
||||
)
|
||||
]
|
||||
previous = None
|
||||
for raw_date in dates:
|
||||
trade_date = _iso_date(raw_date)
|
||||
target.execute(
|
||||
"""INSERT INTO trading_days VALUES (?,1,?,'legacy',?)
|
||||
ON CONFLICT(trade_date) DO UPDATE SET is_open=1,
|
||||
previous_open_date=excluded.previous_open_date""",
|
||||
(trade_date, previous, observed),
|
||||
)
|
||||
previous = trade_date
|
||||
self.counts["trading_days"] = len(dates)
|
||||
for row in source.execute("SELECT * FROM dashboard_snapshots"):
|
||||
target.execute(
|
||||
"""INSERT INTO market_summaries VALUES (?,?,'archive','legacy',1,?,?)
|
||||
ON CONFLICT(trade_date) DO UPDATE SET observed_at=excluded.observed_at,
|
||||
state='archive',source='legacy',coverage=1,payload_json=excluded.payload_json,
|
||||
created_at=excluded.created_at""",
|
||||
(
|
||||
_iso_date(row["trade_date"]),
|
||||
row["updated_at"],
|
||||
row["payload"],
|
||||
row["updated_at"],
|
||||
),
|
||||
)
|
||||
self.counts["market_summaries"] += 1
|
||||
query = """WITH ranked AS (
|
||||
SELECT *,row_number() OVER (PARTITION BY ts_code ORDER BY trade_date DESC) rank
|
||||
FROM daily_bars) SELECT * FROM ranked WHERE rank<=90 ORDER BY ts_code,trade_date"""
|
||||
current = ""
|
||||
points: list[dict[str, Any]] = []
|
||||
for row in source.execute(query):
|
||||
code = str(row["ts_code"])
|
||||
if current and code != current:
|
||||
self._save_chart(target, current, points, observed)
|
||||
points = []
|
||||
current = code
|
||||
points.append(
|
||||
{
|
||||
"time": _iso_date(row["trade_date"]),
|
||||
"open": row["open"],
|
||||
"high": row["high"],
|
||||
"low": row["low"],
|
||||
"close": row["close"],
|
||||
"volume": float(row["vol"] or 0) * 100,
|
||||
"amount": float(row["amount"] or 0) * 1000,
|
||||
"average": None,
|
||||
}
|
||||
)
|
||||
if current:
|
||||
self._save_chart(target, current, points, observed)
|
||||
|
||||
def _save_chart(
|
||||
self,
|
||||
target: sqlite3.Connection,
|
||||
identifier: str,
|
||||
points: list[dict[str, Any]],
|
||||
observed: str,
|
||||
) -> None:
|
||||
previous = points[-2]["close"] if len(points) > 1 else None
|
||||
target.execute(
|
||||
"""INSERT INTO chart_series VALUES ('stock',?,'day',?,?,'tushare','display',
|
||||
'none',1,?,?) ON CONFLICT(entity_type,identifier,interval,trade_date)
|
||||
DO UPDATE SET payload_json=excluded.payload_json,observed_at=excluded.observed_at""",
|
||||
(
|
||||
identifier,
|
||||
points[-1]["time"],
|
||||
observed,
|
||||
_dump({"previous_close": previous, "points": points}),
|
||||
observed,
|
||||
),
|
||||
)
|
||||
self.counts["chart_series"] += 1
|
||||
|
||||
def _private_data(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
stock_ids = {
|
||||
row["code"]: row["ts_code"]
|
||||
for row in source.execute("SELECT code,ts_code FROM stock_master")
|
||||
}
|
||||
for row in source.execute("SELECT * FROM watchlist"):
|
||||
if int(row["user_id"]) not in self.user_ids:
|
||||
continue
|
||||
target.execute(
|
||||
"""INSERT INTO watchlist_entries VALUES (?,?,?,?,?,?)
|
||||
ON CONFLICT(user_id,identifier) DO UPDATE SET name=excluded.name,
|
||||
sector=excluded.sector,remark=excluded.remark""",
|
||||
(
|
||||
row["user_id"],
|
||||
stock_ids.get(row["code"], row["code"]),
|
||||
row["name"],
|
||||
row["sector"] or None,
|
||||
row["created_at"],
|
||||
row["remark"],
|
||||
),
|
||||
)
|
||||
self.counts["watchlist_entries"] += 1
|
||||
self._copy_review_rows(source, target)
|
||||
for row in source.execute("SELECT * FROM mentor_preferences"):
|
||||
target.execute(
|
||||
"INSERT OR REPLACE INTO mentor_preferences VALUES (?,?,?,?,?)",
|
||||
(
|
||||
row["user_id"],
|
||||
row["mentor_id"],
|
||||
row["pinned"],
|
||||
row["sort_order"],
|
||||
row["updated_at"],
|
||||
),
|
||||
)
|
||||
self.counts["mentor_preferences"] += 1
|
||||
for row in source.execute("SELECT * FROM mentor_messages"):
|
||||
target.execute(
|
||||
"""INSERT OR REPLACE INTO mentor_messages
|
||||
(id,user_id,mentor_id,trade_date,role,content,request_id,status,created_at)
|
||||
VALUES (?,?,?,?,?,?,NULL,'complete',?)""",
|
||||
(
|
||||
row["id"],
|
||||
row["user_id"],
|
||||
row["mentor_id"],
|
||||
_iso_date(row["trade_date"]),
|
||||
row["role"],
|
||||
row["content"],
|
||||
row["created_at"],
|
||||
),
|
||||
)
|
||||
self.counts["mentor_messages"] += 1
|
||||
for row in source.execute("SELECT * FROM heaven_readings"):
|
||||
snapshot = _json(row["context_snapshot"], {})
|
||||
if row["subject_detail"]:
|
||||
snapshot.setdefault("legacy_subject_detail", row["subject_detail"])
|
||||
target.execute(
|
||||
"""INSERT OR REPLACE INTO heaven_readings
|
||||
(id,user_id,mode,reading_date,subject_key,result_json,interpretation,
|
||||
interpretation_status,request_id,created_at,updated_at)
|
||||
VALUES (?,?,?,?,?,?,?,'complete',NULL,?,?)""",
|
||||
(
|
||||
row["id"],
|
||||
row["user_id"],
|
||||
row["mode"],
|
||||
_iso_date(row["context_date"]),
|
||||
row["subject"],
|
||||
_dump(snapshot),
|
||||
row["answer"],
|
||||
row["created_at"],
|
||||
row["created_at"],
|
||||
),
|
||||
)
|
||||
self.counts["heaven_readings"] += 1
|
||||
|
||||
def _copy_review_rows(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
for row in source.execute("SELECT * FROM review_notes WHERE user_id IS NOT NULL"):
|
||||
target.execute(
|
||||
"""INSERT OR REPLACE INTO review_notes VALUES (?,?,?,?,?,?,?,?,?,?)""",
|
||||
(
|
||||
row["id"],
|
||||
row["user_id"],
|
||||
row["code"],
|
||||
row["stock_name"],
|
||||
_iso_date(row["trade_date"]),
|
||||
row["summary"],
|
||||
row["content"],
|
||||
row["plan"],
|
||||
row["created_at"],
|
||||
row["updated_at"],
|
||||
),
|
||||
)
|
||||
self.counts["review_notes"] += 1
|
||||
actions = {"buy", "sell", "add", "trim", "watch"}
|
||||
emotions = {"calm", "confident", "hesitant", "anxious", "impulsive"}
|
||||
for row in source.execute("SELECT * FROM trade_entries"):
|
||||
target.execute(
|
||||
"""INSERT OR REPLACE INTO trade_entries
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
||||
(
|
||||
row["id"],
|
||||
row["user_id"],
|
||||
_iso_date(row["trade_date"]),
|
||||
row["code"],
|
||||
row["name"],
|
||||
row["action"] if row["action"] in actions else "watch",
|
||||
row["price"],
|
||||
row["quantity"],
|
||||
row["position_pct"],
|
||||
row["pnl_amount"],
|
||||
row["pnl_pct"],
|
||||
row["emotion"] if row["emotion"] in emotions else "calm",
|
||||
row["tags"],
|
||||
row["thesis"],
|
||||
row["execution"],
|
||||
row["created_at"],
|
||||
row["updated_at"],
|
||||
),
|
||||
)
|
||||
self.counts["trade_entries"] += 1
|
||||
for row in source.execute("SELECT * FROM alerts"):
|
||||
kind = (
|
||||
row["kind"] if row["kind"] in {"manual", "strategy_t1", "strategy_t5"} else "manual"
|
||||
)
|
||||
target.execute(
|
||||
"INSERT OR REPLACE INTO alerts VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
row["id"],
|
||||
row["user_id"],
|
||||
kind,
|
||||
row["title"],
|
||||
row["content"],
|
||||
_iso_date(row["available_date"]),
|
||||
row["code"],
|
||||
row["dedupe_key"],
|
||||
row["is_read"],
|
||||
row["read_at"],
|
||||
row["created_at"],
|
||||
row["updated_at"],
|
||||
),
|
||||
)
|
||||
self.counts["alerts"] += 1
|
||||
for row in source.execute("SELECT * FROM assistant_messages"):
|
||||
target.execute(
|
||||
"""INSERT OR REPLACE INTO review_assistant_messages
|
||||
VALUES (?,?,?,?,?,NULL,'complete',?)""",
|
||||
(
|
||||
row["id"],
|
||||
row["user_id"],
|
||||
row["role"],
|
||||
row["content"],
|
||||
_iso_date(row["context_date"]),
|
||||
row["created_at"],
|
||||
),
|
||||
)
|
||||
self.counts["review_assistant_messages"] += 1
|
||||
|
||||
def _screener(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
snapshots: dict[str, int] = {}
|
||||
for row in source.execute("SELECT * FROM screener_runs ORDER BY id"):
|
||||
trade_date = _iso_date(row["trade_date"])
|
||||
if trade_date not in snapshots:
|
||||
target.execute(
|
||||
"""INSERT OR IGNORE INTO screener_factor_snapshots
|
||||
(trade_date,version,observed_at,state,source_set_json,coverage_json,created_at)
|
||||
VALUES (?,? ,?,'archive','[\"legacy\"]','{}',?)""",
|
||||
(trade_date, ARCHIVE_VERSION, row["created_at"], row["created_at"]),
|
||||
)
|
||||
snapshots[trade_date] = int(
|
||||
target.execute(
|
||||
"SELECT id FROM screener_factor_snapshots WHERE trade_date=? AND version=?",
|
||||
(trade_date, ARCHIVE_VERSION),
|
||||
).fetchone()[0]
|
||||
)
|
||||
result = _json(row["result"], {})
|
||||
candidates = result.get("candidates") if isinstance(result, dict) else []
|
||||
candidates = candidates if isinstance(candidates, list) else []
|
||||
for candidate in candidates:
|
||||
if isinstance(candidate, dict) and not candidate.get("identifier"):
|
||||
candidate["identifier"] = (
|
||||
candidate.get("ts_code") or candidate.get("code") or ""
|
||||
)
|
||||
mode = {"smart": "stage", "curated": "curated", "quant": "custom"}.get(
|
||||
str(row["mode"]), "custom"
|
||||
)
|
||||
formula = _json(row["formula"], {})
|
||||
strategy_id = str(formula.get("id") or f"legacy-{row['id']}")
|
||||
target.execute(
|
||||
"""INSERT OR REPLACE INTO screener_runs
|
||||
(id,owner_user_id,mode,strategy_id,strategy_name,strategy_version,
|
||||
selection_date,factor_snapshot_id,status,started_at,completed_at,coverage,
|
||||
missing_fields_json,result_json,error_message)
|
||||
VALUES (?,?,?,?,?,1,?,?,?, ?,?,0,'[]',?,'')""",
|
||||
(
|
||||
row["id"],
|
||||
row["user_id"],
|
||||
mode,
|
||||
strategy_id,
|
||||
row["strategy_name"],
|
||||
trade_date,
|
||||
snapshots[trade_date],
|
||||
"completed" if candidates else "no_signal",
|
||||
row["created_at"],
|
||||
row["created_at"],
|
||||
_dump(candidates),
|
||||
),
|
||||
)
|
||||
self.run_ids.add(int(row["id"]))
|
||||
self.counts["screener_runs"] += 1
|
||||
for row in source.execute("SELECT * FROM screener_strategies WHERE user_id IS NOT NULL"):
|
||||
target.execute(
|
||||
"""INSERT OR REPLACE INTO custom_screener_strategies
|
||||
(id,user_id,name,version,formula_json,created_at,updated_at)
|
||||
VALUES (?,?,?,1,?,?,?)""",
|
||||
(
|
||||
row["id"],
|
||||
row["user_id"],
|
||||
row["name"],
|
||||
row["formula"],
|
||||
row["created_at"],
|
||||
row["updated_at"],
|
||||
),
|
||||
)
|
||||
self.counts["custom_screener_strategies"] += 1
|
||||
for row in source.execute("SELECT * FROM strategy_tracks"):
|
||||
if int(row["run_id"]) not in self.run_ids:
|
||||
continue
|
||||
target.execute(
|
||||
"""INSERT OR REPLACE INTO strategy_tracks
|
||||
(id,user_id,run_id,identifier,code,name,sector,selection_date,
|
||||
strategy_name,entry_price,added_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
|
||||
(
|
||||
row["id"],
|
||||
row["user_id"],
|
||||
row["run_id"],
|
||||
row["ts_code"],
|
||||
row["code"],
|
||||
row["name"],
|
||||
row["sector"],
|
||||
_iso_date(row["selection_date"]),
|
||||
row["strategy_name"],
|
||||
row["entry_price"],
|
||||
row["created_at"],
|
||||
),
|
||||
)
|
||||
self.counts["strategy_tracks"] += 1
|
||||
|
||||
def _insights(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
||||
mappings = {
|
||||
"auction_center_v6": "auction",
|
||||
"theme_library_v1": "themes",
|
||||
"popularity_v1": "popularity",
|
||||
"dragon_tiger": "dragon-list",
|
||||
}
|
||||
for old_kind, new_kind in mappings.items():
|
||||
for row in source.execute("SELECT * FROM data_snapshots WHERE kind=?", (old_kind,)):
|
||||
trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0])
|
||||
if not trade_date:
|
||||
continue
|
||||
target.execute(
|
||||
"""INSERT OR REPLACE INTO market_insight_snapshots
|
||||
VALUES (?,?,'',?,'archive','legacy',1,?)""",
|
||||
(new_kind, trade_date, row["updated_at"], row["payload"]),
|
||||
)
|
||||
self.counts["market_insight_snapshots"] += 1
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Migrate a read-only legacy database copy")
|
||||
parser.add_argument("--source", type=Path, required=True)
|
||||
parser.add_argument("--target", type=Path, required=True)
|
||||
parser.add_argument("--report", type=Path)
|
||||
return parser
|
||||
|
||||
|
||||
def main(arguments: Sequence[str] | None = None) -> int:
|
||||
parsed = build_parser().parse_args(arguments)
|
||||
report = LegacyMigrator(parsed.source, parsed.target, os.getenv("APP_ENCRYPTION_KEY")).run()
|
||||
rendered = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if parsed.report:
|
||||
parsed.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
parsed.report.write_text(rendered + "\n", encoding="utf-8")
|
||||
print(rendered)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user