from __future__ import annotations import argparse import hashlib import json import os import sqlite3 from collections import defaultdict from collections.abc import Sequence from datetime import datetime from pathlib import Path from typing import Any from cryptography.fernet import Fernet from backend.data.sentiment import calculate_sentiment from backend.database import MIGRATIONS, Database, MigrationRunner from backend.features.screener.catalog import strategy_catalog ARCHIVE_VERSION = "legacy-archive-v1" HANDLED_SOURCE_TABLES = frozenset( { "alerts", "assistant_messages", "dashboard_snapshots", "data_snapshots", "daily_bars", "heaven_readings", "llm_usage", "mentor_messages", "mentor_preferences", "reason_overrides", "review_notes", "screener_runs", "screener_strategies", "seat_aliases", "stock_master", "strategy_tracks", "system_settings", "trade_entries", "user_birth_profiles", "users", "watchlist", } ) SKIPPED_SOURCE_TABLES = { "auction_factors": "governed provider inputs are rebuilt by scheduled jobs", "benchmark_bars": "legacy rows lack OHLC values required by the chart contract", "daily_indicators": "governed provider inputs are rebuilt by scheduled jobs", "earnings_events": "governed provider inputs are rebuilt by scheduled jobs", "fundamental_indicators": "governed provider inputs are rebuilt by scheduled jobs", "job_runs": "legacy operational logs are not user-facing history", "lhb_institution_daily": "governed provider inputs are rebuilt by scheduled jobs", "moneyflow_daily": "governed provider inputs are rebuilt by scheduled jobs", "popularity_factors": "governed provider inputs are rebuilt by scheduled jobs", "schema_migrations": "legacy implementation metadata does not apply to the new schema", "sector_phase_overrides": "sector phase overrides are no longer product-configurable", "sync_runs": "legacy operational logs are not user-facing history", "user_credentials": "per-user LLM configuration was removed from the product", "user_sessions": "sessions are intentionally invalidated during cutover", "wencai_saved_queries": "the WenCai feature was explicitly removed from the product", } HANDLED_SNAPSHOT_KINDS = frozenset( { *(f"auction_center_v{version}" for version in range(1, 7)), "dragon_tiger", "hot_money_detail_v2", "hot_money_detail_v3", "hot_money_profiles_v1", "ifind_event_enrichment_v1", "popularity_v1", "rotation_sector_members_v1", "theme_detail_v1", "theme_directory_v1", "theme_library_v1", } ) SKIPPED_SNAPSHOT_KINDS = { "dashboard_request_v1": "request cache is superseded by archived dashboard snapshots", "heaven_indices": "rebuildable input cache; saved Heaven readings are migrated", "heaven_sector": "rebuildable input cache; saved Heaven readings are migrated", "heaven_stock": "rebuildable input cache; saved Heaven readings are migrated", "screener_auto_v1": "derived cache is superseded by migrated screener runs", "search_directory": "search data is rebuilt from the migrated entity directory", "stock_detail": "rebuildable display cache", "stock_intraday": "rebuildable realtime display cache", } def _iso_date(value: Any) -> str: text = str(value or "").strip() compact = text[:10].replace("-", "") return f"{compact[:4]}-{compact[4:6]}-{compact[6:8]}" if len(compact) >= 8 else "" def _json(value: Any, fallback: Any) -> Any: if isinstance(value, (dict, list)): return value try: return json.loads(str(value)) except (TypeError, ValueError, json.JSONDecodeError): return fallback def _dump(value: Any) -> str: return json.dumps(value, ensure_ascii=False, separators=(",", ":")) def _formula_fingerprint(value: Any) -> str: formula = _json(value, {}) if not isinstance(formula, dict): return "" normalized = json.loads(json.dumps(formula, ensure_ascii=False)) metadata = normalized.get("meta") if isinstance(metadata, dict): metadata.pop("library_version", None) return json.dumps(normalized, ensure_ascii=False, sort_keys=True, separators=(",", ":")) def _hash_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def _table_exists(connection: sqlite3.Connection, table: str) -> bool: return ( connection.execute( "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,) ).fetchone() is not None ) def _table_names(connection: sqlite3.Connection) -> tuple[str, ...]: return tuple( str(row[0]) for row in connection.execute( """SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name""" ) ) def _normalize_identifiers(value: Any) -> Any: if isinstance(value, list): return [_normalize_identifiers(item) for item in value] if not isinstance(value, dict): return value normalized = {key: _normalize_identifiers(item) for key, item in value.items()} if normalized.get("ts_code") and not normalized.get("identifier"): normalized["identifier"] = str(normalized["ts_code"]).upper() return normalized def _normalize_market_units(value: Any) -> Any: if isinstance(value, list): return [_normalize_market_units(item) for item in value] if not isinstance(value, dict): return value normalized = {key: _normalize_market_units(item) for key, item in value.items()} for legacy, canonical, multiplier in ( ("amount_billion", "amount", 100_000_000), ("seal_amount_million", "seal_amount", 1_000_000), ("float_mv_billion", "float_mv", 100_000_000), ): raw = normalized.pop(legacy, None) if canonical not in normalized and raw not in (None, ""): normalized[canonical] = float(raw) * multiplier return normalized def _legacy_sentiment( payload: dict[str, Any], history: list[dict[str, Any]] ) -> dict[str, Any]: calculated = calculate_sentiment(payload, history) overview = payload.get("overview") or {} if int(overview.get("sentiment_engine_version") or 0) < 2: return calculated score = overview.get("sentiment_score") if score is None or not overview.get("sentiment_phase"): return calculated previous_scores = [ float(item["sentiment"]["score"]) for item in history[-3:] if (item.get("sentiment") or {}).get("score") is not None ] numeric_score = float(score) previous_score = previous_scores[-1] if previous_scores else numeric_score baseline = sum(previous_scores) / len(previous_scores) if previous_scores else numeric_score components = overview.get("sentiment_components") or {} if isinstance(components, dict): calculated["components"] = [ {"key": key, **dict(component)} for key, component in components.items() if isinstance(component, dict) ] calculated.update( { "score": int(round(numeric_score)), "label": str(overview.get("sentiment_label") or calculated["label"]), "phase": str(overview["sentiment_phase"]), "direction": str(overview.get("sentiment_direction") or calculated["direction"]), "day_change": round(numeric_score - previous_score, 1), "momentum": round(numeric_score - baseline, 1), } ) return calculated def _normalize_market_snapshot( value: dict[str, Any], trade_date: str, history: list[dict[str, Any]] ) -> dict[str, Any]: payload = _normalize_market_units(_normalize_identifiers(value)) overview = dict(payload.get("overview") or {}) for legacy, canonical, fallback in ( ("limit_up_count", "limit_up", len(payload.get("limits") or [])), ("limit_down_count", "limit_down", len(payload.get("down_limits") or [])), ("broken_count", "broken", len(payload.get("broken") or [])), ): raw = overview.pop(legacy, None) if canonical not in overview: overview[canonical] = raw if raw is not None else fallback if "amount" not in overview: raw_amount = overview.pop("amount_billion", None) overview["amount"] = float(raw_amount or 0) * 100_000_000 meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {} payload["trade_date"] = trade_date previous_date = payload.get("previous_trade_date") or meta.get("previous_trade_date") payload["previous_trade_date"] = _iso_date(str(previous_date)) if previous_date else "" payload["overview"] = overview sentiment = _legacy_sentiment(payload, history) payload["sentiment"] = sentiment payload["temperature"] = sentiment["score"] for key in tuple(overview): if key.startswith("sentiment_"): overview.pop(key) return payload def _normalize_birth_profile(encrypted_payload: str, encryption_key: str | None) -> str | None: if not encryption_key: raise ValueError("legacy birth profiles require APP_ENCRYPTION_KEY") fernet = Fernet(encryption_key.encode("ascii")) decrypted = fernet.decrypt(encrypted_payload.encode("ascii")) try: payload = json.loads(decrypted) gender = str(payload.get("gender") or "") birth_date = str(payload.get("birth_date") or "") birth_time = str(payload.get("birth_time") or "") if (not birth_date or not birth_time) and payload.get("birth_datetime"): stamp = datetime.fromisoformat(str(payload["birth_datetime"]).replace(" ", "T", 1)) birth_date = stamp.date().isoformat() birth_time = stamp.time().replace(tzinfo=None, second=0, microsecond=0).isoformat( timespec="minutes" ) datetime.fromisoformat(f"{birth_date}T{birth_time}") if gender not in {"male", "female"}: return None except (json.JSONDecodeError, TypeError, ValueError): return None normalized = json.dumps( {"birth_date": birth_date, "birth_time": birth_time, "gender": gender}, ensure_ascii=False, separators=(",", ":"), sort_keys=True, ) return fernet.encrypt(normalized.encode("utf-8")).decode("ascii") def _observed_at(payload: dict[str, Any], fallback: str) -> str: meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {} return str( payload.get("observed_at") or meta.get("updated_at") or meta.get("generated_at") or fallback ) def _time(value: Any) -> str: text = str(value or "").strip() return text[:5] if len(text) >= 5 else "" class LegacyMigrator: def __init__(self, source: Path, target: Path, encryption_key: str | None) -> None: self.source_path = source.resolve() self.target_path = target.resolve() self.key = encryption_key self.counts: dict[str, int] = defaultdict(int) self.row_skips: dict[str, dict[str, Any]] = {} self.user_ids: set[int] = set() self.run_ids: set[int] = set() self.run_id_map: dict[int, int] = {} self.stock_ids: dict[str, str] = {} self.admin_id: int | None = None self.dashboard_events: dict[str, dict[str, str]] = {} def run(self) -> dict[str, Any]: if self.source_path == self.target_path: raise ValueError("source and target database paths must differ") if not self.source_path.is_file(): raise FileNotFoundError(self.source_path) database = Database(self.target_path) MigrationRunner(database).upgrade(MIGRATIONS) source = sqlite3.connect(f"file:{self.source_path.as_posix()}?mode=ro", uri=True) source.row_factory = sqlite3.Row try: source_tables = _table_names(source) source_table_counts = { table: int(source.execute(f"SELECT count(*) FROM {table}").fetchone()[0]) for table in source_tables } snapshot_counts = { str(row["kind"]): int(row["count"]) for row in source.execute( "SELECT kind,count(*) AS count FROM data_snapshots GROUP BY kind ORDER BY kind" ) } handled_tables = sorted(set(source_tables) & HANDLED_SOURCE_TABLES) skipped_tables = { table: SKIPPED_SOURCE_TABLES[table] for table in sorted(set(source_tables) & SKIPPED_SOURCE_TABLES.keys()) } unmapped_tables = sorted( set(source_tables) - HANDLED_SOURCE_TABLES - SKIPPED_SOURCE_TABLES.keys() ) handled_snapshots = sorted(set(snapshot_counts) & HANDLED_SNAPSHOT_KINDS) skipped_snapshots = { kind: SKIPPED_SNAPSHOT_KINDS[kind] for kind in sorted(set(snapshot_counts) & SKIPPED_SNAPSHOT_KINDS.keys()) } unmapped_snapshots = sorted( set(snapshot_counts) - HANDLED_SNAPSHOT_KINDS - SKIPPED_SNAPSHOT_KINDS.keys() ) if unmapped_tables or unmapped_snapshots: raise RuntimeError( "unmapped legacy data: " f"tables={unmapped_tables}, snapshot_kinds={unmapped_snapshots}" ) with database.transaction() as target: self._accounts(source, target) self._system_settings(source, target) self._market(source, target) self._private_data(source, target) self._screener(source, target) self._insights(source, target) with database.read() as target: integrity = str(target.execute("PRAGMA integrity_check").fetchone()[0]) foreign_keys = list(target.execute("PRAGMA foreign_key_check")) target_counts = { table: int(target.execute(f"SELECT count(*) FROM {table}").fetchone()[0]) for table in _table_names(target) } finally: source.close() if integrity != "ok" or foreign_keys: raise RuntimeError("migrated database failed integrity validation") return { "source": str(self.source_path), "target": str(self.target_path), "source_sha256": _hash_file(self.source_path), "target_sha256": _hash_file(self.target_path), "integrity": integrity, "foreign_key_violations": 0, "migrated": dict(sorted(self.counts.items())), "source_tables": source_table_counts, "handled_tables": handled_tables, "intentionally_skipped_tables": skipped_tables, "unmapped_tables": unmapped_tables, "source_snapshot_kinds": snapshot_counts, "handled_snapshot_kinds": handled_snapshots, "intentionally_skipped_snapshot_kinds": skipped_snapshots, "unmapped_snapshot_kinds": unmapped_snapshots, "intentionally_skipped_rows": self.row_skips, "target_counts": target_counts, } def _accounts(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None: users = source.execute("SELECT * FROM users ORDER BY id").fetchall() for row in users: user_id = int(row["id"]) self.user_ids.add(user_id) password = f"scrypt$16384$8$1${row['password_salt']}${row['password_hash']}" target.execute( """INSERT INTO users (id,username,username_key,password_hash,is_admin,status,created_at,updated_at) VALUES (?,?,?,?,?,'active',?,?) ON CONFLICT(id) DO UPDATE SET username=excluded.username, username_key=excluded.username_key, password_hash=excluded.password_hash, is_admin=excluded.is_admin, status=excluded.status, updated_at=excluded.updated_at""", ( user_id, row["username"], str(row["username"]).casefold(), password, int(str(row["role"]) == "admin"), row["created_at"], row["updated_at"], ), ) permanent = str(row["membership_plan"]) == "永久" state = "active" if row["membership_status"] == "active" else "inactive" target.execute( """INSERT INTO memberships (user_id,state,expires_at,is_permanent,daily_llm_limit,updated_at,updated_by) VALUES (?,?,?,?,50,?,NULL) ON CONFLICT(user_id) DO UPDATE SET state=excluded.state,expires_at=excluded.expires_at, is_permanent=excluded.is_permanent,updated_at=excluded.updated_at""", (user_id, state, row["membership_expires_at"], int(permanent), row["updated_at"]), ) self.counts["users"] = len(users) self.counts["memberships"] = len(users) administrators = [ int(row["id"]) for row in users if str(row["role"]) == "admin" ] self.admin_id = min(administrators or self.user_ids) if _table_exists(source, "user_birth_profiles"): for row in source.execute("SELECT * FROM user_birth_profiles"): encrypted_profile = _normalize_birth_profile( str(row["encrypted_payload"]), self.key ) if encrypted_profile is None: self.row_skips["birth_profiles_invalid"] = { "count": self.row_skips.get("birth_profiles_invalid", {}).get( "count", 0 ) + 1, "reason": "legacy profile is incomplete and must be configured again", } continue target.execute( """INSERT INTO birth_profiles (user_id,encrypted_payload,created_at,updated_at) VALUES (?,?,?,?) ON CONFLICT(user_id) DO UPDATE SET encrypted_payload=excluded.encrypted_payload,updated_at=excluded.updated_at""", ( row["user_id"], encrypted_profile, row["updated_at"], row["updated_at"], ), ) self.counts["birth_profiles"] += 1 if _table_exists(source, "llm_usage"): for row in source.execute( """SELECT user_id,substr(created_at,1,10) usage_date,count(*) calls, max(created_at) updated_at FROM llm_usage WHERE status='success' GROUP BY user_id,substr(created_at,1,10)""" ): target.execute( """INSERT INTO llm_usage_daily VALUES (?,?,?,?) ON CONFLICT(user_id,usage_date) DO UPDATE SET successful_calls=excluded.successful_calls,updated_at=excluded.updated_at""", (row["user_id"], row["usage_date"], row["calls"], row["updated_at"]), ) self.counts["llm_usage_daily"] += 1 def _system_settings(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None: row = source.execute( """SELECT encrypted_payload,updated_at FROM system_settings WHERE setting_key='credentials'""" ).fetchone() if row is None: return if not self.key: raise ValueError("APP_ENCRYPTION_KEY is required to migrate encrypted settings") fernet = Fernet(self.key.encode("ascii")) credentials = json.loads(fernet.decrypt(str(row["encrypted_payload"]).encode("ascii"))) admin_id = min(self.user_ids) limit = max(1, min(int(credentials.get("member_daily_limit") or 50), 1000)) target.execute("UPDATE memberships SET daily_llm_limit=?", (limit,)) for name in ("tushare_token", "ifind_refresh_token", "ifind_access_token"): value = str(credentials.get(name) or "").strip() if value: target.execute( """INSERT INTO system_credentials VALUES (?,?,?,?) ON CONFLICT(name) DO UPDATE SET encrypted_value=excluded.encrypted_value, updated_at=excluded.updated_at,updated_by=excluded.updated_by""", (name, fernet.encrypt(value.encode()).decode(), row["updated_at"], admin_id), ) self.counts["system_credentials"] += 1 model_map: dict[str, int] = {} for model in credentials.get("llm_models") or []: key = str(model.get("id") or model.get("name") or "") display = str(model.get("name") or model.get("model") or "model").strip() api_key = str(model.get("api_key") or "") target.execute( """INSERT INTO llm_models (display_name,display_name_key,base_url,model_identifier,encrypted_api_key, created_at,updated_at,updated_by) VALUES (?,?,?,?,?,?,?,?) ON CONFLICT(display_name_key) DO UPDATE SET base_url=excluded.base_url, model_identifier=excluded.model_identifier,encrypted_api_key=excluded.encrypted_api_key, updated_at=excluded.updated_at,updated_by=excluded.updated_by""", ( display, display.casefold(), str(model.get("base_url") or ""), str(model.get("model") or ""), fernet.encrypt(api_key.encode()).decode(), row["updated_at"], row["updated_at"], admin_id, ), ) model_id = int( target.execute( "SELECT id FROM llm_models WHERE display_name_key=?", (display.casefold(),) ).fetchone()[0] ) model_map[key] = model_id self.counts["llm_models"] += 1 primary = model_map.get(str(credentials.get("primary_model_id") or "")) fallback = model_map.get(str(credentials.get("fallback_model_id") or "")) if fallback == primary: fallback = None target.execute( """UPDATE llm_configuration SET primary_model_id=?,fallback_model_id=?, updated_at=?,updated_by=? WHERE id=1""", (primary, fallback, row["updated_at"], admin_id), ) def _market(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None: observed = datetime.now().astimezone().isoformat(timespec="seconds") stocks = source.execute("SELECT * FROM stock_master ORDER BY ts_code").fetchall() self.stock_ids = { str(row["code"]): str(row["ts_code"]).upper() for row in stocks } for row in stocks: target.execute( """INSERT INTO market_entities VALUES ('stock',?,?,?,?,?,1,'legacy',?) ON CONFLICT(entity_type,identifier) DO UPDATE SET code=excluded.code, name=excluded.name,search_key=excluded.search_key,sector=excluded.sector, active=1,source='legacy',observed_at=excluded.observed_at""", ( row["ts_code"], row["code"], row["name"], f"{row['code']} {row['ts_code']} {row['name']} {row['industry']}".casefold(), row["industry"] or None, row["updated_at"] or observed, ), ) self.counts["market_entities"] = len(stocks) dates = [ row[0] for row in source.execute( "SELECT DISTINCT trade_date FROM daily_bars ORDER BY trade_date" ) ] previous = None for raw_date in dates: trade_date = _iso_date(raw_date) target.execute( """INSERT INTO trading_days VALUES (?,1,?,'legacy',?) ON CONFLICT(trade_date) DO UPDATE SET is_open=1, previous_open_date=excluded.previous_open_date""", (trade_date, previous, observed), ) previous = trade_date self.counts["trading_days"] = len(dates) snapshots: dict[str, tuple[sqlite3.Row, dict[str, Any]]] = {} for row in source.execute( "SELECT * FROM dashboard_snapshots ORDER BY trade_date,updated_at" ): payload = _json(row["payload"], {}) meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {} trade_date = _iso_date( str(payload.get("trade_date") or meta.get("trade_date") or row["trade_date"]) ) snapshots[trade_date] = (row, payload) history: list[dict[str, Any]] = [] for trade_date in sorted(snapshots): row, raw_payload = snapshots[trade_date] payload = _normalize_market_snapshot(raw_payload, trade_date, history) history.append(payload) events: dict[str, str] = {} for event_type, key in ( ("limit_up", "limits"), ("broken", "broken"), ("limit_down", "down_limits"), ): for item in payload.get(key) or []: if not isinstance(item, dict): continue identity = str( item.get("identifier") or item.get("ts_code") or item.get("code") or "" ).upper() code = identity.split(".")[0] if code: events[code] = event_type self.dashboard_events[trade_date] = events target.execute( """INSERT INTO market_summaries VALUES (?,?,'archive','legacy',1,?,?) ON CONFLICT(trade_date) DO UPDATE SET observed_at=excluded.observed_at, state='archive',source='legacy',coverage=1,payload_json=excluded.payload_json, created_at=excluded.created_at""", ( trade_date, row["updated_at"], _dump(payload), row["updated_at"], ), ) self.counts["market_summaries"] = len(snapshots) query = """WITH ranked AS ( SELECT *,row_number() OVER (PARTITION BY ts_code ORDER BY trade_date DESC) rank FROM daily_bars) SELECT * FROM ranked WHERE rank<=90 ORDER BY ts_code,trade_date""" current = "" points: list[dict[str, Any]] = [] for row in source.execute(query): code = str(row["ts_code"]) if current and code != current: self._save_chart(target, current, points, observed) points = [] current = code points.append( { "time": _iso_date(row["trade_date"]), "open": row["open"], "high": row["high"], "low": row["low"], "close": row["close"], "volume": float(row["vol"] or 0) * 100, "amount": float(row["amount"] or 0) * 1000, "average": None, } ) if current: self._save_chart(target, current, points, observed) self._seat_aliases(source, target) self._reason_overrides(source, target) def _seat_aliases( self, source: sqlite3.Connection, target: sqlite3.Connection ) -> None: if not _table_exists(source, "seat_aliases"): return for row in source.execute("SELECT * FROM seat_aliases ORDER BY seat_name"): target.execute( """INSERT INTO seat_aliases (seat_name,alias_name,updated_at,updated_by) VALUES (?,?,?,?) ON CONFLICT(seat_name) DO UPDATE SET alias_name=excluded.alias_name,updated_at=excluded.updated_at, updated_by=excluded.updated_by""", (row["seat_name"], row["alias"], row["updated_at"], self.admin_id), ) self.counts["seat_aliases"] += 1 def _reason_overrides( self, source: sqlite3.Connection, target: sqlite3.Connection ) -> None: if not _table_exists(source, "reason_overrides"): return for row in source.execute("SELECT * FROM reason_overrides ORDER BY trade_date,code"): trade_date = _iso_date(row["trade_date"]) code = str(row["code"] or "").split(".")[0] event_type = self.dashboard_events.get(trade_date, {}).get(code) if not event_type: self.counts["reason_overrides_unmatched"] += 1 continue inserted = self._save_revision( target, trade_date=trade_date, identifier=self.stock_ids.get(code, str(row["code"]).upper()), event_type=event_type, reason=str(row["reason"] or "").strip(), first_time="", last_time="", open_times=None, source="admin", priority=100, created_by=self.admin_id, created_at=str(row["updated_at"]), ) self.counts["market_event_revisions"] += int(inserted) def _save_chart( self, target: sqlite3.Connection, identifier: str, points: list[dict[str, Any]], observed: str, ) -> None: previous = points[-2]["close"] if len(points) > 1 else None target.execute( """INSERT INTO chart_series VALUES ('stock',?,'day',?,?,'tushare','display', 'none',1,?,?) ON CONFLICT(entity_type,identifier,interval,trade_date) DO UPDATE SET payload_json=excluded.payload_json,observed_at=excluded.observed_at""", ( identifier, points[-1]["time"], observed, _dump({"previous_close": previous, "points": points}), observed, ), ) self.counts["chart_series"] += 1 def _private_data(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None: stock_ids = { row["code"]: row["ts_code"] for row in source.execute("SELECT code,ts_code FROM stock_master") } for row in source.execute("SELECT * FROM watchlist"): if int(row["user_id"]) not in self.user_ids: continue target.execute( """INSERT INTO watchlist_entries VALUES (?,?,?,?,?,?) ON CONFLICT(user_id,identifier) DO UPDATE SET name=excluded.name, sector=excluded.sector,remark=excluded.remark""", ( row["user_id"], stock_ids.get(row["code"], row["code"]), row["name"], row["sector"] or None, row["created_at"], row["remark"], ), ) self.counts["watchlist_entries"] += 1 self._copy_review_rows(source, target) for row in source.execute("SELECT * FROM mentor_preferences"): target.execute( "INSERT OR REPLACE INTO mentor_preferences VALUES (?,?,?,?,?)", ( row["user_id"], row["mentor_id"], row["pinned"], row["sort_order"], row["updated_at"], ), ) self.counts["mentor_preferences"] += 1 for row in source.execute("SELECT * FROM mentor_messages"): target.execute( """INSERT OR REPLACE INTO mentor_messages (id,user_id,mentor_id,trade_date,role,content,request_id,status,created_at) VALUES (?,?,?,?,?,?,NULL,'complete',?)""", ( row["id"], row["user_id"], row["mentor_id"], _iso_date(row["trade_date"]), row["role"], row["content"], row["created_at"], ), ) self.counts["mentor_messages"] += 1 for row in source.execute("SELECT * FROM heaven_readings"): snapshot = _json(row["context_snapshot"], {}) if row["subject_detail"]: snapshot.setdefault("legacy_subject_detail", row["subject_detail"]) target.execute( """INSERT OR REPLACE INTO heaven_readings (id,user_id,mode,reading_date,subject_key,result_json,interpretation, interpretation_status,request_id,created_at,updated_at) VALUES (?,?,?,?,?,?,?,'complete',NULL,?,?)""", ( row["id"], row["user_id"], row["mode"], _iso_date(row["context_date"]), row["subject"], _dump(snapshot), row["answer"], row["created_at"], row["created_at"], ), ) self.counts["heaven_readings"] += 1 def _copy_review_rows(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None: for row in source.execute("SELECT * FROM review_notes WHERE user_id IS NOT NULL"): target.execute( """INSERT OR REPLACE INTO review_notes VALUES (?,?,?,?,?,?,?,?,?,?)""", ( row["id"], row["user_id"], row["code"], row["stock_name"], _iso_date(row["trade_date"]), row["summary"], row["content"], row["plan"], row["created_at"], row["updated_at"], ), ) self.counts["review_notes"] += 1 actions = {"buy", "sell", "add", "trim", "watch"} emotions = {"calm", "confident", "hesitant", "anxious", "impulsive"} for row in source.execute("SELECT * FROM trade_entries"): target.execute( """INSERT OR REPLACE INTO trade_entries VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", ( row["id"], row["user_id"], _iso_date(row["trade_date"]), row["code"], row["name"], row["action"] if row["action"] in actions else "watch", row["price"], row["quantity"], row["position_pct"], row["pnl_amount"], row["pnl_pct"], row["emotion"] if row["emotion"] in emotions else "calm", row["tags"], row["thesis"], row["execution"], row["created_at"], row["updated_at"], ), ) self.counts["trade_entries"] += 1 for row in source.execute("SELECT * FROM alerts"): kind = ( row["kind"] if row["kind"] in {"manual", "strategy_t1", "strategy_t5"} else "manual" ) target.execute( "INSERT OR REPLACE INTO alerts VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", ( row["id"], row["user_id"], kind, row["title"], row["content"], _iso_date(row["available_date"]), row["code"], row["dedupe_key"], row["is_read"], row["read_at"], row["created_at"], row["updated_at"], ), ) self.counts["alerts"] += 1 for row in source.execute("SELECT * FROM assistant_messages"): target.execute( """INSERT OR REPLACE INTO review_assistant_messages VALUES (?,?,?,?,?,NULL,'complete',?)""", ( row["id"], row["user_id"], row["role"], row["content"], _iso_date(row["context_date"]), row["created_at"], ), ) self.counts["review_assistant_messages"] += 1 def _screener(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None: builtin_by_name = { str(item["name"]): item for item in strategy_catalog() if item.get("kind") in {"stage", "curated"} } builtin_by_formula = { (str(item["kind"]), _formula_fingerprint(item["formula"])): item for item in builtin_by_name.values() } prepared: list[dict[str, Any]] = [] selected_by_key: dict[tuple[Any, ...], dict[str, Any]] = {} for row in source.execute("SELECT * FROM screener_runs ORDER BY id"): mode = {"smart": "stage", "curated": "curated", "quant": "custom"}.get( str(row["mode"]), "custom" ) formula = _json(row["formula"], {}) builtin = builtin_by_name.get(str(row["strategy_name"] or "")) or ( builtin_by_formula.get((mode, _formula_fingerprint(formula))) if formula else None ) if builtin is not None and builtin["kind"] == mode: strategy_id = str(builtin["id"]) strategy_name = str(builtin.get("display_name") or builtin["name"]) strategy_version = int(builtin["version"]) owner_user_id = None else: strategy_id = str(formula.get("id") or f"legacy-{row['id']}") strategy_name = str(row["strategy_name"] or strategy_id) raw_version = formula.get("version", 1) strategy_version = ( int(raw_version) if isinstance(raw_version, int) and not isinstance(raw_version, bool) and raw_version > 0 else 1 ) owner_user_id = row["user_id"] item = { "row": row, "mode": mode, "formula": formula, "strategy_id": strategy_id, "strategy_name": strategy_name, "strategy_version": strategy_version, "owner_user_id": owner_user_id, "trade_date": _iso_date(row["trade_date"]), } key = ( owner_user_id, mode, strategy_id, item["trade_date"], strategy_version, ) item["key"] = key prepared.append(item) selected_by_key[key] = item for item in prepared: source_id = int(item["row"]["id"]) self.run_id_map[source_id] = int(selected_by_key[item["key"]]["row"]["id"]) snapshots: dict[str, int] = {} selected = sorted(selected_by_key.values(), key=lambda item: int(item["row"]["id"])) for item in selected: row = item["row"] trade_date = item["trade_date"] if trade_date not in snapshots: target.execute( """INSERT OR IGNORE INTO screener_factor_snapshots (trade_date,version,observed_at,state,source_set_json,coverage_json,created_at) VALUES (?,? ,?,'archive','[\"legacy\"]','{}',?)""", (trade_date, ARCHIVE_VERSION, row["created_at"], row["created_at"]), ) snapshots[trade_date] = int( target.execute( "SELECT id FROM screener_factor_snapshots WHERE trade_date=? AND version=?", (trade_date, ARCHIVE_VERSION), ).fetchone()[0] ) result = _json(row["result"], {}) candidates = result.get("candidates") if isinstance(result, dict) else [] candidates = candidates if isinstance(candidates, list) else [] for candidate in candidates: if isinstance(candidate, dict) and not candidate.get("identifier"): candidate["identifier"] = ( candidate.get("ts_code") or candidate.get("code") or "" ) target.execute( """INSERT OR REPLACE INTO screener_runs (id,owner_user_id,mode,strategy_id,strategy_name,strategy_version, selection_date,factor_snapshot_id,status,started_at,completed_at,coverage, missing_fields_json,result_json,error_message) VALUES (?,?,?,?,?,?,?,?,?, ?,?,0,'[]',?,'')""", ( row["id"], item["owner_user_id"], item["mode"], item["strategy_id"], item["strategy_name"], item["strategy_version"], trade_date, snapshots[trade_date], "completed" if candidates else "no_signal", row["created_at"], row["created_at"], _dump(candidates), ), ) self.run_ids.add(int(row["id"])) self.counts["screener_runs"] += 1 strategy_rows = source.execute("SELECT * FROM screener_strategies").fetchall() builtins = sum(row["user_id"] is None for row in strategy_rows) if builtins: self.row_skips["screener_strategies_builtin"] = { "count": builtins, "reason": ( "legacy built-ins are superseded by the single versioned product catalog; " "only user-created strategies are migrated" ), } for row in (item for item in strategy_rows if item["user_id"] is not None): target.execute( """INSERT OR REPLACE INTO custom_screener_strategies (id,user_id,name,version,formula_json,created_at,updated_at) VALUES (?,?,?,1,?,?,?)""", ( row["id"], row["user_id"], row["name"], row["formula"], row["created_at"], row["updated_at"], ), ) self.counts["custom_screener_strategies"] += 1 for row in source.execute("SELECT * FROM strategy_tracks"): mapped_run_id = self.run_id_map.get(int(row["run_id"])) if mapped_run_id not in self.run_ids: continue target.execute( """INSERT OR REPLACE INTO strategy_tracks (id,user_id,run_id,identifier,code,name,sector,selection_date, strategy_name,entry_price,added_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)""", ( row["id"], row["user_id"], mapped_run_id, row["ts_code"], row["code"], row["name"], row["sector"], _iso_date(row["selection_date"]), row["strategy_name"], row["entry_price"], row["created_at"], ), ) self.counts["strategy_tracks"] += 1 def _save_revision( self, target: sqlite3.Connection, *, trade_date: str, identifier: str, event_type: str, reason: str, first_time: str, last_time: str, open_times: int | None, source: str, priority: int, created_by: int | None, created_at: str, ) -> bool: exists = target.execute( """SELECT 1 FROM market_event_revisions WHERE trade_date=? AND identifier=? AND event_type=? AND reason=? AND first_time=? AND last_time=? AND open_times IS ? AND source=? AND priority=? AND created_by IS ? AND created_at=?""", ( trade_date, identifier, event_type, reason, first_time, last_time, open_times, source, priority, created_by, created_at, ), ).fetchone() if exists: return False target.execute( """INSERT INTO market_event_revisions (trade_date,identifier,event_type,reason,first_time,last_time,open_times, source,priority,created_by,created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)""", ( trade_date, identifier, event_type, reason, first_time, last_time, open_times, source, priority, created_by, created_at, ), ) return True def _insights(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None: self._auction_snapshots(source, target) self._theme_snapshots(source, target) self._standard_insight_snapshots(source, target) self._dragon_snapshots(source, target) self._sector_member_snapshots(source, target) self._ifind_event_revisions(source, target) def _auction_snapshots( self, source: sqlite3.Connection, target: sqlite3.Connection ) -> None: selected: dict[str, tuple[int, sqlite3.Row]] = {} for row in source.execute( "SELECT * FROM data_snapshots WHERE kind LIKE 'auction_center_v%'" ): trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0]) try: version = int(str(row["kind"]).rsplit("v", 1)[1]) except ValueError: continue current = selected.get(trade_date) if trade_date and (current is None or version > current[0]): selected[trade_date] = (version, row) for trade_date, (_, row) in sorted(selected.items()): payload = _normalize_identifiers(_json(row["payload"], {})) payload.update( { "trade_date": trade_date, "observed_at": _observed_at(payload, str(row["updated_at"])), "state": "archive", "message": str(payload.get("message") or ""), } ) payload.setdefault("coverage", 1) payload.setdefault("dynamic", False) payload.setdefault("_market_rows", list(payload.get("rows") or [])) self._save_insight(target, "auction", trade_date, "", payload, 1) def _theme_snapshots( self, source: sqlite3.Connection, target: sqlite3.Connection ) -> None: directory_rows = source.execute( "SELECT * FROM data_snapshots WHERE kind='theme_directory_v1'" ).fetchall() for row in directory_rows: payload = _json(row["payload"], {}) self._save_theme_entities( target, list(payload.get("items") or []), str(row["updated_at"]) ) for row in source.execute( "SELECT * FROM data_snapshots WHERE kind='theme_library_v1' ORDER BY cache_key" ): trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0]) payload = _normalize_identifiers(_json(row["payload"], {})) payload.update( { "trade_date": trade_date, "observed_at": _observed_at(payload, str(row["updated_at"])), "state": "archive", "message": str(payload.get("message") or ""), } ) self._save_theme_entities( target, list(payload.get("items") or []), str(row["updated_at"]) ) self._save_insight(target, "themes", trade_date, "", payload, 1) for row in source.execute( "SELECT * FROM data_snapshots WHERE kind='theme_detail_v1' ORDER BY cache_key" ): cache_key = str(row["cache_key"]) raw_date, _, identifier = cache_key.partition(":") trade_date = _iso_date(raw_date) legacy = _normalize_identifiers(_json(row["payload"], {})) theme = dict(legacy.get("theme") or {}) identifier = (identifier or str(theme.get("code") or "")).upper() members = [] for item in legacy.get("members") or []: if not isinstance(item, dict): continue member_id = str(item.get("identifier") or item.get("ts_code") or "").upper() members.append( { "identifier": member_id, "code": str(item.get("code") or member_id.split(".")[0]), "name": str(item.get("name") or ""), "change": item.get("change"), "close": item.get("close", item.get("price")), "amount": ( float(item["amount_billion"]) * 100_000_000 if item.get("amount_billion") not in (None, "") else item.get("amount") ), "quoted": bool(item.get("quoted", item.get("has_quote"))), } ) summary = dict(legacy.get("summary") or {}) detail = { "trade_date": trade_date, "theme": theme, "summary": summary, "members": members, "message": "" if members else "该题材暂无可核验成分股", "observed_at": _observed_at(legacy, str(row["updated_at"])), "state": "archive", } self._save_insight(target, "themes", trade_date, identifier, detail, 1) self._save_theme_chart( target, identifier, list(legacy.get("series") or []), detail["observed_at"], ) def _save_theme_entities( self, target: sqlite3.Connection, rows: list[dict[str, Any]], observed_at: str ) -> None: for item in rows: identifier = str(item.get("code") or item.get("ts_code") or "").upper() name = str(item.get("name") or "").strip() if not identifier or not name: continue target.execute( """INSERT INTO market_entities (entity_type,identifier,code,name,search_key,sector,active,source,observed_at) VALUES ('theme',?,?,?,?,NULL,1,'legacy',?) ON CONFLICT(entity_type,identifier) DO UPDATE SET code=excluded.code,name=excluded.name,search_key=excluded.search_key, active=1,source='legacy',observed_at=excluded.observed_at""", ( identifier, identifier.split(".")[0], name, f"{identifier} {name}".casefold(), observed_at, ), ) def _save_theme_chart( self, target: sqlite3.Connection, identifier: str, rows: list[dict[str, Any]], observed_at: str, ) -> None: points = [ { "time": _iso_date(item.get("trade_date")), "open": item.get("open"), "high": item.get("high"), "low": item.get("low"), "close": item.get("close"), "volume": item.get("volume"), "amount": item.get("amount"), "average": None, } for item in rows if _iso_date(item.get("trade_date")) and item.get("close") is not None ] if not identifier or not points: return target.execute( """INSERT INTO chart_series (entity_type,identifier,interval,trade_date,observed_at,source,usage, adjustment,coverage,payload_json,created_at) VALUES ('theme',?,'day',?,?,'legacy','display','none',1,?,?) ON CONFLICT(entity_type,identifier,interval,trade_date) DO UPDATE SET observed_at=excluded.observed_at,payload_json=excluded.payload_json""", ( identifier, points[-1]["time"], observed_at, _dump( { "previous_close": points[-2]["close"] if len(points) > 1 else None, "points": points, } ), observed_at, ), ) self.counts["chart_series"] += 1 def _standard_insight_snapshots( self, source: sqlite3.Connection, target: sqlite3.Connection ) -> None: for row in source.execute( "SELECT * FROM data_snapshots WHERE kind='popularity_v1' ORDER BY cache_key" ): trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0]) payload = _normalize_identifiers(_json(row["payload"], {})) payload.update( { "trade_date": trade_date, "observed_at": _observed_at(payload, str(row["updated_at"])), "state": "archive", "message": str(payload.get("message") or ""), } ) self._save_insight(target, "popularity", trade_date, "", payload, 1) def _dragon_snapshots( self, source: sqlite3.Connection, target: sqlite3.Connection ) -> None: profile_row = source.execute( """SELECT * FROM data_snapshots WHERE kind='hot_money_profiles_v1' ORDER BY updated_at DESC LIMIT 1""" ).fetchone() profiles = [] if profile_row: for item in _json(profile_row["payload"], {}).get("profiles") or []: profiles.append( { "name": str(item.get("name") or ""), "desc": str(item.get("description") or item.get("desc") or ""), "orgs": _dump(item.get("organizations") or item.get("orgs") or []), } ) stocks_by_date: dict[str, list[dict[str, Any]]] = {} seats_by_date: dict[str, list[dict[str, Any]]] = {} observed_by_date: dict[str, str] = {} for row in source.execute( "SELECT * FROM data_snapshots WHERE kind='dragon_tiger' ORDER BY cache_key" ): trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0]) payload = _json(row["payload"], {}) observed_by_date[trade_date] = _observed_at(payload, str(row["updated_at"])) stocks = [] seats = [] for item in payload.get("rows") or []: identifier = str(item.get("ts_code") or item.get("identifier") or "").upper() stocks.append( { "ts_code": identifier, "name": str(item.get("name") or ""), "pct_change": item.get("change"), "reason": str(item.get("reason") or ""), } ) for seat in item.get("institutions") or []: seats.append( { "ts_code": identifier, "exalter": str(seat.get("seat_name") or ""), "buy": float(seat.get("buy_million") or 0) * 1_000_000, "sell": float(seat.get("sell_million") or 0) * 1_000_000, "net_buy": float(seat.get("net_buy_million") or 0) * 1_000_000, "reason": str(item.get("reason") or ""), } ) stocks_by_date[trade_date] = stocks seats_by_date[trade_date] = seats details: dict[str, tuple[int, sqlite3.Row]] = {} for row in source.execute( """SELECT * FROM data_snapshots WHERE kind IN ('hot_money_detail_v2','hot_money_detail_v3')""" ): trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0]) version = int(str(row["kind"]).rsplit("v", 1)[1]) if trade_date not in details or version > details[trade_date][0]: details[trade_date] = (version, row) official_by_date: dict[str, list[dict[str, Any]]] = {} for trade_date, (_, row) in details.items(): payload = _json(row["payload"], {}) observed_by_date.setdefault( trade_date, _observed_at(payload, str(row["updated_at"])) ) official = [] derived_stocks: dict[str, dict[str, Any]] = {} for trader in payload.get("traders") or []: trader_name = str(trader.get("name") or "") for operation in trader.get("operations") or []: identifier = str( operation.get("ts_code") or operation.get("identifier") or "" ).upper() official.append( { "ts_code": identifier, "ts_name": str(operation.get("name") or ""), "hm_name": trader_name, "hm_orgs": str(operation.get("seat_name") or ""), "buy_amount": float(operation.get("buy_million") or 0) * 1_000_000, "sell_amount": float(operation.get("sell_million") or 0) * 1_000_000, "net_amount": float(operation.get("net_buy_million") or 0) * 1_000_000, } ) if identifier: derived_stocks[identifier] = { "ts_code": identifier, "name": str(operation.get("name") or ""), "pct_change": operation.get("change"), "reason": str(operation.get("reason") or ""), } official_by_date[trade_date] = official stocks_by_date.setdefault(trade_date, list(derived_stocks.values())) seats_by_date.setdefault(trade_date, []) all_dates = sorted(set(stocks_by_date) | set(official_by_date)) for trade_date in all_dates: raw = { "trade_date": trade_date, "observed_at": observed_by_date.get( trade_date, datetime.now().astimezone().isoformat(timespec="seconds") ), "state": "archive", "official": official_by_date.get(trade_date, []), "profiles": profiles, "stocks": stocks_by_date.get(trade_date, []), "seats": seats_by_date.get(trade_date, []), } self._save_insight(target, "dragon-list", trade_date, "", raw, 1) def _sector_member_snapshots( self, source: sqlite3.Connection, target: sqlite3.Connection ) -> None: for row in source.execute( """SELECT * FROM data_snapshots WHERE kind='rotation_sector_members_v1' ORDER BY cache_key""" ): raw_date, _, sector_name = str(row["cache_key"]).partition(":") trade_date = _iso_date(raw_date) payload = _normalize_identifiers(_json(row["payload"], {})) meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {} payload.update( { "trade_date": trade_date, "sector_name": sector_name, "observed_at": _observed_at(payload, str(row["updated_at"])), "source": "legacy", "coverage": 1, } ) target.execute( """INSERT INTO sector_member_snapshots (trade_date,sector_name,sector_code,observed_at,source,coverage,payload_json) VALUES (?,?,?,?, 'legacy',1,?) ON CONFLICT(trade_date,sector_name) DO UPDATE SET sector_code=excluded.sector_code,observed_at=excluded.observed_at, source=excluded.source,coverage=excluded.coverage,payload_json=excluded.payload_json""", ( trade_date, sector_name, str(meta.get("sector_code") or meta.get("representative") or ""), payload["observed_at"], _dump(payload), ), ) self.counts["sector_member_snapshots"] += 1 def _ifind_event_revisions( self, source: sqlite3.Connection, target: sqlite3.Connection ) -> None: for row in source.execute( """SELECT * FROM data_snapshots WHERE kind='ifind_event_enrichment_v1' ORDER BY cache_key""" ): payload = _json(row["payload"], {}) trade_date = _iso_date(payload.get("trade_date") or row["cache_key"]) created_at = str(payload.get("generated_at") or row["updated_at"]) for event_type, key in ( ("limit_up", "limits"), ("broken", "broken"), ("limit_down", "down_limits"), ): values = payload.get(key) or {} if not isinstance(values, dict): continue for raw_identifier, detail in values.items(): if not isinstance(detail, dict): continue useful = any( detail.get(field) not in (None, "") for field in ("reason", "first_time", "last_time", "open_times") ) if not useful: continue code = str(raw_identifier).split(".")[0] inserted = self._save_revision( target, trade_date=trade_date, identifier=self.stock_ids.get(code, str(raw_identifier).upper()), event_type=event_type, reason=str(detail.get("reason") or "").strip(), first_time=_time(detail.get("first_time")), last_time=_time(detail.get("last_time")), open_times=( int(detail["open_times"]) if detail.get("open_times") not in (None, "") else None ), source="ifind", priority=20, created_by=None, created_at=created_at, ) self.counts["market_event_revisions"] += int(inserted) def _save_insight( self, target: sqlite3.Connection, kind: str, trade_date: str, entity_key: str, payload: dict[str, Any], coverage: float, ) -> None: target.execute( """INSERT INTO market_insight_snapshots (kind,trade_date,entity_key,observed_at,state,source,coverage,payload_json) VALUES (?,?,?,?,'archive','legacy',?,?) ON CONFLICT(kind,trade_date,entity_key) DO UPDATE SET observed_at=excluded.observed_at,state=excluded.state,source=excluded.source, coverage=excluded.coverage,payload_json=excluded.payload_json""", ( kind, trade_date, entity_key, _observed_at(payload, datetime.now().astimezone().isoformat(timespec="seconds")), max(0, min(float(coverage), 1)), _dump(payload), ), ) self.counts["market_insight_snapshots"] += 1 def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Migrate a read-only legacy database copy") parser.add_argument("--source", type=Path, required=True) parser.add_argument("--target", type=Path, required=True) parser.add_argument("--report", type=Path) return parser def main(arguments: Sequence[str] | None = None) -> int: parsed = build_parser().parse_args(arguments) report = LegacyMigrator(parsed.source, parsed.target, os.getenv("APP_ENCRYPTION_KEY")).run() rendered = json.dumps(report, ensure_ascii=False, indent=2) if parsed.report: parsed.report.parent.mkdir(parents=True, exist_ok=True) parsed.report.write_text(rendered + "\n", encoding="utf-8") print(rendered) return 0 if __name__ == "__main__": raise SystemExit(main())