rebuild(stage-14): deliver migration and recovery tooling

next/tools/backup.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sqlite3
+import tempfile
+import zipfile
+from collections.abc import Sequence
+from contextlib import closing
+from datetime import datetime
+from pathlib import Path
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for block in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def snapshot_database(source: Path, destination: Path) -> None:
+    source_uri = f"file:{source.resolve().as_posix()}?mode=ro"
+    with closing(sqlite3.connect(source_uri, uri=True)) as origin:
+        with closing(sqlite3.connect(destination)) as snapshot:
+            origin.backup(snapshot)
+            if snapshot.execute("PRAGMA integrity_check").fetchone()[0] != "ok":
+                raise RuntimeError("database snapshot failed integrity check")
+
+
+def create_backup(
+    database: Path,
+    output: Path,
+    *,
+    private_skills: Path | None = None,
+    environment_file: Path | None = None,
+    encryption_key_file: Path | None = None,
+) -> dict:
+    database = database.resolve()
+    if not database.is_file():
+        raise FileNotFoundError(database)
+    output = output.resolve()
+    output.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix="xiaobai-backup-") as raw:
+        staging = Path(raw)
+        database_copy = staging / "database.sqlite3"
+        snapshot_database(database, database_copy)
+        files: list[tuple[Path, str]] = [(database_copy, "database.sqlite3")]
+        for source, archive_name in (
+            (environment_file, "secrets/environment"),
+            (encryption_key_file, "secrets/app-encryption.key"),
+        ):
+            if source:
+                resolved = source.resolve()
+                if not resolved.is_file():
+                    raise FileNotFoundError(resolved)
+                files.append((resolved, archive_name))
+        if private_skills and private_skills.exists():
+            files.extend(
+                (item, f"private-mentor-skills/{item.relative_to(private_skills).as_posix()}")
+                for item in private_skills.rglob("*")
+                if item.is_file()
+            )
+        manifest = {
+            "format": 1,
+            "created_at": datetime.now().astimezone().isoformat(timespec="seconds"),
+            "files": {
+                name: {"sha256": sha256(path), "size": path.stat().st_size} for path, name in files
+            },
+        }
+        manifest_path = staging / "manifest.json"
+        manifest_path.write_text(
+            json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+        )
+        temporary = output.with_suffix(output.suffix + ".tmp")
+        with zipfile.ZipFile(temporary, "w", compression=zipfile.ZIP_DEFLATED) as archive:
+            archive.write(manifest_path, "manifest.json")
+            for path, name in files:
+                archive.write(path, name)
+        temporary.replace(output)
+    return manifest
+
+
+def apply_retention(directory: Path, *, keep_daily: int = 7, keep_weekly: int = 4) -> list[Path]:
+    backups = sorted(
+        directory.glob("xiaobai-*.zip"), key=lambda item: item.stat().st_mtime, reverse=True
+    )
+    keep = set(backups[:keep_daily])
+    weeks: set[str] = set()
+    for backup in backups:
+        week = datetime.fromtimestamp(backup.stat().st_mtime).strftime("%G-%V")
+        if week not in weeks and len(weeks) < keep_weekly:
+            weeks.add(week)
+            keep.add(backup)
+    removed = []
+    for backup in backups:
+        if backup not in keep:
+            backup.unlink()
+            removed.append(backup)
+    return removed
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Create a consistent Xiaobai backup archive")
+    parser.add_argument("--database", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--private-skills", type=Path)
+    parser.add_argument("--environment-file", type=Path)
+    parser.add_argument("--encryption-key-file", type=Path)
+    parser.add_argument("--apply-retention", action="store_true")
+    return parser
+
+
+def main(arguments: Sequence[str] | None = None) -> int:
+    parsed = build_parser().parse_args(arguments)
+    manifest = create_backup(
+        parsed.database,
+        parsed.output,
+        private_skills=parsed.private_skills,
+        environment_file=parsed.environment_file,
+        encryption_key_file=parsed.encryption_key_file,
+    )
+    removed = apply_retention(parsed.output.parent) if parsed.apply_retention else []
+    print(
+        json.dumps(
+            {
+                "backup": str(parsed.output.resolve()),
+                "files": len(manifest["files"]),
+                "removed_by_retention": len(removed),
+            },
+            ensure_ascii=False,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

next/tools/legacy_migration.py

@@ -0,0 +1,629 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import sqlite3
+from collections import defaultdict
+from collections.abc import Sequence
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from cryptography.fernet import Fernet
+
+from backend.database import MIGRATIONS, Database, MigrationRunner
+
+ARCHIVE_VERSION = "legacy-archive-v1"
+
+
+def _iso_date(value: Any) -> str:
+    text = str(value or "").strip()
+    compact = text[:10].replace("-", "")
+    return f"{compact[:4]}-{compact[4:6]}-{compact[6:8]}" if len(compact) >= 8 else ""
+
+
+def _json(value: Any, fallback: Any) -> Any:
+    if isinstance(value, (dict, list)):
+        return value
+    try:
+        return json.loads(str(value))
+    except (TypeError, ValueError, json.JSONDecodeError):
+        return fallback
+
+
+def _dump(value: Any) -> str:
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
+
+
+def _hash_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for block in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+def _table_exists(connection: sqlite3.Connection, table: str) -> bool:
+    return (
+        connection.execute(
+            "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
+        ).fetchone()
+        is not None
+    )
+
+
+class LegacyMigrator:
+    def __init__(self, source: Path, target: Path, encryption_key: str | None) -> None:
+        self.source_path = source.resolve()
+        self.target_path = target.resolve()
+        self.key = encryption_key
+        self.counts: dict[str, int] = defaultdict(int)
+        self.skipped: dict[str, str] = {
+            "sessions": "sessions are intentionally invalidated during cutover",
+            "user_credentials": "per-user LLM configuration was removed from the product",
+            "raw_factor_tables": "reproducible provider inputs are rebuilt by governed sync jobs",
+            "benchmark_bars": "legacy rows lack OHLC values required by the chart contract",
+        }
+        self.user_ids: set[int] = set()
+        self.run_ids: set[int] = set()
+
+    def run(self) -> dict[str, Any]:
+        if self.source_path == self.target_path:
+            raise ValueError("source and target database paths must differ")
+        if not self.source_path.is_file():
+            raise FileNotFoundError(self.source_path)
+        database = Database(self.target_path)
+        MigrationRunner(database).upgrade(MIGRATIONS)
+        source = sqlite3.connect(f"file:{self.source_path.as_posix()}?mode=ro", uri=True)
+        source.row_factory = sqlite3.Row
+        try:
+            with database.transaction() as target:
+                self._accounts(source, target)
+                self._system_settings(source, target)
+                self._market(source, target)
+                self._private_data(source, target)
+                self._screener(source, target)
+                self._insights(source, target)
+            with database.read() as target:
+                integrity = str(target.execute("PRAGMA integrity_check").fetchone()[0])
+                foreign_keys = list(target.execute("PRAGMA foreign_key_check"))
+                target_counts = {
+                    table: int(target.execute(f"SELECT count(*) FROM {table}").fetchone()[0])
+                    for table in (
+                        "users", "memberships", "market_entities", "market_summaries",
+                        "chart_series", "watchlist_entries", "review_notes", "trade_entries",
+                        "alerts", "mentor_messages", "heaven_readings", "screener_runs",
+                        "custom_screener_strategies", "strategy_tracks",
+                    )
+                }
+        finally:
+            source.close()
+        if integrity != "ok" or foreign_keys:
+            raise RuntimeError("migrated database failed integrity validation")
+        return {
+            "source": str(self.source_path),
+            "target": str(self.target_path),
+            "source_sha256": _hash_file(self.source_path),
+            "target_sha256": _hash_file(self.target_path),
+            "integrity": integrity,
+            "foreign_key_violations": 0,
+            "migrated": dict(sorted(self.counts.items())),
+            "target_counts": target_counts,
+            "intentionally_skipped": self.skipped,
+        }
+
+    def _accounts(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
+        users = source.execute("SELECT * FROM users ORDER BY id").fetchall()
+        for row in users:
+            user_id = int(row["id"])
+            self.user_ids.add(user_id)
+            password = f"scrypt$16384$8$1${row['password_salt']}${row['password_hash']}"
+            target.execute(
+                """INSERT INTO users
+                (id,username,username_key,password_hash,is_admin,status,created_at,updated_at)
+                VALUES (?,?,?,?,?,'active',?,?) ON CONFLICT(id) DO UPDATE SET
+                username=excluded.username, username_key=excluded.username_key,
+                password_hash=excluded.password_hash, is_admin=excluded.is_admin,
+                status=excluded.status, updated_at=excluded.updated_at""",
+                (
+                    user_id,
+                    row["username"],
+                    str(row["username"]).casefold(),
+                    password,
+                    int(str(row["role"]) == "admin"),
+                    row["created_at"],
+                    row["updated_at"],
+                ),
+            )
+            permanent = str(row["membership_plan"]) == "永久"
+            state = "active" if row["membership_status"] == "active" else "inactive"
+            target.execute(
+                """INSERT INTO memberships
+                (user_id,state,expires_at,is_permanent,daily_llm_limit,updated_at,updated_by)
+                VALUES (?,?,?,?,50,?,NULL) ON CONFLICT(user_id) DO UPDATE SET
+                state=excluded.state,expires_at=excluded.expires_at,
+                is_permanent=excluded.is_permanent,updated_at=excluded.updated_at""",
+                (user_id, state, row["membership_expires_at"], int(permanent), row["updated_at"]),
+            )
+        self.counts["users"] = len(users)
+        self.counts["memberships"] = len(users)
+        if _table_exists(source, "user_birth_profiles"):
+            for row in source.execute("SELECT * FROM user_birth_profiles"):
+                target.execute(
+                    """INSERT INTO birth_profiles
+                    (user_id,encrypted_payload,created_at,updated_at) VALUES (?,?,?,?)
+                    ON CONFLICT(user_id) DO UPDATE SET
+                    encrypted_payload=excluded.encrypted_payload,updated_at=excluded.updated_at""",
+                    (
+                        row["user_id"],
+                        row["encrypted_payload"],
+                        row["updated_at"],
+                        row["updated_at"],
+                    ),
+                )
+                self.counts["birth_profiles"] += 1
+        if _table_exists(source, "llm_usage"):
+            for row in source.execute(
+                """SELECT user_id,substr(created_at,1,10) usage_date,count(*) calls,
+                max(created_at) updated_at FROM llm_usage WHERE status='success'
+                GROUP BY user_id,substr(created_at,1,10)"""
+            ):
+                target.execute(
+                    """INSERT INTO llm_usage_daily VALUES (?,?,?,?)
+                    ON CONFLICT(user_id,usage_date) DO UPDATE SET
+                    successful_calls=excluded.successful_calls,updated_at=excluded.updated_at""",
+                    (row["user_id"], row["usage_date"], row["calls"], row["updated_at"]),
+                )
+                self.counts["llm_usage_daily"] += 1
+
+    def _system_settings(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
+        row = source.execute(
+            """SELECT encrypted_payload,updated_at FROM system_settings
+            WHERE setting_key='credentials'"""
+        ).fetchone()
+        if row is None:
+            return
+        if not self.key:
+            raise ValueError("APP_ENCRYPTION_KEY is required to migrate encrypted settings")
+        fernet = Fernet(self.key.encode("ascii"))
+        credentials = json.loads(fernet.decrypt(str(row["encrypted_payload"]).encode("ascii")))
+        admin_id = min(self.user_ids)
+        limit = max(1, min(int(credentials.get("member_daily_limit") or 50), 1000))
+        target.execute("UPDATE memberships SET daily_llm_limit=?", (limit,))
+        for name in ("tushare_token", "ifind_refresh_token", "ifind_access_token"):
+            value = str(credentials.get(name) or "").strip()
+            if value:
+                target.execute(
+                    """INSERT INTO system_credentials VALUES (?,?,?,?)
+                    ON CONFLICT(name) DO UPDATE SET encrypted_value=excluded.encrypted_value,
+                    updated_at=excluded.updated_at,updated_by=excluded.updated_by""",
+                    (name, fernet.encrypt(value.encode()).decode(), row["updated_at"], admin_id),
+                )
+                self.counts["system_credentials"] += 1
+        model_map: dict[str, int] = {}
+        for model in credentials.get("llm_models") or []:
+            key = str(model.get("id") or model.get("name") or "")
+            display = str(model.get("name") or model.get("model") or "model").strip()
+            api_key = str(model.get("api_key") or "")
+            target.execute(
+                """INSERT INTO llm_models
+                (display_name,display_name_key,base_url,model_identifier,encrypted_api_key,
+                 created_at,updated_at,updated_by) VALUES (?,?,?,?,?,?,?,?)
+                ON CONFLICT(display_name_key) DO UPDATE SET base_url=excluded.base_url,
+                model_identifier=excluded.model_identifier,encrypted_api_key=excluded.encrypted_api_key,
+                updated_at=excluded.updated_at,updated_by=excluded.updated_by""",
+                (
+                    display,
+                    display.casefold(),
+                    str(model.get("base_url") or ""),
+                    str(model.get("model") or ""),
+                    fernet.encrypt(api_key.encode()).decode(),
+                    row["updated_at"],
+                    row["updated_at"],
+                    admin_id,
+                ),
+            )
+            model_id = int(
+                target.execute(
+                    "SELECT id FROM llm_models WHERE display_name_key=?", (display.casefold(),)
+                ).fetchone()[0]
+            )
+            model_map[key] = model_id
+            self.counts["llm_models"] += 1
+        primary = model_map.get(str(credentials.get("primary_model_id") or ""))
+        fallback = model_map.get(str(credentials.get("fallback_model_id") or ""))
+        if fallback == primary:
+            fallback = None
+        target.execute(
+            """UPDATE llm_configuration SET primary_model_id=?,fallback_model_id=?,
+            updated_at=?,updated_by=? WHERE id=1""",
+            (primary, fallback, row["updated_at"], admin_id),
+        )
+
+    def _market(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
+        observed = datetime.now().astimezone().isoformat(timespec="seconds")
+        stocks = source.execute("SELECT * FROM stock_master ORDER BY ts_code").fetchall()
+        for row in stocks:
+            target.execute(
+                """INSERT INTO market_entities VALUES ('stock',?,?,?,?,?,1,'legacy',?)
+                ON CONFLICT(entity_type,identifier) DO UPDATE SET code=excluded.code,
+                name=excluded.name,search_key=excluded.search_key,sector=excluded.sector,
+                active=1,source='legacy',observed_at=excluded.observed_at""",
+                (
+                    row["ts_code"],
+                    row["code"],
+                    row["name"],
+                    f"{row['code']} {row['ts_code']} {row['name']} {row['industry']}".casefold(),
+                    row["industry"] or None,
+                    row["updated_at"] or observed,
+                ),
+            )
+        self.counts["market_entities"] = len(stocks)
+        dates = [
+            row[0]
+            for row in source.execute(
+                "SELECT DISTINCT trade_date FROM daily_bars ORDER BY trade_date"
+            )
+        ]
+        previous = None
+        for raw_date in dates:
+            trade_date = _iso_date(raw_date)
+            target.execute(
+                """INSERT INTO trading_days VALUES (?,1,?,'legacy',?)
+                ON CONFLICT(trade_date) DO UPDATE SET is_open=1,
+                previous_open_date=excluded.previous_open_date""",
+                (trade_date, previous, observed),
+            )
+            previous = trade_date
+        self.counts["trading_days"] = len(dates)
+        for row in source.execute("SELECT * FROM dashboard_snapshots"):
+            target.execute(
+                """INSERT INTO market_summaries VALUES (?,?,'archive','legacy',1,?,?)
+                ON CONFLICT(trade_date) DO UPDATE SET observed_at=excluded.observed_at,
+                state='archive',source='legacy',coverage=1,payload_json=excluded.payload_json,
+                created_at=excluded.created_at""",
+                (
+                    _iso_date(row["trade_date"]),
+                    row["updated_at"],
+                    row["payload"],
+                    row["updated_at"],
+                ),
+            )
+            self.counts["market_summaries"] += 1
+        query = """WITH ranked AS (
+            SELECT *,row_number() OVER (PARTITION BY ts_code ORDER BY trade_date DESC) rank
+            FROM daily_bars) SELECT * FROM ranked WHERE rank<=90 ORDER BY ts_code,trade_date"""
+        current = ""
+        points: list[dict[str, Any]] = []
+        for row in source.execute(query):
+            code = str(row["ts_code"])
+            if current and code != current:
+                self._save_chart(target, current, points, observed)
+                points = []
+            current = code
+            points.append(
+                {
+                    "time": _iso_date(row["trade_date"]),
+                    "open": row["open"],
+                    "high": row["high"],
+                    "low": row["low"],
+                    "close": row["close"],
+                    "volume": float(row["vol"] or 0) * 100,
+                    "amount": float(row["amount"] or 0) * 1000,
+                    "average": None,
+                }
+            )
+        if current:
+            self._save_chart(target, current, points, observed)
+
+    def _save_chart(
+        self,
+        target: sqlite3.Connection,
+        identifier: str,
+        points: list[dict[str, Any]],
+        observed: str,
+    ) -> None:
+        previous = points[-2]["close"] if len(points) > 1 else None
+        target.execute(
+            """INSERT INTO chart_series VALUES ('stock',?,'day',?,?,'tushare','display',
+            'none',1,?,?) ON CONFLICT(entity_type,identifier,interval,trade_date)
+            DO UPDATE SET payload_json=excluded.payload_json,observed_at=excluded.observed_at""",
+            (
+                identifier,
+                points[-1]["time"],
+                observed,
+                _dump({"previous_close": previous, "points": points}),
+                observed,
+            ),
+        )
+        self.counts["chart_series"] += 1
+
+    def _private_data(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
+        stock_ids = {
+            row["code"]: row["ts_code"]
+            for row in source.execute("SELECT code,ts_code FROM stock_master")
+        }
+        for row in source.execute("SELECT * FROM watchlist"):
+            if int(row["user_id"]) not in self.user_ids:
+                continue
+            target.execute(
+                """INSERT INTO watchlist_entries VALUES (?,?,?,?,?,?)
+                ON CONFLICT(user_id,identifier) DO UPDATE SET name=excluded.name,
+                sector=excluded.sector,remark=excluded.remark""",
+                (
+                    row["user_id"],
+                    stock_ids.get(row["code"], row["code"]),
+                    row["name"],
+                    row["sector"] or None,
+                    row["created_at"],
+                    row["remark"],
+                ),
+            )
+            self.counts["watchlist_entries"] += 1
+        self._copy_review_rows(source, target)
+        for row in source.execute("SELECT * FROM mentor_preferences"):
+            target.execute(
+                "INSERT OR REPLACE INTO mentor_preferences VALUES (?,?,?,?,?)",
+                (
+                    row["user_id"],
+                    row["mentor_id"],
+                    row["pinned"],
+                    row["sort_order"],
+                    row["updated_at"],
+                ),
+            )
+            self.counts["mentor_preferences"] += 1
+        for row in source.execute("SELECT * FROM mentor_messages"):
+            target.execute(
+                """INSERT OR REPLACE INTO mentor_messages
+                (id,user_id,mentor_id,trade_date,role,content,request_id,status,created_at)
+                VALUES (?,?,?,?,?,?,NULL,'complete',?)""",
+                (
+                    row["id"],
+                    row["user_id"],
+                    row["mentor_id"],
+                    _iso_date(row["trade_date"]),
+                    row["role"],
+                    row["content"],
+                    row["created_at"],
+                ),
+            )
+            self.counts["mentor_messages"] += 1
+        for row in source.execute("SELECT * FROM heaven_readings"):
+            snapshot = _json(row["context_snapshot"], {})
+            if row["subject_detail"]:
+                snapshot.setdefault("legacy_subject_detail", row["subject_detail"])
+            target.execute(
+                """INSERT OR REPLACE INTO heaven_readings
+                (id,user_id,mode,reading_date,subject_key,result_json,interpretation,
+                 interpretation_status,request_id,created_at,updated_at)
+                VALUES (?,?,?,?,?,?,?,'complete',NULL,?,?)""",
+                (
+                    row["id"],
+                    row["user_id"],
+                    row["mode"],
+                    _iso_date(row["context_date"]),
+                    row["subject"],
+                    _dump(snapshot),
+                    row["answer"],
+                    row["created_at"],
+                    row["created_at"],
+                ),
+            )
+            self.counts["heaven_readings"] += 1
+
+    def _copy_review_rows(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
+        for row in source.execute("SELECT * FROM review_notes WHERE user_id IS NOT NULL"):
+            target.execute(
+                """INSERT OR REPLACE INTO review_notes VALUES (?,?,?,?,?,?,?,?,?,?)""",
+                (
+                    row["id"],
+                    row["user_id"],
+                    row["code"],
+                    row["stock_name"],
+                    _iso_date(row["trade_date"]),
+                    row["summary"],
+                    row["content"],
+                    row["plan"],
+                    row["created_at"],
+                    row["updated_at"],
+                ),
+            )
+            self.counts["review_notes"] += 1
+        actions = {"buy", "sell", "add", "trim", "watch"}
+        emotions = {"calm", "confident", "hesitant", "anxious", "impulsive"}
+        for row in source.execute("SELECT * FROM trade_entries"):
+            target.execute(
+                """INSERT OR REPLACE INTO trade_entries
+                VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
+                (
+                    row["id"],
+                    row["user_id"],
+                    _iso_date(row["trade_date"]),
+                    row["code"],
+                    row["name"],
+                    row["action"] if row["action"] in actions else "watch",
+                    row["price"],
+                    row["quantity"],
+                    row["position_pct"],
+                    row["pnl_amount"],
+                    row["pnl_pct"],
+                    row["emotion"] if row["emotion"] in emotions else "calm",
+                    row["tags"],
+                    row["thesis"],
+                    row["execution"],
+                    row["created_at"],
+                    row["updated_at"],
+                ),
+            )
+            self.counts["trade_entries"] += 1
+        for row in source.execute("SELECT * FROM alerts"):
+            kind = (
+                row["kind"] if row["kind"] in {"manual", "strategy_t1", "strategy_t5"} else "manual"
+            )
+            target.execute(
+                "INSERT OR REPLACE INTO alerts VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
+                (
+                    row["id"],
+                    row["user_id"],
+                    kind,
+                    row["title"],
+                    row["content"],
+                    _iso_date(row["available_date"]),
+                    row["code"],
+                    row["dedupe_key"],
+                    row["is_read"],
+                    row["read_at"],
+                    row["created_at"],
+                    row["updated_at"],
+                ),
+            )
+            self.counts["alerts"] += 1
+        for row in source.execute("SELECT * FROM assistant_messages"):
+            target.execute(
+                """INSERT OR REPLACE INTO review_assistant_messages
+                VALUES (?,?,?,?,?,NULL,'complete',?)""",
+                (
+                    row["id"],
+                    row["user_id"],
+                    row["role"],
+                    row["content"],
+                    _iso_date(row["context_date"]),
+                    row["created_at"],
+                ),
+            )
+            self.counts["review_assistant_messages"] += 1
+
+    def _screener(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
+        snapshots: dict[str, int] = {}
+        for row in source.execute("SELECT * FROM screener_runs ORDER BY id"):
+            trade_date = _iso_date(row["trade_date"])
+            if trade_date not in snapshots:
+                target.execute(
+                    """INSERT OR IGNORE INTO screener_factor_snapshots
+                    (trade_date,version,observed_at,state,source_set_json,coverage_json,created_at)
+                    VALUES (?,? ,?,'archive','[\"legacy\"]','{}',?)""",
+                    (trade_date, ARCHIVE_VERSION, row["created_at"], row["created_at"]),
+                )
+                snapshots[trade_date] = int(
+                    target.execute(
+                        "SELECT id FROM screener_factor_snapshots WHERE trade_date=? AND version=?",
+                        (trade_date, ARCHIVE_VERSION),
+                    ).fetchone()[0]
+                )
+            result = _json(row["result"], {})
+            candidates = result.get("candidates") if isinstance(result, dict) else []
+            candidates = candidates if isinstance(candidates, list) else []
+            for candidate in candidates:
+                if isinstance(candidate, dict) and not candidate.get("identifier"):
+                    candidate["identifier"] = (
+                        candidate.get("ts_code") or candidate.get("code") or ""
+                    )
+            mode = {"smart": "stage", "curated": "curated", "quant": "custom"}.get(
+                str(row["mode"]), "custom"
+            )
+            formula = _json(row["formula"], {})
+            strategy_id = str(formula.get("id") or f"legacy-{row['id']}")
+            target.execute(
+                """INSERT OR REPLACE INTO screener_runs
+                (id,owner_user_id,mode,strategy_id,strategy_name,strategy_version,
+                 selection_date,factor_snapshot_id,status,started_at,completed_at,coverage,
+                 missing_fields_json,result_json,error_message)
+                VALUES (?,?,?,?,?,1,?,?,?, ?,?,0,'[]',?,'')""",
+                (
+                    row["id"],
+                    row["user_id"],
+                    mode,
+                    strategy_id,
+                    row["strategy_name"],
+                    trade_date,
+                    snapshots[trade_date],
+                    "completed" if candidates else "no_signal",
+                    row["created_at"],
+                    row["created_at"],
+                    _dump(candidates),
+                ),
+            )
+            self.run_ids.add(int(row["id"]))
+            self.counts["screener_runs"] += 1
+        for row in source.execute("SELECT * FROM screener_strategies WHERE user_id IS NOT NULL"):
+            target.execute(
+                """INSERT OR REPLACE INTO custom_screener_strategies
+                (id,user_id,name,version,formula_json,created_at,updated_at)
+                VALUES (?,?,?,1,?,?,?)""",
+                (
+                    row["id"],
+                    row["user_id"],
+                    row["name"],
+                    row["formula"],
+                    row["created_at"],
+                    row["updated_at"],
+                ),
+            )
+            self.counts["custom_screener_strategies"] += 1
+        for row in source.execute("SELECT * FROM strategy_tracks"):
+            if int(row["run_id"]) not in self.run_ids:
+                continue
+            target.execute(
+                """INSERT OR REPLACE INTO strategy_tracks
+                (id,user_id,run_id,identifier,code,name,sector,selection_date,
+                 strategy_name,entry_price,added_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
+                (
+                    row["id"],
+                    row["user_id"],
+                    row["run_id"],
+                    row["ts_code"],
+                    row["code"],
+                    row["name"],
+                    row["sector"],
+                    _iso_date(row["selection_date"]),
+                    row["strategy_name"],
+                    row["entry_price"],
+                    row["created_at"],
+                ),
+            )
+            self.counts["strategy_tracks"] += 1
+
+    def _insights(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
+        mappings = {
+            "auction_center_v6": "auction",
+            "theme_library_v1": "themes",
+            "popularity_v1": "popularity",
+            "dragon_tiger": "dragon-list",
+        }
+        for old_kind, new_kind in mappings.items():
+            for row in source.execute("SELECT * FROM data_snapshots WHERE kind=?", (old_kind,)):
+                trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0])
+                if not trade_date:
+                    continue
+                target.execute(
+                    """INSERT OR REPLACE INTO market_insight_snapshots
+                    VALUES (?,?,'',?,'archive','legacy',1,?)""",
+                    (new_kind, trade_date, row["updated_at"], row["payload"]),
+                )
+                self.counts["market_insight_snapshots"] += 1
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Migrate a read-only legacy database copy")
+    parser.add_argument("--source", type=Path, required=True)
+    parser.add_argument("--target", type=Path, required=True)
+    parser.add_argument("--report", type=Path)
+    return parser
+
+
+def main(arguments: Sequence[str] | None = None) -> int:
+    parsed = build_parser().parse_args(arguments)
+    report = LegacyMigrator(parsed.source, parsed.target, os.getenv("APP_ENCRYPTION_KEY")).run()
+    rendered = json.dumps(report, ensure_ascii=False, indent=2)
+    if parsed.report:
+        parsed.report.parent.mkdir(parents=True, exist_ok=True)
+        parsed.report.write_text(rendered + "\n", encoding="utf-8")
+    print(rendered)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

next/tools/restore.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import argparse
+import json
+import shutil
+import sqlite3
+import tempfile
+import zipfile
+from collections.abc import Sequence
+from contextlib import closing
+from pathlib import Path
+
+from tools.backup import sha256
+
+
+def _safe_members(archive: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
+    members = archive.infolist()
+    for member in members:
+        path = Path(member.filename)
+        if path.is_absolute() or ".." in path.parts:
+            raise ValueError("backup contains an unsafe path")
+    return members
+
+
+def restore_backup(
+    archive_path: Path,
+    database: Path,
+    *,
+    private_skills: Path | None = None,
+    environment_file: Path | None = None,
+    encryption_key_file: Path | None = None,
+    overwrite: bool = False,
+) -> dict:
+    archive_path = archive_path.resolve()
+    database = database.resolve()
+    destinations = [database]
+    destinations.extend(path.resolve() for path in (environment_file, encryption_key_file) if path)
+    if private_skills:
+        destinations.append(private_skills.resolve())
+    if not overwrite and any(path.exists() for path in destinations):
+        raise FileExistsError(
+            "restore destination exists; explicit overwrite confirmation is required"
+        )
+    with tempfile.TemporaryDirectory(prefix="xiaobai-restore-") as raw:
+        staging = Path(raw)
+        with zipfile.ZipFile(archive_path) as archive:
+            _safe_members(archive)
+            archive.extractall(staging)
+        manifest = json.loads((staging / "manifest.json").read_text(encoding="utf-8"))
+        if manifest.get("format") != 1 or not isinstance(manifest.get("files"), dict):
+            raise ValueError("backup manifest is invalid")
+        for name, metadata in manifest["files"].items():
+            path = staging / name
+            if not path.is_file() or sha256(path) != metadata.get("sha256"):
+                raise ValueError(f"backup checksum mismatch: {name}")
+        restored_db = staging / "database.sqlite3"
+        with closing(
+            sqlite3.connect(f"file:{restored_db.as_posix()}?mode=ro", uri=True)
+        ) as connection:
+            if connection.execute("PRAGMA integrity_check").fetchone()[0] != "ok":
+                raise ValueError("restored database failed integrity check")
+        database.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(restored_db, database)
+        for destination, name in (
+            (environment_file, "secrets/environment"),
+            (encryption_key_file, "secrets/app-encryption.key"),
+        ):
+            source = staging / name
+            if destination and source.is_file():
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy2(source, destination)
+        skills_source = staging / "private-mentor-skills"
+        if private_skills and skills_source.is_dir():
+            private_skills.mkdir(parents=True, exist_ok=True)
+            shutil.copytree(skills_source, private_skills, dirs_exist_ok=True)
+    return manifest
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Validate and restore a Xiaobai backup")
+    parser.add_argument("--archive", type=Path, required=True)
+    parser.add_argument("--database", type=Path, required=True)
+    parser.add_argument("--private-skills", type=Path)
+    parser.add_argument("--environment-file", type=Path)
+    parser.add_argument("--encryption-key-file", type=Path)
+    parser.add_argument("--confirm-restore", action="store_true")
+    parser.add_argument("--confirm-overwrite", action="store_true")
+    return parser
+
+
+def main(arguments: Sequence[str] | None = None) -> int:
+    parsed = build_parser().parse_args(arguments)
+    if not parsed.confirm_restore:
+        raise SystemExit("restore requires --confirm-restore")
+    manifest = restore_backup(
+        parsed.archive,
+        parsed.database,
+        private_skills=parsed.private_skills,
+        environment_file=parsed.environment_file,
+        encryption_key_file=parsed.encryption_key_file,
+        overwrite=parsed.confirm_overwrite,
+    )
+    print(
+        json.dumps(
+            {"restored": str(parsed.database.resolve()), "files": len(manifest["files"])},
+            ensure_ascii=False,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())