rebuild(migration): validate local legacy cutover

next/tools/legacy_migration.py

@@ -13,6 +13,7 @@ from typing import Any
 
 from cryptography.fernet import Fernet
 
+from backend.data.sentiment import calculate_sentiment
 from backend.database import MIGRATIONS, Database, MigrationRunner
 
 ARCHIVE_VERSION = "legacy-archive-v1"
@@ -146,6 +147,121 @@ def _normalize_identifiers(value: Any) -> Any:
     return normalized
 
 
+def _normalize_market_units(value: Any) -> Any:
+    if isinstance(value, list):
+        return [_normalize_market_units(item) for item in value]
+    if not isinstance(value, dict):
+        return value
+    normalized = {key: _normalize_market_units(item) for key, item in value.items()}
+    for legacy, canonical, multiplier in (
+        ("amount_billion", "amount", 100_000_000),
+        ("seal_amount_million", "seal_amount", 1_000_000),
+        ("float_mv_billion", "float_mv", 100_000_000),
+    ):
+        raw = normalized.pop(legacy, None)
+        if canonical not in normalized and raw not in (None, ""):
+            normalized[canonical] = float(raw) * multiplier
+    return normalized
+
+
+def _legacy_sentiment(
+    payload: dict[str, Any], history: list[dict[str, Any]]
+) -> dict[str, Any]:
+    calculated = calculate_sentiment(payload, history)
+    overview = payload.get("overview") or {}
+    if int(overview.get("sentiment_engine_version") or 0) < 2:
+        return calculated
+    score = overview.get("sentiment_score")
+    if score is None or not overview.get("sentiment_phase"):
+        return calculated
+    previous_scores = [
+        float(item["sentiment"]["score"])
+        for item in history[-3:]
+        if (item.get("sentiment") or {}).get("score") is not None
+    ]
+    numeric_score = float(score)
+    previous_score = previous_scores[-1] if previous_scores else numeric_score
+    baseline = sum(previous_scores) / len(previous_scores) if previous_scores else numeric_score
+    components = overview.get("sentiment_components") or {}
+    if isinstance(components, dict):
+        calculated["components"] = [
+            {"key": key, **dict(component)}
+            for key, component in components.items()
+            if isinstance(component, dict)
+        ]
+    calculated.update(
+        {
+            "score": int(round(numeric_score)),
+            "label": str(overview.get("sentiment_label") or calculated["label"]),
+            "phase": str(overview["sentiment_phase"]),
+            "direction": str(overview.get("sentiment_direction") or calculated["direction"]),
+            "day_change": round(numeric_score - previous_score, 1),
+            "momentum": round(numeric_score - baseline, 1),
+        }
+    )
+    return calculated
+
+
+def _normalize_market_snapshot(
+    value: dict[str, Any], trade_date: str, history: list[dict[str, Any]]
+) -> dict[str, Any]:
+    payload = _normalize_market_units(_normalize_identifiers(value))
+    overview = dict(payload.get("overview") or {})
+    for legacy, canonical, fallback in (
+        ("limit_up_count", "limit_up", len(payload.get("limits") or [])),
+        ("limit_down_count", "limit_down", len(payload.get("down_limits") or [])),
+        ("broken_count", "broken", len(payload.get("broken") or [])),
+    ):
+        raw = overview.pop(legacy, None)
+        if canonical not in overview:
+            overview[canonical] = raw if raw is not None else fallback
+    if "amount" not in overview:
+        raw_amount = overview.pop("amount_billion", None)
+        overview["amount"] = float(raw_amount or 0) * 100_000_000
+    meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {}
+    payload["trade_date"] = trade_date
+    previous_date = payload.get("previous_trade_date") or meta.get("previous_trade_date")
+    payload["previous_trade_date"] = _iso_date(str(previous_date)) if previous_date else ""
+    payload["overview"] = overview
+    sentiment = _legacy_sentiment(payload, history)
+    payload["sentiment"] = sentiment
+    payload["temperature"] = sentiment["score"]
+    for key in tuple(overview):
+        if key.startswith("sentiment_"):
+            overview.pop(key)
+    return payload
+
+
+def _normalize_birth_profile(encrypted_payload: str, encryption_key: str | None) -> str | None:
+    if not encryption_key:
+        raise ValueError("legacy birth profiles require APP_ENCRYPTION_KEY")
+    fernet = Fernet(encryption_key.encode("ascii"))
+    decrypted = fernet.decrypt(encrypted_payload.encode("ascii"))
+    try:
+        payload = json.loads(decrypted)
+        gender = str(payload.get("gender") or "")
+        birth_date = str(payload.get("birth_date") or "")
+        birth_time = str(payload.get("birth_time") or "")
+        if (not birth_date or not birth_time) and payload.get("birth_datetime"):
+            stamp = datetime.fromisoformat(str(payload["birth_datetime"]).replace(" ", "T", 1))
+            birth_date = stamp.date().isoformat()
+            birth_time = stamp.time().replace(tzinfo=None, second=0, microsecond=0).isoformat(
+                timespec="minutes"
+            )
+        datetime.fromisoformat(f"{birth_date}T{birth_time}")
+        if gender not in {"male", "female"}:
+            return None
+    except (json.JSONDecodeError, TypeError, ValueError):
+        return None
+    normalized = json.dumps(
+        {"birth_date": birth_date, "birth_time": birth_time, "gender": gender},
+        ensure_ascii=False,
+        separators=(",", ":"),
+        sort_keys=True,
+    )
+    return fernet.encrypt(normalized.encode("utf-8")).decode("ascii")
+
+
 def _observed_at(payload: dict[str, Any], fallback: str) -> str:
     meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {}
     return str(
@@ -295,6 +411,18 @@ class LegacyMigrator:
         self.admin_id = min(administrators or self.user_ids)
         if _table_exists(source, "user_birth_profiles"):
             for row in source.execute("SELECT * FROM user_birth_profiles"):
+                encrypted_profile = _normalize_birth_profile(
+                    str(row["encrypted_payload"]), self.key
+                )
+                if encrypted_profile is None:
+                    self.row_skips["birth_profiles_invalid"] = {
+                        "count": self.row_skips.get("birth_profiles_invalid", {}).get(
+                            "count", 0
+                        )
+                        + 1,
+                        "reason": "legacy profile is incomplete and must be configured again",
+                    }
+                    continue
                 target.execute(
                     """INSERT INTO birth_profiles
                     (user_id,encrypted_payload,created_at,updated_at) VALUES (?,?,?,?)
@@ -302,7 +430,7 @@ class LegacyMigrator:
                     encrypted_payload=excluded.encrypted_payload,updated_at=excluded.updated_at""",
                     (
                         row["user_id"],
-                        row["encrypted_payload"],
+                        encrypted_profile,
                         row["updated_at"],
                         row["updated_at"],
                     ),
@@ -425,9 +553,21 @@ class LegacyMigrator:
             )
             previous = trade_date
         self.counts["trading_days"] = len(dates)
-        for row in source.execute("SELECT * FROM dashboard_snapshots"):
-            payload = _normalize_identifiers(_json(row["payload"], {}))
-            trade_date = _iso_date(row["trade_date"])
+        snapshots: dict[str, tuple[sqlite3.Row, dict[str, Any]]] = {}
+        for row in source.execute(
+            "SELECT * FROM dashboard_snapshots ORDER BY trade_date,updated_at"
+        ):
+            payload = _json(row["payload"], {})
+            meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {}
+            trade_date = _iso_date(
+                str(payload.get("trade_date") or meta.get("trade_date") or row["trade_date"])
+            )
+            snapshots[trade_date] = (row, payload)
+        history: list[dict[str, Any]] = []
+        for trade_date in sorted(snapshots):
+            row, raw_payload = snapshots[trade_date]
+            payload = _normalize_market_snapshot(raw_payload, trade_date, history)
+            history.append(payload)
             events: dict[str, str] = {}
             for event_type, key in (
                 ("limit_up", "limits"),
@@ -456,7 +596,7 @@ class LegacyMigrator:
                     row["updated_at"],
                 ),
             )
-            self.counts["market_summaries"] += 1
+        self.counts["market_summaries"] = len(snapshots)
         query = """WITH ranked AS (
             SELECT *,row_number() OVER (PARTITION BY ts_code ORDER BY trade_date DESC) rank
             FROM daily_bars) SELECT * FROM ranked WHERE rank<=90 ORDER BY ts_code,trade_date"""