1525 lines
65 KiB
Python
1525 lines
65 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
from collections import defaultdict
|
|
from collections.abc import Sequence
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from cryptography.fernet import Fernet
|
|
|
|
from backend.data.sentiment import calculate_sentiment
|
|
from backend.database import MIGRATIONS, Database, MigrationRunner
|
|
from backend.features.screener.catalog import strategy_catalog
|
|
|
|
ARCHIVE_VERSION = "legacy-archive-v1"
|
|
|
|
HANDLED_SOURCE_TABLES = frozenset(
|
|
{
|
|
"alerts",
|
|
"assistant_messages",
|
|
"dashboard_snapshots",
|
|
"data_snapshots",
|
|
"daily_bars",
|
|
"heaven_readings",
|
|
"llm_usage",
|
|
"mentor_messages",
|
|
"mentor_preferences",
|
|
"reason_overrides",
|
|
"review_notes",
|
|
"screener_runs",
|
|
"screener_strategies",
|
|
"seat_aliases",
|
|
"stock_master",
|
|
"strategy_tracks",
|
|
"system_settings",
|
|
"trade_entries",
|
|
"user_birth_profiles",
|
|
"users",
|
|
"watchlist",
|
|
}
|
|
)
|
|
|
|
SKIPPED_SOURCE_TABLES = {
|
|
"auction_factors": "governed provider inputs are rebuilt by scheduled jobs",
|
|
"benchmark_bars": "legacy rows lack OHLC values required by the chart contract",
|
|
"daily_indicators": "governed provider inputs are rebuilt by scheduled jobs",
|
|
"earnings_events": "governed provider inputs are rebuilt by scheduled jobs",
|
|
"fundamental_indicators": "governed provider inputs are rebuilt by scheduled jobs",
|
|
"job_runs": "legacy operational logs are not user-facing history",
|
|
"lhb_institution_daily": "governed provider inputs are rebuilt by scheduled jobs",
|
|
"moneyflow_daily": "governed provider inputs are rebuilt by scheduled jobs",
|
|
"popularity_factors": "governed provider inputs are rebuilt by scheduled jobs",
|
|
"schema_migrations": "legacy implementation metadata does not apply to the new schema",
|
|
"sector_phase_overrides": "sector phase overrides are no longer product-configurable",
|
|
"sync_runs": "legacy operational logs are not user-facing history",
|
|
"user_credentials": "per-user LLM configuration was removed from the product",
|
|
"user_sessions": "sessions are intentionally invalidated during cutover",
|
|
"wencai_saved_queries": "the WenCai feature was explicitly removed from the product",
|
|
}
|
|
|
|
HANDLED_SNAPSHOT_KINDS = frozenset(
|
|
{
|
|
*(f"auction_center_v{version}" for version in range(1, 7)),
|
|
"dragon_tiger",
|
|
"hot_money_detail_v2",
|
|
"hot_money_detail_v3",
|
|
"hot_money_profiles_v1",
|
|
"ifind_event_enrichment_v1",
|
|
"popularity_v1",
|
|
"rotation_sector_members_v1",
|
|
"theme_detail_v1",
|
|
"theme_directory_v1",
|
|
"theme_library_v1",
|
|
}
|
|
)
|
|
|
|
SKIPPED_SNAPSHOT_KINDS = {
|
|
"dashboard_request_v1": "request cache is superseded by archived dashboard snapshots",
|
|
"heaven_indices": "rebuildable input cache; saved Heaven readings are migrated",
|
|
"heaven_sector": "rebuildable input cache; saved Heaven readings are migrated",
|
|
"heaven_stock": "rebuildable input cache; saved Heaven readings are migrated",
|
|
"screener_auto_v1": "derived cache is superseded by migrated screener runs",
|
|
"search_directory": "search data is rebuilt from the migrated entity directory",
|
|
"stock_detail": "rebuildable display cache",
|
|
"stock_intraday": "rebuildable realtime display cache",
|
|
}
|
|
|
|
|
|
def _iso_date(value: Any) -> str:
|
|
text = str(value or "").strip()
|
|
compact = text[:10].replace("-", "")
|
|
return f"{compact[:4]}-{compact[4:6]}-{compact[6:8]}" if len(compact) >= 8 else ""
|
|
|
|
|
|
def _json(value: Any, fallback: Any) -> Any:
|
|
if isinstance(value, (dict, list)):
|
|
return value
|
|
try:
|
|
return json.loads(str(value))
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
return fallback
|
|
|
|
|
|
def _dump(value: Any) -> str:
|
|
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
|
|
|
|
def _formula_fingerprint(value: Any) -> str:
|
|
formula = _json(value, {})
|
|
if not isinstance(formula, dict):
|
|
return ""
|
|
normalized = json.loads(json.dumps(formula, ensure_ascii=False))
|
|
metadata = normalized.get("meta")
|
|
if isinstance(metadata, dict):
|
|
metadata.pop("library_version", None)
|
|
return json.dumps(normalized, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
|
|
|
|
def _hash_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _table_exists(connection: sqlite3.Connection, table: str) -> bool:
|
|
return (
|
|
connection.execute(
|
|
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)
|
|
).fetchone()
|
|
is not None
|
|
)
|
|
|
|
|
|
def _table_names(connection: sqlite3.Connection) -> tuple[str, ...]:
|
|
return tuple(
|
|
str(row[0])
|
|
for row in connection.execute(
|
|
"""SELECT name FROM sqlite_master
|
|
WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"""
|
|
)
|
|
)
|
|
|
|
|
|
def _normalize_identifiers(value: Any) -> Any:
|
|
if isinstance(value, list):
|
|
return [_normalize_identifiers(item) for item in value]
|
|
if not isinstance(value, dict):
|
|
return value
|
|
normalized = {key: _normalize_identifiers(item) for key, item in value.items()}
|
|
if normalized.get("ts_code") and not normalized.get("identifier"):
|
|
normalized["identifier"] = str(normalized["ts_code"]).upper()
|
|
return normalized
|
|
|
|
|
|
def _normalize_market_units(value: Any) -> Any:
|
|
if isinstance(value, list):
|
|
return [_normalize_market_units(item) for item in value]
|
|
if not isinstance(value, dict):
|
|
return value
|
|
normalized = {key: _normalize_market_units(item) for key, item in value.items()}
|
|
for legacy, canonical, multiplier in (
|
|
("amount_billion", "amount", 100_000_000),
|
|
("seal_amount_million", "seal_amount", 1_000_000),
|
|
("float_mv_billion", "float_mv", 100_000_000),
|
|
):
|
|
raw = normalized.pop(legacy, None)
|
|
if canonical not in normalized and raw not in (None, ""):
|
|
normalized[canonical] = float(raw) * multiplier
|
|
return normalized
|
|
|
|
|
|
def _legacy_sentiment(
|
|
payload: dict[str, Any], history: list[dict[str, Any]]
|
|
) -> dict[str, Any]:
|
|
calculated = calculate_sentiment(payload, history)
|
|
overview = payload.get("overview") or {}
|
|
if int(overview.get("sentiment_engine_version") or 0) < 2:
|
|
return calculated
|
|
score = overview.get("sentiment_score")
|
|
if score is None or not overview.get("sentiment_phase"):
|
|
return calculated
|
|
previous_scores = [
|
|
float(item["sentiment"]["score"])
|
|
for item in history[-3:]
|
|
if (item.get("sentiment") or {}).get("score") is not None
|
|
]
|
|
numeric_score = float(score)
|
|
previous_score = previous_scores[-1] if previous_scores else numeric_score
|
|
baseline = sum(previous_scores) / len(previous_scores) if previous_scores else numeric_score
|
|
components = overview.get("sentiment_components") or {}
|
|
if isinstance(components, dict):
|
|
calculated["components"] = [
|
|
{"key": key, **dict(component)}
|
|
for key, component in components.items()
|
|
if isinstance(component, dict)
|
|
]
|
|
calculated.update(
|
|
{
|
|
"score": int(round(numeric_score)),
|
|
"label": str(overview.get("sentiment_label") or calculated["label"]),
|
|
"phase": str(overview["sentiment_phase"]),
|
|
"direction": str(overview.get("sentiment_direction") or calculated["direction"]),
|
|
"day_change": round(numeric_score - previous_score, 1),
|
|
"momentum": round(numeric_score - baseline, 1),
|
|
}
|
|
)
|
|
return calculated
|
|
|
|
|
|
def _normalize_market_snapshot(
|
|
value: dict[str, Any], trade_date: str, history: list[dict[str, Any]]
|
|
) -> dict[str, Any]:
|
|
payload = _normalize_market_units(_normalize_identifiers(value))
|
|
overview = dict(payload.get("overview") or {})
|
|
for legacy, canonical, fallback in (
|
|
("limit_up_count", "limit_up", len(payload.get("limits") or [])),
|
|
("limit_down_count", "limit_down", len(payload.get("down_limits") or [])),
|
|
("broken_count", "broken", len(payload.get("broken") or [])),
|
|
):
|
|
raw = overview.pop(legacy, None)
|
|
if canonical not in overview:
|
|
overview[canonical] = raw if raw is not None else fallback
|
|
if "amount" not in overview:
|
|
raw_amount = overview.pop("amount_billion", None)
|
|
overview["amount"] = float(raw_amount or 0) * 100_000_000
|
|
meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {}
|
|
payload["trade_date"] = trade_date
|
|
previous_date = payload.get("previous_trade_date") or meta.get("previous_trade_date")
|
|
payload["previous_trade_date"] = _iso_date(str(previous_date)) if previous_date else ""
|
|
payload["overview"] = overview
|
|
sentiment = _legacy_sentiment(payload, history)
|
|
payload["sentiment"] = sentiment
|
|
payload["temperature"] = sentiment["score"]
|
|
for key in tuple(overview):
|
|
if key.startswith("sentiment_"):
|
|
overview.pop(key)
|
|
return payload
|
|
|
|
|
|
def _normalize_birth_profile(encrypted_payload: str, encryption_key: str | None) -> str | None:
|
|
if not encryption_key:
|
|
raise ValueError("legacy birth profiles require APP_ENCRYPTION_KEY")
|
|
fernet = Fernet(encryption_key.encode("ascii"))
|
|
decrypted = fernet.decrypt(encrypted_payload.encode("ascii"))
|
|
try:
|
|
payload = json.loads(decrypted)
|
|
gender = str(payload.get("gender") or "")
|
|
birth_date = str(payload.get("birth_date") or "")
|
|
birth_time = str(payload.get("birth_time") or "")
|
|
if (not birth_date or not birth_time) and payload.get("birth_datetime"):
|
|
stamp = datetime.fromisoformat(str(payload["birth_datetime"]).replace(" ", "T", 1))
|
|
birth_date = stamp.date().isoformat()
|
|
birth_time = stamp.time().replace(tzinfo=None, second=0, microsecond=0).isoformat(
|
|
timespec="minutes"
|
|
)
|
|
datetime.fromisoformat(f"{birth_date}T{birth_time}")
|
|
if gender not in {"male", "female"}:
|
|
return None
|
|
except (json.JSONDecodeError, TypeError, ValueError):
|
|
return None
|
|
normalized = json.dumps(
|
|
{"birth_date": birth_date, "birth_time": birth_time, "gender": gender},
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
sort_keys=True,
|
|
)
|
|
return fernet.encrypt(normalized.encode("utf-8")).decode("ascii")
|
|
|
|
|
|
def _observed_at(payload: dict[str, Any], fallback: str) -> str:
|
|
meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {}
|
|
return str(
|
|
payload.get("observed_at")
|
|
or meta.get("updated_at")
|
|
or meta.get("generated_at")
|
|
or fallback
|
|
)
|
|
|
|
|
|
def _time(value: Any) -> str:
|
|
text = str(value or "").strip()
|
|
return text[:5] if len(text) >= 5 else ""
|
|
|
|
|
|
class LegacyMigrator:
|
|
def __init__(self, source: Path, target: Path, encryption_key: str | None) -> None:
|
|
self.source_path = source.resolve()
|
|
self.target_path = target.resolve()
|
|
self.key = encryption_key
|
|
self.counts: dict[str, int] = defaultdict(int)
|
|
self.row_skips: dict[str, dict[str, Any]] = {}
|
|
self.user_ids: set[int] = set()
|
|
self.run_ids: set[int] = set()
|
|
self.run_id_map: dict[int, int] = {}
|
|
self.stock_ids: dict[str, str] = {}
|
|
self.admin_id: int | None = None
|
|
self.dashboard_events: dict[str, dict[str, str]] = {}
|
|
|
|
def run(self) -> dict[str, Any]:
|
|
if self.source_path == self.target_path:
|
|
raise ValueError("source and target database paths must differ")
|
|
if not self.source_path.is_file():
|
|
raise FileNotFoundError(self.source_path)
|
|
database = Database(self.target_path)
|
|
MigrationRunner(database).upgrade(MIGRATIONS)
|
|
source = sqlite3.connect(f"file:{self.source_path.as_posix()}?mode=ro", uri=True)
|
|
source.row_factory = sqlite3.Row
|
|
try:
|
|
source_tables = _table_names(source)
|
|
source_table_counts = {
|
|
table: int(source.execute(f"SELECT count(*) FROM {table}").fetchone()[0])
|
|
for table in source_tables
|
|
}
|
|
snapshot_counts = {
|
|
str(row["kind"]): int(row["count"])
|
|
for row in source.execute(
|
|
"SELECT kind,count(*) AS count FROM data_snapshots GROUP BY kind ORDER BY kind"
|
|
)
|
|
}
|
|
handled_tables = sorted(set(source_tables) & HANDLED_SOURCE_TABLES)
|
|
skipped_tables = {
|
|
table: SKIPPED_SOURCE_TABLES[table]
|
|
for table in sorted(set(source_tables) & SKIPPED_SOURCE_TABLES.keys())
|
|
}
|
|
unmapped_tables = sorted(
|
|
set(source_tables) - HANDLED_SOURCE_TABLES - SKIPPED_SOURCE_TABLES.keys()
|
|
)
|
|
handled_snapshots = sorted(set(snapshot_counts) & HANDLED_SNAPSHOT_KINDS)
|
|
skipped_snapshots = {
|
|
kind: SKIPPED_SNAPSHOT_KINDS[kind]
|
|
for kind in sorted(set(snapshot_counts) & SKIPPED_SNAPSHOT_KINDS.keys())
|
|
}
|
|
unmapped_snapshots = sorted(
|
|
set(snapshot_counts) - HANDLED_SNAPSHOT_KINDS - SKIPPED_SNAPSHOT_KINDS.keys()
|
|
)
|
|
if unmapped_tables or unmapped_snapshots:
|
|
raise RuntimeError(
|
|
"unmapped legacy data: "
|
|
f"tables={unmapped_tables}, snapshot_kinds={unmapped_snapshots}"
|
|
)
|
|
with database.transaction() as target:
|
|
self._accounts(source, target)
|
|
self._system_settings(source, target)
|
|
self._market(source, target)
|
|
self._private_data(source, target)
|
|
self._screener(source, target)
|
|
self._insights(source, target)
|
|
with database.read() as target:
|
|
integrity = str(target.execute("PRAGMA integrity_check").fetchone()[0])
|
|
foreign_keys = list(target.execute("PRAGMA foreign_key_check"))
|
|
target_counts = {
|
|
table: int(target.execute(f"SELECT count(*) FROM {table}").fetchone()[0])
|
|
for table in _table_names(target)
|
|
}
|
|
finally:
|
|
source.close()
|
|
if integrity != "ok" or foreign_keys:
|
|
raise RuntimeError("migrated database failed integrity validation")
|
|
return {
|
|
"source": str(self.source_path),
|
|
"target": str(self.target_path),
|
|
"source_sha256": _hash_file(self.source_path),
|
|
"target_sha256": _hash_file(self.target_path),
|
|
"integrity": integrity,
|
|
"foreign_key_violations": 0,
|
|
"migrated": dict(sorted(self.counts.items())),
|
|
"source_tables": source_table_counts,
|
|
"handled_tables": handled_tables,
|
|
"intentionally_skipped_tables": skipped_tables,
|
|
"unmapped_tables": unmapped_tables,
|
|
"source_snapshot_kinds": snapshot_counts,
|
|
"handled_snapshot_kinds": handled_snapshots,
|
|
"intentionally_skipped_snapshot_kinds": skipped_snapshots,
|
|
"unmapped_snapshot_kinds": unmapped_snapshots,
|
|
"intentionally_skipped_rows": self.row_skips,
|
|
"target_counts": target_counts,
|
|
}
|
|
|
|
def _accounts(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
|
users = source.execute("SELECT * FROM users ORDER BY id").fetchall()
|
|
for row in users:
|
|
user_id = int(row["id"])
|
|
self.user_ids.add(user_id)
|
|
password = f"scrypt$16384$8$1${row['password_salt']}${row['password_hash']}"
|
|
target.execute(
|
|
"""INSERT INTO users
|
|
(id,username,username_key,password_hash,is_admin,status,created_at,updated_at)
|
|
VALUES (?,?,?,?,?,'active',?,?) ON CONFLICT(id) DO UPDATE SET
|
|
username=excluded.username, username_key=excluded.username_key,
|
|
password_hash=excluded.password_hash, is_admin=excluded.is_admin,
|
|
status=excluded.status, updated_at=excluded.updated_at""",
|
|
(
|
|
user_id,
|
|
row["username"],
|
|
str(row["username"]).casefold(),
|
|
password,
|
|
int(str(row["role"]) == "admin"),
|
|
row["created_at"],
|
|
row["updated_at"],
|
|
),
|
|
)
|
|
permanent = str(row["membership_plan"]) == "永久"
|
|
state = "active" if row["membership_status"] == "active" else "inactive"
|
|
target.execute(
|
|
"""INSERT INTO memberships
|
|
(user_id,state,expires_at,is_permanent,daily_llm_limit,updated_at,updated_by)
|
|
VALUES (?,?,?,?,50,?,NULL) ON CONFLICT(user_id) DO UPDATE SET
|
|
state=excluded.state,expires_at=excluded.expires_at,
|
|
is_permanent=excluded.is_permanent,updated_at=excluded.updated_at""",
|
|
(user_id, state, row["membership_expires_at"], int(permanent), row["updated_at"]),
|
|
)
|
|
self.counts["users"] = len(users)
|
|
self.counts["memberships"] = len(users)
|
|
administrators = [
|
|
int(row["id"]) for row in users if str(row["role"]) == "admin"
|
|
]
|
|
self.admin_id = min(administrators or self.user_ids)
|
|
if _table_exists(source, "user_birth_profiles"):
|
|
for row in source.execute("SELECT * FROM user_birth_profiles"):
|
|
encrypted_profile = _normalize_birth_profile(
|
|
str(row["encrypted_payload"]), self.key
|
|
)
|
|
if encrypted_profile is None:
|
|
self.row_skips["birth_profiles_invalid"] = {
|
|
"count": self.row_skips.get("birth_profiles_invalid", {}).get(
|
|
"count", 0
|
|
)
|
|
+ 1,
|
|
"reason": "legacy profile is incomplete and must be configured again",
|
|
}
|
|
continue
|
|
target.execute(
|
|
"""INSERT INTO birth_profiles
|
|
(user_id,encrypted_payload,created_at,updated_at) VALUES (?,?,?,?)
|
|
ON CONFLICT(user_id) DO UPDATE SET
|
|
encrypted_payload=excluded.encrypted_payload,updated_at=excluded.updated_at""",
|
|
(
|
|
row["user_id"],
|
|
encrypted_profile,
|
|
row["updated_at"],
|
|
row["updated_at"],
|
|
),
|
|
)
|
|
self.counts["birth_profiles"] += 1
|
|
if _table_exists(source, "llm_usage"):
|
|
for row in source.execute(
|
|
"""SELECT user_id,substr(created_at,1,10) usage_date,count(*) calls,
|
|
max(created_at) updated_at FROM llm_usage WHERE status='success'
|
|
GROUP BY user_id,substr(created_at,1,10)"""
|
|
):
|
|
target.execute(
|
|
"""INSERT INTO llm_usage_daily VALUES (?,?,?,?)
|
|
ON CONFLICT(user_id,usage_date) DO UPDATE SET
|
|
successful_calls=excluded.successful_calls,updated_at=excluded.updated_at""",
|
|
(row["user_id"], row["usage_date"], row["calls"], row["updated_at"]),
|
|
)
|
|
self.counts["llm_usage_daily"] += 1
|
|
|
|
def _system_settings(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
|
row = source.execute(
|
|
"""SELECT encrypted_payload,updated_at FROM system_settings
|
|
WHERE setting_key='credentials'"""
|
|
).fetchone()
|
|
if row is None:
|
|
return
|
|
if not self.key:
|
|
raise ValueError("APP_ENCRYPTION_KEY is required to migrate encrypted settings")
|
|
fernet = Fernet(self.key.encode("ascii"))
|
|
credentials = json.loads(fernet.decrypt(str(row["encrypted_payload"]).encode("ascii")))
|
|
admin_id = min(self.user_ids)
|
|
limit = max(1, min(int(credentials.get("member_daily_limit") or 50), 1000))
|
|
target.execute("UPDATE memberships SET daily_llm_limit=?", (limit,))
|
|
for name in ("tushare_token", "ifind_refresh_token", "ifind_access_token"):
|
|
value = str(credentials.get(name) or "").strip()
|
|
if value:
|
|
target.execute(
|
|
"""INSERT INTO system_credentials VALUES (?,?,?,?)
|
|
ON CONFLICT(name) DO UPDATE SET encrypted_value=excluded.encrypted_value,
|
|
updated_at=excluded.updated_at,updated_by=excluded.updated_by""",
|
|
(name, fernet.encrypt(value.encode()).decode(), row["updated_at"], admin_id),
|
|
)
|
|
self.counts["system_credentials"] += 1
|
|
model_map: dict[str, int] = {}
|
|
for model in credentials.get("llm_models") or []:
|
|
key = str(model.get("id") or model.get("name") or "")
|
|
display = str(model.get("name") or model.get("model") or "model").strip()
|
|
api_key = str(model.get("api_key") or "")
|
|
target.execute(
|
|
"""INSERT INTO llm_models
|
|
(display_name,display_name_key,base_url,model_identifier,encrypted_api_key,
|
|
created_at,updated_at,updated_by) VALUES (?,?,?,?,?,?,?,?)
|
|
ON CONFLICT(display_name_key) DO UPDATE SET base_url=excluded.base_url,
|
|
model_identifier=excluded.model_identifier,encrypted_api_key=excluded.encrypted_api_key,
|
|
updated_at=excluded.updated_at,updated_by=excluded.updated_by""",
|
|
(
|
|
display,
|
|
display.casefold(),
|
|
str(model.get("base_url") or ""),
|
|
str(model.get("model") or ""),
|
|
fernet.encrypt(api_key.encode()).decode(),
|
|
row["updated_at"],
|
|
row["updated_at"],
|
|
admin_id,
|
|
),
|
|
)
|
|
model_id = int(
|
|
target.execute(
|
|
"SELECT id FROM llm_models WHERE display_name_key=?", (display.casefold(),)
|
|
).fetchone()[0]
|
|
)
|
|
model_map[key] = model_id
|
|
self.counts["llm_models"] += 1
|
|
primary = model_map.get(str(credentials.get("primary_model_id") or ""))
|
|
fallback = model_map.get(str(credentials.get("fallback_model_id") or ""))
|
|
if fallback == primary:
|
|
fallback = None
|
|
target.execute(
|
|
"""UPDATE llm_configuration SET primary_model_id=?,fallback_model_id=?,
|
|
updated_at=?,updated_by=? WHERE id=1""",
|
|
(primary, fallback, row["updated_at"], admin_id),
|
|
)
|
|
|
|
def _market(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
|
observed = datetime.now().astimezone().isoformat(timespec="seconds")
|
|
stocks = source.execute("SELECT * FROM stock_master ORDER BY ts_code").fetchall()
|
|
self.stock_ids = {
|
|
str(row["code"]): str(row["ts_code"]).upper() for row in stocks
|
|
}
|
|
for row in stocks:
|
|
target.execute(
|
|
"""INSERT INTO market_entities VALUES ('stock',?,?,?,?,?,1,'legacy',?)
|
|
ON CONFLICT(entity_type,identifier) DO UPDATE SET code=excluded.code,
|
|
name=excluded.name,search_key=excluded.search_key,sector=excluded.sector,
|
|
active=1,source='legacy',observed_at=excluded.observed_at""",
|
|
(
|
|
row["ts_code"],
|
|
row["code"],
|
|
row["name"],
|
|
f"{row['code']} {row['ts_code']} {row['name']} {row['industry']}".casefold(),
|
|
row["industry"] or None,
|
|
row["updated_at"] or observed,
|
|
),
|
|
)
|
|
self.counts["market_entities"] = len(stocks)
|
|
dates = [
|
|
row[0]
|
|
for row in source.execute(
|
|
"SELECT DISTINCT trade_date FROM daily_bars ORDER BY trade_date"
|
|
)
|
|
]
|
|
previous = None
|
|
for raw_date in dates:
|
|
trade_date = _iso_date(raw_date)
|
|
target.execute(
|
|
"""INSERT INTO trading_days VALUES (?,1,?,'legacy',?)
|
|
ON CONFLICT(trade_date) DO UPDATE SET is_open=1,
|
|
previous_open_date=excluded.previous_open_date""",
|
|
(trade_date, previous, observed),
|
|
)
|
|
previous = trade_date
|
|
self.counts["trading_days"] = len(dates)
|
|
snapshots: dict[str, tuple[sqlite3.Row, dict[str, Any]]] = {}
|
|
for row in source.execute(
|
|
"SELECT * FROM dashboard_snapshots ORDER BY trade_date,updated_at"
|
|
):
|
|
payload = _json(row["payload"], {})
|
|
meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {}
|
|
trade_date = _iso_date(
|
|
str(payload.get("trade_date") or meta.get("trade_date") or row["trade_date"])
|
|
)
|
|
snapshots[trade_date] = (row, payload)
|
|
history: list[dict[str, Any]] = []
|
|
for trade_date in sorted(snapshots):
|
|
row, raw_payload = snapshots[trade_date]
|
|
payload = _normalize_market_snapshot(raw_payload, trade_date, history)
|
|
history.append(payload)
|
|
events: dict[str, str] = {}
|
|
for event_type, key in (
|
|
("limit_up", "limits"),
|
|
("broken", "broken"),
|
|
("limit_down", "down_limits"),
|
|
):
|
|
for item in payload.get(key) or []:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
identity = str(
|
|
item.get("identifier") or item.get("ts_code") or item.get("code") or ""
|
|
).upper()
|
|
code = identity.split(".")[0]
|
|
if code:
|
|
events[code] = event_type
|
|
self.dashboard_events[trade_date] = events
|
|
target.execute(
|
|
"""INSERT INTO market_summaries VALUES (?,?,'archive','legacy',1,?,?)
|
|
ON CONFLICT(trade_date) DO UPDATE SET observed_at=excluded.observed_at,
|
|
state='archive',source='legacy',coverage=1,payload_json=excluded.payload_json,
|
|
created_at=excluded.created_at""",
|
|
(
|
|
trade_date,
|
|
row["updated_at"],
|
|
_dump(payload),
|
|
row["updated_at"],
|
|
),
|
|
)
|
|
self.counts["market_summaries"] = len(snapshots)
|
|
query = """WITH ranked AS (
|
|
SELECT *,row_number() OVER (PARTITION BY ts_code ORDER BY trade_date DESC) rank
|
|
FROM daily_bars) SELECT * FROM ranked WHERE rank<=90 ORDER BY ts_code,trade_date"""
|
|
current = ""
|
|
points: list[dict[str, Any]] = []
|
|
for row in source.execute(query):
|
|
code = str(row["ts_code"])
|
|
if current and code != current:
|
|
self._save_chart(target, current, points, observed)
|
|
points = []
|
|
current = code
|
|
points.append(
|
|
{
|
|
"time": _iso_date(row["trade_date"]),
|
|
"open": row["open"],
|
|
"high": row["high"],
|
|
"low": row["low"],
|
|
"close": row["close"],
|
|
"volume": float(row["vol"] or 0) * 100,
|
|
"amount": float(row["amount"] or 0) * 1000,
|
|
"average": None,
|
|
}
|
|
)
|
|
if current:
|
|
self._save_chart(target, current, points, observed)
|
|
self._seat_aliases(source, target)
|
|
self._reason_overrides(source, target)
|
|
|
|
def _seat_aliases(
|
|
self, source: sqlite3.Connection, target: sqlite3.Connection
|
|
) -> None:
|
|
if not _table_exists(source, "seat_aliases"):
|
|
return
|
|
for row in source.execute("SELECT * FROM seat_aliases ORDER BY seat_name"):
|
|
target.execute(
|
|
"""INSERT INTO seat_aliases (seat_name,alias_name,updated_at,updated_by)
|
|
VALUES (?,?,?,?) ON CONFLICT(seat_name) DO UPDATE SET
|
|
alias_name=excluded.alias_name,updated_at=excluded.updated_at,
|
|
updated_by=excluded.updated_by""",
|
|
(row["seat_name"], row["alias"], row["updated_at"], self.admin_id),
|
|
)
|
|
self.counts["seat_aliases"] += 1
|
|
|
|
def _reason_overrides(
|
|
self, source: sqlite3.Connection, target: sqlite3.Connection
|
|
) -> None:
|
|
if not _table_exists(source, "reason_overrides"):
|
|
return
|
|
for row in source.execute("SELECT * FROM reason_overrides ORDER BY trade_date,code"):
|
|
trade_date = _iso_date(row["trade_date"])
|
|
code = str(row["code"] or "").split(".")[0]
|
|
event_type = self.dashboard_events.get(trade_date, {}).get(code)
|
|
if not event_type:
|
|
self.counts["reason_overrides_unmatched"] += 1
|
|
continue
|
|
inserted = self._save_revision(
|
|
target,
|
|
trade_date=trade_date,
|
|
identifier=self.stock_ids.get(code, str(row["code"]).upper()),
|
|
event_type=event_type,
|
|
reason=str(row["reason"] or "").strip(),
|
|
first_time="",
|
|
last_time="",
|
|
open_times=None,
|
|
source="admin",
|
|
priority=100,
|
|
created_by=self.admin_id,
|
|
created_at=str(row["updated_at"]),
|
|
)
|
|
self.counts["market_event_revisions"] += int(inserted)
|
|
|
|
def _save_chart(
|
|
self,
|
|
target: sqlite3.Connection,
|
|
identifier: str,
|
|
points: list[dict[str, Any]],
|
|
observed: str,
|
|
) -> None:
|
|
previous = points[-2]["close"] if len(points) > 1 else None
|
|
target.execute(
|
|
"""INSERT INTO chart_series VALUES ('stock',?,'day',?,?,'tushare','display',
|
|
'none',1,?,?) ON CONFLICT(entity_type,identifier,interval,trade_date)
|
|
DO UPDATE SET payload_json=excluded.payload_json,observed_at=excluded.observed_at""",
|
|
(
|
|
identifier,
|
|
points[-1]["time"],
|
|
observed,
|
|
_dump({"previous_close": previous, "points": points}),
|
|
observed,
|
|
),
|
|
)
|
|
self.counts["chart_series"] += 1
|
|
|
|
def _private_data(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
|
stock_ids = {
|
|
row["code"]: row["ts_code"]
|
|
for row in source.execute("SELECT code,ts_code FROM stock_master")
|
|
}
|
|
for row in source.execute("SELECT * FROM watchlist"):
|
|
if int(row["user_id"]) not in self.user_ids:
|
|
continue
|
|
target.execute(
|
|
"""INSERT INTO watchlist_entries VALUES (?,?,?,?,?,?)
|
|
ON CONFLICT(user_id,identifier) DO UPDATE SET name=excluded.name,
|
|
sector=excluded.sector,remark=excluded.remark""",
|
|
(
|
|
row["user_id"],
|
|
stock_ids.get(row["code"], row["code"]),
|
|
row["name"],
|
|
row["sector"] or None,
|
|
row["created_at"],
|
|
row["remark"],
|
|
),
|
|
)
|
|
self.counts["watchlist_entries"] += 1
|
|
self._copy_review_rows(source, target)
|
|
for row in source.execute("SELECT * FROM mentor_preferences"):
|
|
target.execute(
|
|
"INSERT OR REPLACE INTO mentor_preferences VALUES (?,?,?,?,?)",
|
|
(
|
|
row["user_id"],
|
|
row["mentor_id"],
|
|
row["pinned"],
|
|
row["sort_order"],
|
|
row["updated_at"],
|
|
),
|
|
)
|
|
self.counts["mentor_preferences"] += 1
|
|
for row in source.execute("SELECT * FROM mentor_messages"):
|
|
target.execute(
|
|
"""INSERT OR REPLACE INTO mentor_messages
|
|
(id,user_id,mentor_id,trade_date,role,content,request_id,status,created_at)
|
|
VALUES (?,?,?,?,?,?,NULL,'complete',?)""",
|
|
(
|
|
row["id"],
|
|
row["user_id"],
|
|
row["mentor_id"],
|
|
_iso_date(row["trade_date"]),
|
|
row["role"],
|
|
row["content"],
|
|
row["created_at"],
|
|
),
|
|
)
|
|
self.counts["mentor_messages"] += 1
|
|
for row in source.execute("SELECT * FROM heaven_readings"):
|
|
snapshot = _json(row["context_snapshot"], {})
|
|
if row["subject_detail"]:
|
|
snapshot.setdefault("legacy_subject_detail", row["subject_detail"])
|
|
target.execute(
|
|
"""INSERT OR REPLACE INTO heaven_readings
|
|
(id,user_id,mode,reading_date,subject_key,result_json,interpretation,
|
|
interpretation_status,request_id,created_at,updated_at)
|
|
VALUES (?,?,?,?,?,?,?,'complete',NULL,?,?)""",
|
|
(
|
|
row["id"],
|
|
row["user_id"],
|
|
row["mode"],
|
|
_iso_date(row["context_date"]),
|
|
row["subject"],
|
|
_dump(snapshot),
|
|
row["answer"],
|
|
row["created_at"],
|
|
row["created_at"],
|
|
),
|
|
)
|
|
self.counts["heaven_readings"] += 1
|
|
|
|
def _copy_review_rows(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
|
for row in source.execute("SELECT * FROM review_notes WHERE user_id IS NOT NULL"):
|
|
target.execute(
|
|
"""INSERT OR REPLACE INTO review_notes VALUES (?,?,?,?,?,?,?,?,?,?)""",
|
|
(
|
|
row["id"],
|
|
row["user_id"],
|
|
row["code"],
|
|
row["stock_name"],
|
|
_iso_date(row["trade_date"]),
|
|
row["summary"],
|
|
row["content"],
|
|
row["plan"],
|
|
row["created_at"],
|
|
row["updated_at"],
|
|
),
|
|
)
|
|
self.counts["review_notes"] += 1
|
|
actions = {"buy", "sell", "add", "trim", "watch"}
|
|
emotions = {"calm", "confident", "hesitant", "anxious", "impulsive"}
|
|
for row in source.execute("SELECT * FROM trade_entries"):
|
|
target.execute(
|
|
"""INSERT OR REPLACE INTO trade_entries
|
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
|
(
|
|
row["id"],
|
|
row["user_id"],
|
|
_iso_date(row["trade_date"]),
|
|
row["code"],
|
|
row["name"],
|
|
row["action"] if row["action"] in actions else "watch",
|
|
row["price"],
|
|
row["quantity"],
|
|
row["position_pct"],
|
|
row["pnl_amount"],
|
|
row["pnl_pct"],
|
|
row["emotion"] if row["emotion"] in emotions else "calm",
|
|
row["tags"],
|
|
row["thesis"],
|
|
row["execution"],
|
|
row["created_at"],
|
|
row["updated_at"],
|
|
),
|
|
)
|
|
self.counts["trade_entries"] += 1
|
|
for row in source.execute("SELECT * FROM alerts"):
|
|
kind = (
|
|
row["kind"] if row["kind"] in {"manual", "strategy_t1", "strategy_t5"} else "manual"
|
|
)
|
|
target.execute(
|
|
"INSERT OR REPLACE INTO alerts VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
(
|
|
row["id"],
|
|
row["user_id"],
|
|
kind,
|
|
row["title"],
|
|
row["content"],
|
|
_iso_date(row["available_date"]),
|
|
row["code"],
|
|
row["dedupe_key"],
|
|
row["is_read"],
|
|
row["read_at"],
|
|
row["created_at"],
|
|
row["updated_at"],
|
|
),
|
|
)
|
|
self.counts["alerts"] += 1
|
|
for row in source.execute("SELECT * FROM assistant_messages"):
|
|
target.execute(
|
|
"""INSERT OR REPLACE INTO review_assistant_messages
|
|
VALUES (?,?,?,?,?,NULL,'complete',?)""",
|
|
(
|
|
row["id"],
|
|
row["user_id"],
|
|
row["role"],
|
|
row["content"],
|
|
_iso_date(row["context_date"]),
|
|
row["created_at"],
|
|
),
|
|
)
|
|
self.counts["review_assistant_messages"] += 1
|
|
|
|
def _screener(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
|
builtin_by_name = {
|
|
str(item["name"]): item
|
|
for item in strategy_catalog()
|
|
if item.get("kind") in {"stage", "curated"}
|
|
}
|
|
builtin_by_formula = {
|
|
(str(item["kind"]), _formula_fingerprint(item["formula"])): item
|
|
for item in builtin_by_name.values()
|
|
}
|
|
prepared: list[dict[str, Any]] = []
|
|
selected_by_key: dict[tuple[Any, ...], dict[str, Any]] = {}
|
|
for row in source.execute("SELECT * FROM screener_runs ORDER BY id"):
|
|
mode = {"smart": "stage", "curated": "curated", "quant": "custom"}.get(
|
|
str(row["mode"]), "custom"
|
|
)
|
|
formula = _json(row["formula"], {})
|
|
builtin = builtin_by_name.get(str(row["strategy_name"] or "")) or (
|
|
builtin_by_formula.get((mode, _formula_fingerprint(formula)))
|
|
if formula
|
|
else None
|
|
)
|
|
if builtin is not None and builtin["kind"] == mode:
|
|
strategy_id = str(builtin["id"])
|
|
strategy_name = str(builtin.get("display_name") or builtin["name"])
|
|
strategy_version = int(builtin["version"])
|
|
owner_user_id = None
|
|
else:
|
|
strategy_id = str(formula.get("id") or f"legacy-{row['id']}")
|
|
strategy_name = str(row["strategy_name"] or strategy_id)
|
|
raw_version = formula.get("version", 1)
|
|
strategy_version = (
|
|
int(raw_version)
|
|
if isinstance(raw_version, int) and not isinstance(raw_version, bool)
|
|
and raw_version > 0
|
|
else 1
|
|
)
|
|
owner_user_id = row["user_id"]
|
|
item = {
|
|
"row": row,
|
|
"mode": mode,
|
|
"formula": formula,
|
|
"strategy_id": strategy_id,
|
|
"strategy_name": strategy_name,
|
|
"strategy_version": strategy_version,
|
|
"owner_user_id": owner_user_id,
|
|
"trade_date": _iso_date(row["trade_date"]),
|
|
}
|
|
key = (
|
|
owner_user_id,
|
|
mode,
|
|
strategy_id,
|
|
item["trade_date"],
|
|
strategy_version,
|
|
)
|
|
item["key"] = key
|
|
prepared.append(item)
|
|
selected_by_key[key] = item
|
|
|
|
for item in prepared:
|
|
source_id = int(item["row"]["id"])
|
|
self.run_id_map[source_id] = int(selected_by_key[item["key"]]["row"]["id"])
|
|
|
|
snapshots: dict[str, int] = {}
|
|
selected = sorted(selected_by_key.values(), key=lambda item: int(item["row"]["id"]))
|
|
for item in selected:
|
|
row = item["row"]
|
|
trade_date = item["trade_date"]
|
|
if trade_date not in snapshots:
|
|
target.execute(
|
|
"""INSERT OR IGNORE INTO screener_factor_snapshots
|
|
(trade_date,version,observed_at,state,source_set_json,coverage_json,created_at)
|
|
VALUES (?,? ,?,'archive','[\"legacy\"]','{}',?)""",
|
|
(trade_date, ARCHIVE_VERSION, row["created_at"], row["created_at"]),
|
|
)
|
|
snapshots[trade_date] = int(
|
|
target.execute(
|
|
"SELECT id FROM screener_factor_snapshots WHERE trade_date=? AND version=?",
|
|
(trade_date, ARCHIVE_VERSION),
|
|
).fetchone()[0]
|
|
)
|
|
result = _json(row["result"], {})
|
|
candidates = result.get("candidates") if isinstance(result, dict) else []
|
|
candidates = candidates if isinstance(candidates, list) else []
|
|
for candidate in candidates:
|
|
if isinstance(candidate, dict) and not candidate.get("identifier"):
|
|
candidate["identifier"] = (
|
|
candidate.get("ts_code") or candidate.get("code") or ""
|
|
)
|
|
target.execute(
|
|
"""INSERT OR REPLACE INTO screener_runs
|
|
(id,owner_user_id,mode,strategy_id,strategy_name,strategy_version,
|
|
selection_date,factor_snapshot_id,status,started_at,completed_at,coverage,
|
|
missing_fields_json,result_json,error_message)
|
|
VALUES (?,?,?,?,?,?,?,?,?, ?,?,0,'[]',?,'')""",
|
|
(
|
|
row["id"],
|
|
item["owner_user_id"],
|
|
item["mode"],
|
|
item["strategy_id"],
|
|
item["strategy_name"],
|
|
item["strategy_version"],
|
|
trade_date,
|
|
snapshots[trade_date],
|
|
"completed" if candidates else "no_signal",
|
|
row["created_at"],
|
|
row["created_at"],
|
|
_dump(candidates),
|
|
),
|
|
)
|
|
self.run_ids.add(int(row["id"]))
|
|
self.counts["screener_runs"] += 1
|
|
strategy_rows = source.execute("SELECT * FROM screener_strategies").fetchall()
|
|
builtins = sum(row["user_id"] is None for row in strategy_rows)
|
|
if builtins:
|
|
self.row_skips["screener_strategies_builtin"] = {
|
|
"count": builtins,
|
|
"reason": (
|
|
"legacy built-ins are superseded by the single versioned product catalog; "
|
|
"only user-created strategies are migrated"
|
|
),
|
|
}
|
|
for row in (item for item in strategy_rows if item["user_id"] is not None):
|
|
target.execute(
|
|
"""INSERT OR REPLACE INTO custom_screener_strategies
|
|
(id,user_id,name,version,formula_json,created_at,updated_at)
|
|
VALUES (?,?,?,1,?,?,?)""",
|
|
(
|
|
row["id"],
|
|
row["user_id"],
|
|
row["name"],
|
|
row["formula"],
|
|
row["created_at"],
|
|
row["updated_at"],
|
|
),
|
|
)
|
|
self.counts["custom_screener_strategies"] += 1
|
|
for row in source.execute("SELECT * FROM strategy_tracks"):
|
|
mapped_run_id = self.run_id_map.get(int(row["run_id"]))
|
|
if mapped_run_id not in self.run_ids:
|
|
continue
|
|
target.execute(
|
|
"""INSERT OR REPLACE INTO strategy_tracks
|
|
(id,user_id,run_id,identifier,code,name,sector,selection_date,
|
|
strategy_name,entry_price,added_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
|
|
(
|
|
row["id"],
|
|
row["user_id"],
|
|
mapped_run_id,
|
|
row["ts_code"],
|
|
row["code"],
|
|
row["name"],
|
|
row["sector"],
|
|
_iso_date(row["selection_date"]),
|
|
row["strategy_name"],
|
|
row["entry_price"],
|
|
row["created_at"],
|
|
),
|
|
)
|
|
self.counts["strategy_tracks"] += 1
|
|
|
|
def _save_revision(
|
|
self,
|
|
target: sqlite3.Connection,
|
|
*,
|
|
trade_date: str,
|
|
identifier: str,
|
|
event_type: str,
|
|
reason: str,
|
|
first_time: str,
|
|
last_time: str,
|
|
open_times: int | None,
|
|
source: str,
|
|
priority: int,
|
|
created_by: int | None,
|
|
created_at: str,
|
|
) -> bool:
|
|
exists = target.execute(
|
|
"""SELECT 1 FROM market_event_revisions
|
|
WHERE trade_date=? AND identifier=? AND event_type=? AND reason=?
|
|
AND first_time=? AND last_time=? AND open_times IS ? AND source=?
|
|
AND priority=? AND created_by IS ? AND created_at=?""",
|
|
(
|
|
trade_date,
|
|
identifier,
|
|
event_type,
|
|
reason,
|
|
first_time,
|
|
last_time,
|
|
open_times,
|
|
source,
|
|
priority,
|
|
created_by,
|
|
created_at,
|
|
),
|
|
).fetchone()
|
|
if exists:
|
|
return False
|
|
target.execute(
|
|
"""INSERT INTO market_event_revisions
|
|
(trade_date,identifier,event_type,reason,first_time,last_time,open_times,
|
|
source,priority,created_by,created_at) VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
|
|
(
|
|
trade_date,
|
|
identifier,
|
|
event_type,
|
|
reason,
|
|
first_time,
|
|
last_time,
|
|
open_times,
|
|
source,
|
|
priority,
|
|
created_by,
|
|
created_at,
|
|
),
|
|
)
|
|
return True
|
|
|
|
def _insights(self, source: sqlite3.Connection, target: sqlite3.Connection) -> None:
|
|
self._auction_snapshots(source, target)
|
|
self._theme_snapshots(source, target)
|
|
self._standard_insight_snapshots(source, target)
|
|
self._dragon_snapshots(source, target)
|
|
self._sector_member_snapshots(source, target)
|
|
self._ifind_event_revisions(source, target)
|
|
|
|
def _auction_snapshots(
|
|
self, source: sqlite3.Connection, target: sqlite3.Connection
|
|
) -> None:
|
|
selected: dict[str, tuple[int, sqlite3.Row]] = {}
|
|
for row in source.execute(
|
|
"SELECT * FROM data_snapshots WHERE kind LIKE 'auction_center_v%'"
|
|
):
|
|
trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0])
|
|
try:
|
|
version = int(str(row["kind"]).rsplit("v", 1)[1])
|
|
except ValueError:
|
|
continue
|
|
current = selected.get(trade_date)
|
|
if trade_date and (current is None or version > current[0]):
|
|
selected[trade_date] = (version, row)
|
|
for trade_date, (_, row) in sorted(selected.items()):
|
|
payload = _normalize_identifiers(_json(row["payload"], {}))
|
|
payload.update(
|
|
{
|
|
"trade_date": trade_date,
|
|
"observed_at": _observed_at(payload, str(row["updated_at"])),
|
|
"state": "archive",
|
|
"message": str(payload.get("message") or ""),
|
|
}
|
|
)
|
|
payload.setdefault("coverage", 1)
|
|
payload.setdefault("dynamic", False)
|
|
payload.setdefault("_market_rows", list(payload.get("rows") or []))
|
|
self._save_insight(target, "auction", trade_date, "", payload, 1)
|
|
|
|
def _theme_snapshots(
|
|
self, source: sqlite3.Connection, target: sqlite3.Connection
|
|
) -> None:
|
|
directory_rows = source.execute(
|
|
"SELECT * FROM data_snapshots WHERE kind='theme_directory_v1'"
|
|
).fetchall()
|
|
for row in directory_rows:
|
|
payload = _json(row["payload"], {})
|
|
self._save_theme_entities(
|
|
target, list(payload.get("items") or []), str(row["updated_at"])
|
|
)
|
|
for row in source.execute(
|
|
"SELECT * FROM data_snapshots WHERE kind='theme_library_v1' ORDER BY cache_key"
|
|
):
|
|
trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0])
|
|
payload = _normalize_identifiers(_json(row["payload"], {}))
|
|
payload.update(
|
|
{
|
|
"trade_date": trade_date,
|
|
"observed_at": _observed_at(payload, str(row["updated_at"])),
|
|
"state": "archive",
|
|
"message": str(payload.get("message") or ""),
|
|
}
|
|
)
|
|
self._save_theme_entities(
|
|
target, list(payload.get("items") or []), str(row["updated_at"])
|
|
)
|
|
self._save_insight(target, "themes", trade_date, "", payload, 1)
|
|
for row in source.execute(
|
|
"SELECT * FROM data_snapshots WHERE kind='theme_detail_v1' ORDER BY cache_key"
|
|
):
|
|
cache_key = str(row["cache_key"])
|
|
raw_date, _, identifier = cache_key.partition(":")
|
|
trade_date = _iso_date(raw_date)
|
|
legacy = _normalize_identifiers(_json(row["payload"], {}))
|
|
theme = dict(legacy.get("theme") or {})
|
|
identifier = (identifier or str(theme.get("code") or "")).upper()
|
|
members = []
|
|
for item in legacy.get("members") or []:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
member_id = str(item.get("identifier") or item.get("ts_code") or "").upper()
|
|
members.append(
|
|
{
|
|
"identifier": member_id,
|
|
"code": str(item.get("code") or member_id.split(".")[0]),
|
|
"name": str(item.get("name") or ""),
|
|
"change": item.get("change"),
|
|
"close": item.get("close", item.get("price")),
|
|
"amount": (
|
|
float(item["amount_billion"]) * 100_000_000
|
|
if item.get("amount_billion") not in (None, "")
|
|
else item.get("amount")
|
|
),
|
|
"quoted": bool(item.get("quoted", item.get("has_quote"))),
|
|
}
|
|
)
|
|
summary = dict(legacy.get("summary") or {})
|
|
detail = {
|
|
"trade_date": trade_date,
|
|
"theme": theme,
|
|
"summary": summary,
|
|
"members": members,
|
|
"message": "" if members else "该题材暂无可核验成分股",
|
|
"observed_at": _observed_at(legacy, str(row["updated_at"])),
|
|
"state": "archive",
|
|
}
|
|
self._save_insight(target, "themes", trade_date, identifier, detail, 1)
|
|
self._save_theme_chart(
|
|
target,
|
|
identifier,
|
|
list(legacy.get("series") or []),
|
|
detail["observed_at"],
|
|
)
|
|
|
|
def _save_theme_entities(
|
|
self, target: sqlite3.Connection, rows: list[dict[str, Any]], observed_at: str
|
|
) -> None:
|
|
for item in rows:
|
|
identifier = str(item.get("code") or item.get("ts_code") or "").upper()
|
|
name = str(item.get("name") or "").strip()
|
|
if not identifier or not name:
|
|
continue
|
|
target.execute(
|
|
"""INSERT INTO market_entities
|
|
(entity_type,identifier,code,name,search_key,sector,active,source,observed_at)
|
|
VALUES ('theme',?,?,?,?,NULL,1,'legacy',?)
|
|
ON CONFLICT(entity_type,identifier) DO UPDATE SET
|
|
code=excluded.code,name=excluded.name,search_key=excluded.search_key,
|
|
active=1,source='legacy',observed_at=excluded.observed_at""",
|
|
(
|
|
identifier,
|
|
identifier.split(".")[0],
|
|
name,
|
|
f"{identifier} {name}".casefold(),
|
|
observed_at,
|
|
),
|
|
)
|
|
|
|
def _save_theme_chart(
|
|
self,
|
|
target: sqlite3.Connection,
|
|
identifier: str,
|
|
rows: list[dict[str, Any]],
|
|
observed_at: str,
|
|
) -> None:
|
|
points = [
|
|
{
|
|
"time": _iso_date(item.get("trade_date")),
|
|
"open": item.get("open"),
|
|
"high": item.get("high"),
|
|
"low": item.get("low"),
|
|
"close": item.get("close"),
|
|
"volume": item.get("volume"),
|
|
"amount": item.get("amount"),
|
|
"average": None,
|
|
}
|
|
for item in rows
|
|
if _iso_date(item.get("trade_date")) and item.get("close") is not None
|
|
]
|
|
if not identifier or not points:
|
|
return
|
|
target.execute(
|
|
"""INSERT INTO chart_series
|
|
(entity_type,identifier,interval,trade_date,observed_at,source,usage,
|
|
adjustment,coverage,payload_json,created_at)
|
|
VALUES ('theme',?,'day',?,?,'legacy','display','none',1,?,?)
|
|
ON CONFLICT(entity_type,identifier,interval,trade_date) DO UPDATE SET
|
|
observed_at=excluded.observed_at,payload_json=excluded.payload_json""",
|
|
(
|
|
identifier,
|
|
points[-1]["time"],
|
|
observed_at,
|
|
_dump(
|
|
{
|
|
"previous_close": points[-2]["close"] if len(points) > 1 else None,
|
|
"points": points,
|
|
}
|
|
),
|
|
observed_at,
|
|
),
|
|
)
|
|
self.counts["chart_series"] += 1
|
|
|
|
def _standard_insight_snapshots(
|
|
self, source: sqlite3.Connection, target: sqlite3.Connection
|
|
) -> None:
|
|
for row in source.execute(
|
|
"SELECT * FROM data_snapshots WHERE kind='popularity_v1' ORDER BY cache_key"
|
|
):
|
|
trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0])
|
|
payload = _normalize_identifiers(_json(row["payload"], {}))
|
|
payload.update(
|
|
{
|
|
"trade_date": trade_date,
|
|
"observed_at": _observed_at(payload, str(row["updated_at"])),
|
|
"state": "archive",
|
|
"message": str(payload.get("message") or ""),
|
|
}
|
|
)
|
|
self._save_insight(target, "popularity", trade_date, "", payload, 1)
|
|
|
|
def _dragon_snapshots(
|
|
self, source: sqlite3.Connection, target: sqlite3.Connection
|
|
) -> None:
|
|
profile_row = source.execute(
|
|
"""SELECT * FROM data_snapshots WHERE kind='hot_money_profiles_v1'
|
|
ORDER BY updated_at DESC LIMIT 1"""
|
|
).fetchone()
|
|
profiles = []
|
|
if profile_row:
|
|
for item in _json(profile_row["payload"], {}).get("profiles") or []:
|
|
profiles.append(
|
|
{
|
|
"name": str(item.get("name") or ""),
|
|
"desc": str(item.get("description") or item.get("desc") or ""),
|
|
"orgs": _dump(item.get("organizations") or item.get("orgs") or []),
|
|
}
|
|
)
|
|
stocks_by_date: dict[str, list[dict[str, Any]]] = {}
|
|
seats_by_date: dict[str, list[dict[str, Any]]] = {}
|
|
observed_by_date: dict[str, str] = {}
|
|
for row in source.execute(
|
|
"SELECT * FROM data_snapshots WHERE kind='dragon_tiger' ORDER BY cache_key"
|
|
):
|
|
trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0])
|
|
payload = _json(row["payload"], {})
|
|
observed_by_date[trade_date] = _observed_at(payload, str(row["updated_at"]))
|
|
stocks = []
|
|
seats = []
|
|
for item in payload.get("rows") or []:
|
|
identifier = str(item.get("ts_code") or item.get("identifier") or "").upper()
|
|
stocks.append(
|
|
{
|
|
"ts_code": identifier,
|
|
"name": str(item.get("name") or ""),
|
|
"pct_change": item.get("change"),
|
|
"reason": str(item.get("reason") or ""),
|
|
}
|
|
)
|
|
for seat in item.get("institutions") or []:
|
|
seats.append(
|
|
{
|
|
"ts_code": identifier,
|
|
"exalter": str(seat.get("seat_name") or ""),
|
|
"buy": float(seat.get("buy_million") or 0) * 1_000_000,
|
|
"sell": float(seat.get("sell_million") or 0) * 1_000_000,
|
|
"net_buy": float(seat.get("net_buy_million") or 0) * 1_000_000,
|
|
"reason": str(item.get("reason") or ""),
|
|
}
|
|
)
|
|
stocks_by_date[trade_date] = stocks
|
|
seats_by_date[trade_date] = seats
|
|
details: dict[str, tuple[int, sqlite3.Row]] = {}
|
|
for row in source.execute(
|
|
"""SELECT * FROM data_snapshots
|
|
WHERE kind IN ('hot_money_detail_v2','hot_money_detail_v3')"""
|
|
):
|
|
trade_date = _iso_date(str(row["cache_key"]).split(":", 1)[0])
|
|
version = int(str(row["kind"]).rsplit("v", 1)[1])
|
|
if trade_date not in details or version > details[trade_date][0]:
|
|
details[trade_date] = (version, row)
|
|
official_by_date: dict[str, list[dict[str, Any]]] = {}
|
|
for trade_date, (_, row) in details.items():
|
|
payload = _json(row["payload"], {})
|
|
observed_by_date.setdefault(
|
|
trade_date, _observed_at(payload, str(row["updated_at"]))
|
|
)
|
|
official = []
|
|
derived_stocks: dict[str, dict[str, Any]] = {}
|
|
for trader in payload.get("traders") or []:
|
|
trader_name = str(trader.get("name") or "")
|
|
for operation in trader.get("operations") or []:
|
|
identifier = str(
|
|
operation.get("ts_code") or operation.get("identifier") or ""
|
|
).upper()
|
|
official.append(
|
|
{
|
|
"ts_code": identifier,
|
|
"ts_name": str(operation.get("name") or ""),
|
|
"hm_name": trader_name,
|
|
"hm_orgs": str(operation.get("seat_name") or ""),
|
|
"buy_amount": float(operation.get("buy_million") or 0) * 1_000_000,
|
|
"sell_amount": float(operation.get("sell_million") or 0) * 1_000_000,
|
|
"net_amount": float(operation.get("net_buy_million") or 0)
|
|
* 1_000_000,
|
|
}
|
|
)
|
|
if identifier:
|
|
derived_stocks[identifier] = {
|
|
"ts_code": identifier,
|
|
"name": str(operation.get("name") or ""),
|
|
"pct_change": operation.get("change"),
|
|
"reason": str(operation.get("reason") or ""),
|
|
}
|
|
official_by_date[trade_date] = official
|
|
stocks_by_date.setdefault(trade_date, list(derived_stocks.values()))
|
|
seats_by_date.setdefault(trade_date, [])
|
|
all_dates = sorted(set(stocks_by_date) | set(official_by_date))
|
|
for trade_date in all_dates:
|
|
raw = {
|
|
"trade_date": trade_date,
|
|
"observed_at": observed_by_date.get(
|
|
trade_date, datetime.now().astimezone().isoformat(timespec="seconds")
|
|
),
|
|
"state": "archive",
|
|
"official": official_by_date.get(trade_date, []),
|
|
"profiles": profiles,
|
|
"stocks": stocks_by_date.get(trade_date, []),
|
|
"seats": seats_by_date.get(trade_date, []),
|
|
}
|
|
self._save_insight(target, "dragon-list", trade_date, "", raw, 1)
|
|
|
|
def _sector_member_snapshots(
|
|
self, source: sqlite3.Connection, target: sqlite3.Connection
|
|
) -> None:
|
|
for row in source.execute(
|
|
"""SELECT * FROM data_snapshots
|
|
WHERE kind='rotation_sector_members_v1' ORDER BY cache_key"""
|
|
):
|
|
raw_date, _, sector_name = str(row["cache_key"]).partition(":")
|
|
trade_date = _iso_date(raw_date)
|
|
payload = _normalize_identifiers(_json(row["payload"], {}))
|
|
meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {}
|
|
payload.update(
|
|
{
|
|
"trade_date": trade_date,
|
|
"sector_name": sector_name,
|
|
"observed_at": _observed_at(payload, str(row["updated_at"])),
|
|
"source": "legacy",
|
|
"coverage": 1,
|
|
}
|
|
)
|
|
target.execute(
|
|
"""INSERT INTO sector_member_snapshots
|
|
(trade_date,sector_name,sector_code,observed_at,source,coverage,payload_json)
|
|
VALUES (?,?,?,?, 'legacy',1,?)
|
|
ON CONFLICT(trade_date,sector_name) DO UPDATE SET
|
|
sector_code=excluded.sector_code,observed_at=excluded.observed_at,
|
|
source=excluded.source,coverage=excluded.coverage,payload_json=excluded.payload_json""",
|
|
(
|
|
trade_date,
|
|
sector_name,
|
|
str(meta.get("sector_code") or meta.get("representative") or ""),
|
|
payload["observed_at"],
|
|
_dump(payload),
|
|
),
|
|
)
|
|
self.counts["sector_member_snapshots"] += 1
|
|
|
|
def _ifind_event_revisions(
|
|
self, source: sqlite3.Connection, target: sqlite3.Connection
|
|
) -> None:
|
|
for row in source.execute(
|
|
"""SELECT * FROM data_snapshots
|
|
WHERE kind='ifind_event_enrichment_v1' ORDER BY cache_key"""
|
|
):
|
|
payload = _json(row["payload"], {})
|
|
trade_date = _iso_date(payload.get("trade_date") or row["cache_key"])
|
|
created_at = str(payload.get("generated_at") or row["updated_at"])
|
|
for event_type, key in (
|
|
("limit_up", "limits"),
|
|
("broken", "broken"),
|
|
("limit_down", "down_limits"),
|
|
):
|
|
values = payload.get(key) or {}
|
|
if not isinstance(values, dict):
|
|
continue
|
|
for raw_identifier, detail in values.items():
|
|
if not isinstance(detail, dict):
|
|
continue
|
|
useful = any(
|
|
detail.get(field) not in (None, "")
|
|
for field in ("reason", "first_time", "last_time", "open_times")
|
|
)
|
|
if not useful:
|
|
continue
|
|
code = str(raw_identifier).split(".")[0]
|
|
inserted = self._save_revision(
|
|
target,
|
|
trade_date=trade_date,
|
|
identifier=self.stock_ids.get(code, str(raw_identifier).upper()),
|
|
event_type=event_type,
|
|
reason=str(detail.get("reason") or "").strip(),
|
|
first_time=_time(detail.get("first_time")),
|
|
last_time=_time(detail.get("last_time")),
|
|
open_times=(
|
|
int(detail["open_times"])
|
|
if detail.get("open_times") not in (None, "")
|
|
else None
|
|
),
|
|
source="ifind",
|
|
priority=20,
|
|
created_by=None,
|
|
created_at=created_at,
|
|
)
|
|
self.counts["market_event_revisions"] += int(inserted)
|
|
|
|
def _save_insight(
|
|
self,
|
|
target: sqlite3.Connection,
|
|
kind: str,
|
|
trade_date: str,
|
|
entity_key: str,
|
|
payload: dict[str, Any],
|
|
coverage: float,
|
|
) -> None:
|
|
target.execute(
|
|
"""INSERT INTO market_insight_snapshots
|
|
(kind,trade_date,entity_key,observed_at,state,source,coverage,payload_json)
|
|
VALUES (?,?,?,?,'archive','legacy',?,?)
|
|
ON CONFLICT(kind,trade_date,entity_key) DO UPDATE SET
|
|
observed_at=excluded.observed_at,state=excluded.state,source=excluded.source,
|
|
coverage=excluded.coverage,payload_json=excluded.payload_json""",
|
|
(
|
|
kind,
|
|
trade_date,
|
|
entity_key,
|
|
_observed_at(payload, datetime.now().astimezone().isoformat(timespec="seconds")),
|
|
max(0, min(float(coverage), 1)),
|
|
_dump(payload),
|
|
),
|
|
)
|
|
self.counts["market_insight_snapshots"] += 1
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(description="Migrate a read-only legacy database copy")
|
|
parser.add_argument("--source", type=Path, required=True)
|
|
parser.add_argument("--target", type=Path, required=True)
|
|
parser.add_argument("--report", type=Path)
|
|
return parser
|
|
|
|
|
|
def main(arguments: Sequence[str] | None = None) -> int:
|
|
parsed = build_parser().parse_args(arguments)
|
|
report = LegacyMigrator(parsed.source, parsed.target, os.getenv("APP_ENCRYPTION_KEY")).run()
|
|
rendered = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if parsed.report:
|
|
parsed.report.parent.mkdir(parents=True, exist_ok=True)
|
|
parsed.report.write_text(rendered + "\n", encoding="utf-8")
|
|
print(rendered)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|