leefer/xiaobaifupan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rebuild(migration): audit all legacy data

Browse Source
This commit is contained in:
leefer
2026-07-30 11:42:33 +08:00
parent 227028f499
commit 9e694ee34a
5 changed files with 1120 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
next/tests/test_legacy_migration.py
+214 -1
View File
@@ -9,6 +9,7 @@ import sqlite3
from cryptography.fernet import Fernet
from backend.features.market.events import apply_event_revisions
from backend.security.passwords import PasswordHasher
from tools.legacy_migration import LegacyMigrator
@@ -81,6 +82,9 @@ def _legacy_database(path, key: str) -> None:
entry_price REAL,created_at TEXT,updated_at TEXT);
CREATE TABLE data_snapshots (kind TEXT,cache_key TEXT,source TEXT,payload TEXT,
updated_at TEXT,PRIMARY KEY(kind,cache_key));
CREATE TABLE reason_overrides (trade_date TEXT,code TEXT,reason TEXT,
updated_at TEXT,PRIMARY KEY(trade_date,code));
CREATE TABLE seat_aliases (seat_name TEXT PRIMARY KEY,alias TEXT,updated_at TEXT);
"""
)
now = "2026-07-29T16:00:00+08:00"
@@ -103,8 +107,23 @@ def _legacy_database(path, key: str) -> None:
connection.execute(
"INSERT INTO daily_bars VALUES ('20260729','000001.SZ',10,11,9,10.5,5,100,1000)"
)
dashboard = json.dumps(
{
"limits": [
{
"ts_code": "000001.SZ",
"code": "000001",
"name": "Ping An Bank",
"reason": "legacy",
}
],
"broken": [],
"down_limits": [],
}
)
connection.execute(
"INSERT INTO dashboard_snapshots VALUES ('20260729','tushare','{}',1,?)", (now,)
"INSERT INTO dashboard_snapshots VALUES ('20260729','tushare',?,1,?)",
(dashboard, now),
)
connection.execute(
"INSERT INTO watchlist VALUES (8,'000001','平安银行','银行','red',?,?, '长期')",
@@ -139,6 +158,10 @@ def _legacy_database(path, key: str) -> None:
"INSERT INTO screener_strategies VALUES (1,'自定义','','[]','{}',0,?,?,8)",
(now, now),
)
connection.execute(
"INSERT INTO screener_strategies VALUES (2,'legacy builtin','','[]','{}',1,?,?,NULL)",
(now, now),
)
connection.execute(
"INSERT INTO strategy_tracks VALUES (1,8,1,'20260729','策略','000001.SZ','000001','平安银行','银行',10.5,?,?)",
(now, now),
@@ -146,6 +169,159 @@ def _legacy_database(path, key: str) -> None:
connection.execute(
"INSERT INTO data_snapshots VALUES ('popularity_v1','20260729','legacy','{}',?)", (now,)
)
connection.execute(
"INSERT INTO reason_overrides VALUES ('20260729','000001','admin reason',?)",
(now,),
)
connection.execute(
"INSERT INTO seat_aliases VALUES ('seat-a','trader-a',?)", (now,)
)
snapshots = {
"ifind_event_enrichment_v1": (
"20260729",
{
"trade_date": "20260729",
"generated_at": now,
"limits": {
"000001": {
"reason": "ifind reason",
"first_time": "09:31:03",
"last_time": "14:52:01",
"open_times": 1,
}
},
"broken": {},
"down_limits": {},
},
),
"rotation_sector_members_v1": (
"20260729:Bank",
{
"meta": {"sector_code": "801780.SI"},
"rows": [
{
"ts_code": "000001.SZ",
"code": "000001",
"name": "Ping An Bank",
"quoted": True,
}
],
},
),
"theme_directory_v1": (
"ths",
{"items": [{"ts_code": "885001.TI", "name": "Theme A"}]},
),
"theme_library_v1": (
"20260729",
{
"meta": {"updated_at": now},
"summary": {"theme_count": 1},
"items": [{"code": "885001.TI", "name": "Theme A"}],
},
),
"theme_detail_v1": (
"20260729:885001.TI",
{
"meta": {"updated_at": now},
"theme": {"code": "885001.TI", "name": "Theme A"},
"series": [
{
"trade_date": "20260728",
"open": 10,
"high": 11,
"low": 9,
"close": 10,
"volume": 100,
},
{
"trade_date": "20260729",
"open": 10,
"high": 12,
"low": 10,
"close": 11,
"volume": 120,
},
],
"members": [
{
"ts_code": "000001.SZ",
"code": "000001",
"name": "Ping An Bank",
"price": 10.5,
"change": 5,
"amount_billion": 1.2,
"has_quote": True,
}
],
"summary": {"member_count": 1},
},
),
"hot_money_profiles_v1": (
"directory",
{
"profiles": [
{
"name": "trader-a",
"description": "profile",
"organizations": ["seat-a"],
}
]
},
),
"dragon_tiger": (
"20260729",
{
"meta": {"updated_at": now},
"rows": [
{
"ts_code": "000001.SZ",
"name": "Ping An Bank",
"change": 5,
"reason": "listed",
"institutions": [
{
"seat_name": "seat-a",
"buy_million": 2,
"sell_million": 1,
"net_buy_million": 1,
}
],
}
],
},
),
"hot_money_detail_v3": (
"20260729",
{
"meta": {"updated_at": now},
"traders": [
{
"name": "trader-a",
"operations": [
{
"ts_code": "000001.SZ",
"name": "Ping An Bank",
"change": 5,
"seat_name": "seat-a",
"buy_million": 2,
"sell_million": 1,
"net_buy_million": 1,
"reason": "listed",
}
],
}
],
},
),
}
connection.executemany(
"INSERT INTO data_snapshots VALUES (?,?, 'legacy',?,?)",
[
(kind, cache_key, json.dumps(payload), now)
for kind, (cache_key, payload) in snapshots.items()
],
)
def test_legacy_migration_is_idempotent_and_preserves_login(tmp_path) -> None:
@@ -158,13 +334,50 @@ def test_legacy_migration_is_idempotent_and_preserves_login(tmp_path) -> None:
second = LegacyMigrator(source, target, key).run()
assert first["integrity"] == second["integrity"] == "ok"
assert first["unmapped_tables"] == []
assert first["unmapped_snapshot_kinds"] == []
assert first["source_tables"]["reason_overrides"] == 1
assert first["intentionally_skipped_rows"]["screener_strategies_builtin"]["count"] == 1
assert "market_event_revisions" in first["target_counts"]
rendered = json.dumps(first)
assert "secret-tushare" not in rendered
assert "secret-model" not in rendered
with sqlite3.connect(target) as connection:
connection.row_factory = sqlite3.Row
password = connection.execute("SELECT password_hash FROM users WHERE id=8").fetchone()[0]
assert PasswordHasher().verify("Password123", password)
assert connection.execute("SELECT count(*) FROM watchlist_entries").fetchone()[0] == 1
assert connection.execute("SELECT count(*) FROM screener_runs").fetchone()[0] == 1
assert connection.execute("SELECT count(*) FROM strategy_tracks").fetchone()[0] == 1
assert connection.execute("SELECT daily_llm_limit FROM memberships").fetchone()[0] == 61
assert connection.execute("SELECT count(*) FROM seat_aliases").fetchone()[0] == 1
assert connection.execute(
"SELECT count(*) FROM sector_member_snapshots"
).fetchone()[0] == 1
assert connection.execute(
"SELECT count(*) FROM chart_series WHERE entity_type='theme'"
).fetchone()[0] == 1
summary = json.loads(
connection.execute(
"SELECT payload_json FROM market_summaries WHERE trade_date='2026-07-29'"
).fetchone()[0]
)
assert summary["limits"][0]["identifier"] == "000001.SZ"
all_revisions = tuple(
connection.execute(
"""SELECT * FROM market_event_revisions WHERE trade_date='2026-07-29'
ORDER BY priority DESC,id DESC"""
).fetchall()
)
assert len(all_revisions) == 2
revisions = all_revisions[:1]
revised = apply_event_revisions(summary, revisions)
assert revised["limits"][0]["reason"] == "admin reason"
raw_dragon = json.loads(
connection.execute(
"""SELECT payload_json FROM market_insight_snapshots
WHERE kind='dragon-list' AND trade_date='2026-07-29'"""
).fetchone()[0]
)
assert raw_dragon["profiles"][0]["desc"] == "profile"
assert raw_dragon["official"][0]["hm_name"] == "trader-a"