From 198806c1bdf964e0a3b589518a38ac9e1c9b0b3d Mon Sep 17 00:00:00 2001 From: leefer Date: Thu, 30 Jul 2026 17:26:15 +0800 Subject: [PATCH] rebuild(migration): validate local legacy cutover --- next/backend/data/sentiment.py | 14 +- next/backend/features/accounts/service.py | 12 +- next/backend/features/market/sync.py | 2 - next/backend/jobs/repository.py | 21 +++ next/backend/jobs/service.py | 9 +- .../evidence/local-preflight-2026-07-30.md | 50 ++++++ next/docs/final/completion-audit.md | 20 ++- next/docs/final/page-state-matrix.md | 2 + next/frontend/src/shared/styles/mobile.css | 4 + next/tests/e2e/operations.spec.js | 1 + next/tests/e2e/stage10.spec.js | 1 + next/tests/e2e/stage11.spec.js | 1 + next/tests/e2e/stage12.spec.js | 1 + next/tests/e2e/stage13.spec.js | 1 + next/tests/e2e/stage5.spec.js | 1 + next/tests/e2e/stage6.spec.js | 1 + next/tests/e2e/stage7.spec.js | 1 + next/tests/e2e/stage8.spec.js | 1 + next/tests/test_accounts.py | 19 +++ next/tests/test_legacy_migration.py | 51 +++++- next/tests/test_market_data.py | 51 ++++++ next/tests/test_operations.py | 31 ++++ next/tools/legacy_migration.py | 150 +++++++++++++++++- 23 files changed, 418 insertions(+), 27 deletions(-) create mode 100644 next/docs/evidence/local-preflight-2026-07-30.md diff --git a/next/backend/data/sentiment.py b/next/backend/data/sentiment.py index 76faf4c..afe1fbc 100644 --- a/next/backend/data/sentiment.py +++ b/next/backend/data/sentiment.py @@ -128,7 +128,7 @@ def _stats(snapshot: dict[str, Any]) -> dict[str, float]: max_height = max(streaks, default=0) active = _number(overview.get("up_count")) + _number(overview.get("down_count")) changes = [_number(row.get("current_change")) for row in yesterday] - previous_count = len(yesterday) + yesterday_count = len(yesterday) return { "breadth_ratio": _number(overview.get("up_count")) / max(active, 1) * 100, "limit_up": _number(overview.get("limit_up")), @@ -145,22 +145,22 @@ def _stats(snapshot: dict[str, Any]) -> dict[str, float]: if max_height else 0 ), - "previous_count": previous_count, - "positive_rate": sum(change > 0 for change in changes) / max(previous_count, 1) * 100, + "yesterday_count": yesterday_count, + "positive_rate": sum(change > 0 for change in changes) / max(yesterday_count, 1) * 100, "advance_rate": sum(row.get("outcome") == "晋级" for row in yesterday) - / max(previous_count, 1) + / max(yesterday_count, 1) * 100, "average_change": mean(changes) if changes else 0, "median_change": median(changes) if changes else 0, - "severe_loss_rate": sum(change <= -5 for change in changes) / max(previous_count, 1) * 100, + "severe_loss_rate": sum(change <= -5 for change in changes) / max(yesterday_count, 1) * 100, "previous_down_rate": sum(row.get("outcome") == "跌停" for row in yesterday) - / max(previous_count, 1) + / max(yesterday_count, 1) * 100, } def _profit(stats: dict[str, float]) -> float: - if not stats["previous_count"]: + if not stats["yesterday_count"]: return 50 median_score = _clamp(50 + stats["median_change"] * 7) average_score = _clamp(50 + stats["average_change"] * 6) diff --git a/next/backend/features/accounts/service.py b/next/backend/features/accounts/service.py index 0d2b46b..d067ac1 100644 --- a/next/backend/features/accounts/service.py +++ b/next/backend/features/accounts/service.py @@ -210,16 +210,16 @@ class AccountService: return None try: payload = json.loads(self._cipher.decrypt(str(row["encrypted_payload"]))) + return BirthProfile( + birth_date=str(payload["birth_date"]), + birth_time=str(payload["birth_time"]), + gender=str(payload["gender"]), + updated_at=datetime.fromisoformat(str(row["updated_at"])), + ) except (ValueError, json.JSONDecodeError, KeyError) as exc: raise BusinessError( "profile_unavailable", "个人资料暂时无法读取，请联系管理员。" ) from exc - return BirthProfile( - birth_date=str(payload["birth_date"]), - birth_time=str(payload["birth_time"]), - gender=str(payload["gender"]), - updated_at=datetime.fromisoformat(str(row["updated_at"])), - ) def save_profile( self, user_id: int, birth_date: date, birth_time: str, gender: str diff --git a/next/backend/features/market/sync.py b/next/backend/features/market/sync.py index 8132604..a282984 100644 --- a/next/backend/features/market/sync.py +++ b/next/backend/features/market/sync.py @@ -68,7 +68,6 @@ class MarketSnapshotService: history = [json.loads(str(row["payload_json"])) for row in rows] sentiment = calculate_sentiment(snapshot, history) snapshot["sentiment"] = sentiment - snapshot.update(snapshot["overview"]) snapshot["temperature"] = sentiment["score"] observed_at = datetime.combine( date.fromisoformat(trade_date), time(15), tzinfo=SHANGHAI @@ -144,7 +143,6 @@ class MarketSnapshotService: history = [json.loads(str(row["payload_json"])) for row in rows] sentiment = calculate_sentiment(snapshot, history) snapshot["sentiment"] = sentiment - snapshot.update(snapshot["overview"]) snapshot["temperature"] = sentiment["score"] observed_at = clock.isoformat(timespec="seconds") with self._database.transaction() as connection: diff --git a/next/backend/jobs/repository.py b/next/backend/jobs/repository.py index e8a8d74..f4640c2 100644 --- a/next/backend/jobs/repository.py +++ b/next/backend/jobs/repository.py @@ -1,5 +1,6 @@ from __future__ import annotations +import ast import json import sqlite3 from datetime import datetime, timedelta @@ -147,8 +148,28 @@ def public_job(row: sqlite3.Row) -> dict[str, Any]: result = dict(row) result["source_set"] = json.loads(str(result.pop("source_set_json"))) result["payload"] = json.loads(str(result.pop("payload_json"))) + result["error_message"] = _public_error_message(str(result["error_message"])) return result +def _public_error_message(value: str) -> str: + """Normalize AppError tuples written before structured error storage.""" + if not value.startswith("("): + return value + try: + legacy = ast.literal_eval(value) + except (SyntaxError, ValueError): + return value + if ( + isinstance(legacy, tuple) + and len(legacy) == 3 + and isinstance(legacy[0], str) + and isinstance(legacy[1], str) + and isinstance(legacy[2], int) + ): + return legacy[1] + return value + + def _json(value: Any) -> str: return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) diff --git a/next/backend/jobs/service.py b/next/backend/jobs/service.py index 295d5fb..6c8a285 100644 --- a/next/backend/jobs/service.py +++ b/next/backend/jobs/service.py @@ -57,7 +57,7 @@ class JobService: finished_at=finished, duration_ms=_duration(started, finished), error_code=type(exc).__name__, - error_message=str(exc) or "任务执行失败", + error_message=_exception_message(exc), ) raise finished = datetime.now(SHANGHAI) @@ -120,3 +120,10 @@ class JobService: def _duration(started: datetime, finished: datetime) -> int: return max(0, round((finished - started).total_seconds() * 1000)) + + +def _exception_message(error: Exception) -> str: + message = getattr(error, "message", None) + if isinstance(message, str) and message.strip(): + return message.strip() + return str(error).strip() or "任务执行失败" diff --git a/next/docs/evidence/local-preflight-2026-07-30.md b/next/docs/evidence/local-preflight-2026-07-30.md new file mode 100644 index 0000000..08742e4 --- /dev/null +++ b/next/docs/evidence/local-preflight-2026-07-30.md @@ -0,0 +1,50 @@ +# 本地迁移预检证据 + +日期：2026-07-30 + +## 范围 + +- 新版地址：`http://127.0.0.1:8877` +- 新版数据库：`next/data/local-preflight/xiaobai-normalized-v5.db` +- 旧版`8765`未启动、未占用、未修改。 +- 本轮未连接NAS，未执行Docker构建或容器切换。 + +## 迁移校验 + +- SQLite integrity：`ok` +- schema版本：12 +- 用户：3；自选：6；复盘笔记：3；问师消息：28；问天记录：31；策略跟踪：16 +- 未映射旧表：0；未映射快照类型：0；外键异常：0 +- 情绪快照按真实交易日去重；旧金额单位、涨跌停字段、情绪样本和个人出生信息已转换为新版契约。 + +## 真实浏览器 + +- 管理员会员`leefer`：同时显示管理员、会员；后台刷新和系统管理可见。 +- 会员非管理员`xiaobai`：仅显示会员；后台刷新和系统管理不可见。 +- 普通用户`ceshi`：显示普通用户；后台刷新和系统管理不可见；智能选股、问师、问天和复盘助手保持完整灰态结构。 +- 默认数据日期：`2026-07-30`；情绪周期温度34、退潮、20条交易日明细。 +- 涨停池56、炸板池23、跌停池83、昨日涨停81；复盘自选6；问师24位思维模型。 +- 龙虎榜`2026-07-24`归档：70只上榜股票、244条操作、23位活跃游资。`2026-07-29`和`2026-07-30`源缓存为空，不冒充解析结果。 +- 系统管理任务失败只显示“收盘行情读取失败，已保留原有快照”，不显示内部错误码或元组。 + +## 分辨率与持久化 + +- 1920×1080：情绪、轮动、智能选股、问天、复盘、系统管理均整页纵向滚动且无横向溢出。 +- 3840×2160：内容区最大宽度2200px，居中且无横向溢出，基础字号不随视口缩放。 +- 390×844：侧栏隐藏、底部导航和移动上下文栏显示；表格转卡片；交互控件触控高度不低于44px。 +- 停止并重启`8877`后：健康检查通过，管理员会话、默认日期、20行情绪明细、6只自选和系统配置保持。 + +## 自动质量门 + +- Ruff：通过 +- pytest：111项通过 +- Vue类型检查：通过 +- Vitest：7项通过 +- Vite生产构建：通过 +- Playwright：26项通过；在全新空白测试数据库中重跑，包含注册、权限、16工作区、日夜、390px和4K结构断言。 + +## 未关闭项 + +- 4K信息密度和审美仍需用户人工确认。 +- Docker/NAS构建、数据卷持久化、回退和正式切换未执行。 +- 旧运行时与一次性迁移器在正式切换和回退保留期结束前不清理。 diff --git a/next/docs/final/completion-audit.md b/next/docs/final/completion-audit.md index dde9fda..7d39127 100644 --- a/next/docs/final/completion-audit.md +++ b/next/docs/final/completion-audit.md @@ -26,6 +26,17 @@ 1. `U04` 已增加4K内容最大宽度、基础字号、Shell高度和无横向溢出断言，仍需最终人工视觉验收信息密度。 2. 第26节逐页状态证据已登记在`page-state-matrix.md`；4K人工视觉单元格在正式切换前保持未关闭。 +## 本地迁移预检 + +本轮按用户确认仅执行本地预检，未连接NAS或Docker。新版使用`127.0.0.1:8877`，旧版`8765` +未被占用或替换。完整证据见`../evidence/local-preflight-2026-07-30.md`。 + +- 旧库只读迁移到独立新版数据库后，SQLite完整性、外键、迁移版本和未映射类型均通过校验。 +- 管理员会员、会员非管理员、普通用户三个真实账户已验证双维权限；普通用户智能工具保持完整结构并锁定。 +- 情绪、四类股池、天梯、轮动、竞价、题材、热榜、龙虎榜归档、智能选股、问师、问天、复盘和系统管理均读取到迁移后的真实数据。 +- 1920×1080、3840×2160和390×844均无页面级横向溢出；移动端侧栏、底部导航、表格卡片化和44px触控目标已直接验证。 +- 服务进程停止并重新启动后，会话、默认数据日期、20行情绪明细、6只自选和系统配置均保持一致。 + ## 已在本轮纠正 - 阶段选股不再在当前阶段无结果时错误回退到第一条运行结果。 @@ -55,12 +66,13 @@ 没有补造无法验证的策略元信息。 - 图表夜间加载首帧、主题同步切换、7板布局和观心夜间关键文字对比度均已有浏览器直接断言，不再只依赖截图。 - 16个主工作区、策略跟踪、提醒、复盘助手、账户/会员及系统管理已逐项区分加载、真实空和失败；统一API客户端不会向页面泄露上游错误正文。 -- 本轮验证为Ruff、106项pytest、Vue类型检查、7项Vitest、生产构建和26项Playwright全部通过。 +- 后台任务不再把`AppError`内部元组直接展示到系统管理；历史记录读取时兼容清理，新增失败按用户可读说明入库。 +- 本轮验证为Ruff、111项pytest、Vue类型检查、7项Vitest、生产构建和26项Playwright全部通过。 -## 外部环境阻断项 +## 本轮范围外项目 -- 当前本机没有可用 Docker 引擎，镜像尚未在真实 Docker 环境构建和启动。 -- NAS 生产容器尚未切换；持久化重启、健康检查和回退必须在 Docker/NAS 上验证。 +- 按用户确认，本轮不执行Docker和NAS验收；镜像构建、容器启动和生产数据卷验证仍待正式切换阶段执行。 +- NAS 生产容器尚未切换；容器级持久化重启、健康检查和回退必须在Docker/NAS中另行验证。 - 旧运行时在正式切换前继续保留。切换 NAS 和清理旧实现必须取得用户最终明确确认。 - 旧实现的逐路径清理、数据归档和清理后验收已固化在 `legacy-cleanup-plan.md`； 一次性旧库迁移器也将在回退保留期结束后删除，不进入长期维护面。 diff --git a/next/docs/final/page-state-matrix.md b/next/docs/final/page-state-matrix.md index a1c883f..4fb7e39 100644 --- a/next/docs/final/page-state-matrix.md +++ b/next/docs/final/page-state-matrix.md @@ -15,6 +15,7 @@ - `A账户`：`test_accounts.py`；`A行情`：`test_market_data.py`、`test_market_insights.py`。 - `A智能`：`test_screener.py`、`test_mentor.py`、`test_heaven.py`；`A私有`：`test_review.py`。 - `A运维`：`test_operations.py`、`test_model_pool.py`、`test_backup_restore.py`及migration测试。 +- `L8877`：迁移数据库在本地生产模式`127.0.0.1:8877`的真实账户、真实数据、重启及三档视口预检。 | 页面/能力 | 权限 | 正常数据 | 空/缺失 | 加载/失败 | 持久化 | 日夜 | 1080P/390px | 4K | |---|---|---|---|---|---|---|---|---| @@ -38,6 +39,7 @@ ## 当前结论 - 权限、正常、空、缺失、加载、失败、持久化、日间、夜间、1080P和390px均已有直接代码或浏览器证据。 +- `L8877`已覆盖管理员会员、会员非管理员、普通用户，1920×1080、3840×2160、390×844，以及进程重启后的真实迁移数据保持。 - 市场工作区、智能工具、私有页面、全局弹窗和系统管理不会再把请求中或请求失败伪装成“暂无数据”。 - 4K已自动验证Shell尺寸、内容最大宽度、基础字号和无横向溢出；情绪、股池、系统管理和问师保留直接4K截图。 - 4K“信息密度是否舒适”属于审美判断，仍需正式切换前人工验收，不能由尺寸断言冒充完成。 diff --git a/next/frontend/src/shared/styles/mobile.css b/next/frontend/src/shared/styles/mobile.css index 0d52093..a1b4050 100644 --- a/next/frontend/src/shared/styles/mobile.css +++ b/next/frontend/src/shared/styles/mobile.css @@ -14,6 +14,10 @@ button, .btn, .icon-button, + .input, + .select, + .review-remark, + .tracking-entry, .seg-control button, .segmented button, .mobile-nav-item, diff --git a/next/tests/e2e/operations.spec.js b/next/tests/e2e/operations.spec.js index 3a97890..f87ef3d 100644 --- a/next/tests/e2e/operations.spec.js +++ b/next/tests/e2e/operations.spec.js @@ -9,6 +9,7 @@ async function authenticate(page) { if (!(await page.locator(".sidebar").isVisible())) { await page.getByRole("tab", { name: "注册" }).click(); await page.getByRole("button", { name: "注册并登录" }).click(); + await expect(page.locator(".sidebar")).toBeVisible(); } } diff --git a/next/tests/e2e/stage10.spec.js b/next/tests/e2e/stage10.spec.js index 5fd4c98..bb630d1 100644 --- a/next/tests/e2e/stage10.spec.js +++ b/next/tests/e2e/stage10.spec.js @@ -15,6 +15,7 @@ async function authenticate(page, username, password) { if (!(await page.locator(".sidebar").isVisible())) { await page.getByRole("tab", { name: "注册" }).click(); await page.getByRole("button", { name: "注册并登录" }).click(); + await expect(page.locator(".sidebar")).toBeVisible(); } } diff --git a/next/tests/e2e/stage11.spec.js b/next/tests/e2e/stage11.spec.js index 92e5c13..f89d02b 100644 --- a/next/tests/e2e/stage11.spec.js +++ b/next/tests/e2e/stage11.spec.js @@ -33,6 +33,7 @@ async function authenticate(page, username, password) { if (!(await page.locator(".sidebar").isVisible())) { await page.getByRole("tab", { name: "注册" }).click(); await page.getByRole("button", { name: "注册并登录" }).click(); + await expect(page.locator(".sidebar")).toBeVisible(); } } diff --git a/next/tests/e2e/stage12.spec.js b/next/tests/e2e/stage12.spec.js index 1458a60..a6f3e9c 100644 --- a/next/tests/e2e/stage12.spec.js +++ b/next/tests/e2e/stage12.spec.js @@ -15,6 +15,7 @@ async function authenticate(page, username, password) { if (!(await page.locator(".sidebar").isVisible())) { await page.getByRole("tab", { name: "注册" }).click(); await page.getByRole("button", { name: "注册并登录" }).click(); + await expect(page.locator(".sidebar")).toBeVisible(); } } diff --git a/next/tests/e2e/stage13.spec.js b/next/tests/e2e/stage13.spec.js index b745286..094d060 100644 --- a/next/tests/e2e/stage13.spec.js +++ b/next/tests/e2e/stage13.spec.js @@ -15,6 +15,7 @@ async function authenticate(page) { if (!(await page.locator(".sidebar").isVisible())) { await page.getByRole("tab", { name: "注册" }).click(); await page.getByRole("button", { name: "注册并登录" }).click(); + await expect(page.locator(".sidebar")).toBeVisible(); } } diff --git a/next/tests/e2e/stage5.spec.js b/next/tests/e2e/stage5.spec.js index 2429ebb..fccd978 100644 --- a/next/tests/e2e/stage5.spec.js +++ b/next/tests/e2e/stage5.spec.js @@ -16,6 +16,7 @@ async function authenticate(page) { if (!(await page.locator(".sidebar").isVisible())) { await page.getByRole("tab", { name: "注册" }).click(); await page.getByRole("button", { name: "注册并登录" }).click(); + await expect(page.locator(".sidebar")).toBeVisible(); } } diff --git a/next/tests/e2e/stage6.spec.js b/next/tests/e2e/stage6.spec.js index 76c12ed..9f2fffa 100644 --- a/next/tests/e2e/stage6.spec.js +++ b/next/tests/e2e/stage6.spec.js @@ -15,6 +15,7 @@ async function authenticate(page) { if (!(await page.locator(".sidebar").isVisible())) { await page.getByRole("tab", { name: "注册" }).click(); await page.getByRole("button", { name: "注册并登录" }).click(); + await expect(page.locator(".sidebar")).toBeVisible(); } } diff --git a/next/tests/e2e/stage7.spec.js b/next/tests/e2e/stage7.spec.js index ac2f708..97a7d4f 100644 --- a/next/tests/e2e/stage7.spec.js +++ b/next/tests/e2e/stage7.spec.js @@ -15,6 +15,7 @@ async function authenticate(page) { if (!(await page.locator(".sidebar").isVisible())) { await page.getByRole("tab", { name: "注册" }).click(); await page.getByRole("button", { name: "注册并登录" }).click(); + await expect(page.locator(".sidebar")).toBeVisible(); } } diff --git a/next/tests/e2e/stage8.spec.js b/next/tests/e2e/stage8.spec.js index 28e9818..89a1d78 100644 --- a/next/tests/e2e/stage8.spec.js +++ b/next/tests/e2e/stage8.spec.js @@ -15,6 +15,7 @@ async function authenticate(page) { if (!(await page.locator(".sidebar").isVisible())) { await page.getByRole("tab", { name: "注册" }).click(); await page.getByRole("button", { name: "注册并登录" }).click(); + await expect(page.locator(".sidebar")).toBeVisible(); } } diff --git a/next/tests/test_accounts.py b/next/tests/test_accounts.py index 4319dca..8dc6ccb 100644 --- a/next/tests/test_accounts.py +++ b/next/tests/test_accounts.py @@ -187,6 +187,25 @@ def test_birth_profile_is_encrypted_and_isolated_by_account(tmp_path) -> None: run_scenario(application, scenario) +def test_incomplete_encrypted_profile_returns_controlled_error(tmp_path) -> None: + application = create_application(Settings.for_test(tmp_path)) + + async def scenario(client: httpx.AsyncClient) -> None: + response, _session = await register(client, "broken-profile", USER_PASSWORD) + user_id = response.json()["account"]["id"] + encrypted = application.state.container.accounts._cipher.encrypt('{"gender":"male"}') + with sqlite3.connect(application.state.settings.database_path) as connection: + connection.execute( + "INSERT INTO birth_profiles VALUES (?,?,datetime('now'),datetime('now'))", + (user_id, encrypted), + ) + result = await client.get("/api/account/profile") + assert result.status_code == 503 + assert result.json()["error"]["code"] == "profile_unavailable" + + run_scenario(application, scenario) + + def test_membership_and_admin_are_independent_dimensions(tmp_path) -> None: application = create_application(Settings.for_test(tmp_path)) diff --git a/next/tests/test_legacy_migration.py b/next/tests/test_legacy_migration.py index c562e53..1d4c180 100644 --- a/next/tests/test_legacy_migration.py +++ b/next/tests/test_legacy_migration.py @@ -92,7 +92,9 @@ def _legacy_database(path, key: str) -> None: "INSERT INTO users VALUES (8,'leefer',?,?,?,?,'admin','auto','active','永久',?,NULL)", (encoded_salt, encoded_hash, now, now, now), ) - profile = fernet.encrypt(b'{"gender":"male"}').decode() + profile = fernet.encrypt( + b'{"birth_datetime":"1990-03-08T08:30:00","gender":"male"}' + ).decode() connection.execute("INSERT INTO user_birth_profiles VALUES (8,?,?)", (profile, now)) connection.execute( "INSERT INTO llm_usage VALUES (1,8,'mentor','model','','success',3,?,'','','',0,0)", @@ -109,12 +111,37 @@ def _legacy_database(path, key: str) -> None: ) dashboard = json.dumps( { + "meta": { + "requested_date": "2026-07-30", + "trade_date": "2026-07-29", + "previous_trade_date": "2026-07-28", + }, + "overview": { + "up_count": 1, + "down_count": 0, + "flat_count": 0, + "limit_up_count": 1, + "limit_down_count": 0, + "broken_count": 0, + "amount_billion": 12.5, + "seal_rate": 100, + "sentiment_score": 64, + "sentiment_label": "情绪偏强", + "sentiment_phase": "修复", + "sentiment_direction": "升温", + "sentiment_components": { + "breadth": {"label": "市场宽度", "score": 100, "weight": 20} + }, + "sentiment_engine_version": 2, + }, "limits": [ { "ts_code": "000001.SZ", "code": "000001", "name": "Ping An Bank", "reason": "legacy", + "amount_billion": 1.25, + "seal_amount_million": 20, } ], "broken": [], @@ -122,7 +149,7 @@ def _legacy_database(path, key: str) -> None: } ) connection.execute( - "INSERT INTO dashboard_snapshots VALUES ('20260729','tushare',?,1,?)", + "INSERT INTO dashboard_snapshots VALUES ('20260730','tushare',?,1,?)", (dashboard, now), ) connection.execute( @@ -347,6 +374,18 @@ def test_legacy_migration_is_idempotent_and_preserves_login(tmp_path) -> None: password = connection.execute("SELECT password_hash FROM users WHERE id=8").fetchone()[0] assert PasswordHasher().verify("Password123", password) assert connection.execute("SELECT count(*) FROM watchlist_entries").fetchone()[0] == 1 + profile_payload = json.loads( + Fernet(key.encode()).decrypt( + connection.execute( + "SELECT encrypted_payload FROM birth_profiles WHERE user_id=8" + ).fetchone()[0].encode() + ) + ) + assert profile_payload == { + "birth_date": "1990-03-08", + "birth_time": "08:30", + "gender": "male", + } assert connection.execute("SELECT count(*) FROM screener_runs").fetchone()[0] == 1 assert connection.execute("SELECT count(*) FROM strategy_tracks").fetchone()[0] == 1 assert connection.execute("SELECT daily_llm_limit FROM memberships").fetchone()[0] == 61 @@ -363,6 +402,14 @@ def test_legacy_migration_is_idempotent_and_preserves_login(tmp_path) -> None: ).fetchone()[0] ) assert summary["limits"][0]["identifier"] == "000001.SZ" + assert summary["trade_date"] == "2026-07-29" + assert summary["overview"]["limit_up"] == 1 + assert summary["overview"]["amount"] == 1_250_000_000 + assert summary["limits"][0]["amount"] == 125_000_000 + assert summary["limits"][0]["seal_amount"] == 20_000_000 + assert summary["sentiment"]["score"] == 64 + assert summary["sentiment"]["phase"] == "修复" + assert summary["sentiment"]["components"][0]["key"] == "breadth" all_revisions = tuple( connection.execute( """SELECT * FROM market_event_revisions WHERE trade_date='2026-07-29' diff --git a/next/tests/test_market_data.py b/next/tests/test_market_data.py index a6f639a..6a1ae7c 100644 --- a/next/tests/test_market_data.py +++ b/next/tests/test_market_data.py @@ -475,6 +475,7 @@ def test_sentiment_has_all_weighted_components_and_extreme_risk_cap() -> None: sentiment = calculate_sentiment(snapshot, []) assert sentiment["score"] <= 15 assert sentiment["phase"] == "冰点" + assert sentiment["stats"]["yesterday_count"] == 0 assert {item["key"]: item["weight"] for item in sentiment["components"]} == { "breadth": 20, "limit_ecology": 25, @@ -551,3 +552,53 @@ def test_incomplete_daily_snapshot_is_rejected_without_overwriting(tmp_path) -> service.sync("2026-07-29", datetime(2026, 7, 30, 16, tzinfo=SHANGHAI)) with database.read() as connection: assert repository.latest_summary(connection, "2026-07-29") is None + + +def test_synced_snapshot_keeps_event_lists_separate_from_overview_counts(tmp_path) -> None: + database = Database(tmp_path / "sync-complete.db") + MigrationRunner(database).upgrade(MIGRATIONS) + repository = MarketRepository() + with database.transaction() as connection: + repository.replace_calendar( + connection, + ( + {"cal_date": "20260728", "is_open": 1, "pretrade_date": "20260727"}, + {"cal_date": "20260729", "is_open": 1, "pretrade_date": "20260728"}, + ), + "tushare", + "2026-07-29T15:00:00+08:00", + ) + repository.replace_stocks( + connection, + ({ + "ts_code": "000001.SZ", "symbol": "000001", "name": "平安银行", + "industry": "银行", "list_status": "L", + },), + "tushare", + "2026-07-29T15:00:00+08:00", + ) + provider = TushareProvider("test-token") + stock = { + "ts_code": "000001.SZ", "name": "平安银行", "industry": "银行", + "close": 10, "pct_chg": 5, "amount": 100, "limit_times": 1, + } + provider.snapshot_inputs = lambda *_: { + "daily": calculation_result([stock]), + "limit_up": calculation_result([]), + "limit_down": calculation_result([]), + "broken": calculation_result([stock]), + "previous_limit_up": calculation_result([]), + "price_limits": calculation_result([{"ts_code": "000001.SZ", "up_limit": 11}]), + } + service = MarketSnapshotService( + database, + repository, + DataGateway(database, repository, (provider,), DataSourcePolicy()), + ) + service.sync("2026-07-29", datetime(2026, 7, 30, 16, tzinfo=SHANGHAI)) + + broken = service.workspace("broken", "2026-07-29") + emotion = service.workspace("emotion", "2026-07-29") + assert len(broken["items"]) == 1 + assert broken["items"][0]["identifier"] == "000001.SZ" + assert emotion["overview"]["broken"] == 1 diff --git a/next/tests/test_operations.py b/next/tests/test_operations.py index e0e92a3..826bd52 100644 --- a/next/tests/test_operations.py +++ b/next/tests/test_operations.py @@ -21,6 +21,7 @@ from backend.database.connection import Database from backend.database.migrations import MIGRATIONS, MigrationRunner from backend.features.market.events import MarketEventService, apply_event_revisions from backend.features.market.snapshot import build_realtime_inputs, build_snapshot +from backend.http.errors import AppError from backend.jobs.repository import JobRepository from backend.jobs.service import JobAlreadyRunning, JobService from tests.support import run_scenario @@ -108,6 +109,36 @@ def test_job_service_records_success_failure_attempts_and_exclusion(tmp_path) -> ) +def test_job_service_exposes_app_error_message_without_internal_tuple(tmp_path) -> None: + database = Database(tmp_path / "jobs.db") + MigrationRunner(database).upgrade(MIGRATIONS) + jobs = JobService(database, JobRepository()) + + with pytest.raises(AppError): + jobs.execute( + kind="market.refresh", + run_key="2026-07-30:failed", + requested_date="2026-07-30", + trigger="administrator", + operation=lambda: (_ for _ in ()).throw( + AppError("market_data_unavailable", "收盘行情读取失败，已保留原有快照", 503) + ), + stale_after_seconds=120, + ) + + assert jobs.latest()[0]["error_message"] == "收盘行情读取失败，已保留原有快照" + + with database.transaction() as connection: + connection.execute( + "UPDATE job_runs SET error_message = ? WHERE id = ?", + ( + "('market_data_unavailable', '收盘行情读取失败，已保留原有快照', 503)", + jobs.latest()[0]["id"], + ), + ) + assert jobs.latest()[0]["error_message"] == "收盘行情读取失败，已保留原有快照" + + class EventGateway: reason = "银行板块走强" diff --git a/next/tools/legacy_migration.py b/next/tools/legacy_migration.py index 95d3dd6..59730a5 100644 --- a/next/tools/legacy_migration.py +++ b/next/tools/legacy_migration.py @@ -13,6 +13,7 @@ from typing import Any from cryptography.fernet import Fernet +from backend.data.sentiment import calculate_sentiment from backend.database import MIGRATIONS, Database, MigrationRunner ARCHIVE_VERSION = "legacy-archive-v1" @@ -146,6 +147,121 @@ def _normalize_identifiers(value: Any) -> Any: return normalized +def _normalize_market_units(value: Any) -> Any: + if isinstance(value, list): + return [_normalize_market_units(item) for item in value] + if not isinstance(value, dict): + return value + normalized = {key: _normalize_market_units(item) for key, item in value.items()} + for legacy, canonical, multiplier in ( + ("amount_billion", "amount", 100_000_000), + ("seal_amount_million", "seal_amount", 1_000_000), + ("float_mv_billion", "float_mv", 100_000_000), + ): + raw = normalized.pop(legacy, None) + if canonical not in normalized and raw not in (None, ""): + normalized[canonical] = float(raw) * multiplier + return normalized + + +def _legacy_sentiment( + payload: dict[str, Any], history: list[dict[str, Any]] +) -> dict[str, Any]: + calculated = calculate_sentiment(payload, history) + overview = payload.get("overview") or {} + if int(overview.get("sentiment_engine_version") or 0) < 2: + return calculated + score = overview.get("sentiment_score") + if score is None or not overview.get("sentiment_phase"): + return calculated + previous_scores = [ + float(item["sentiment"]["score"]) + for item in history[-3:] + if (item.get("sentiment") or {}).get("score") is not None + ] + numeric_score = float(score) + previous_score = previous_scores[-1] if previous_scores else numeric_score + baseline = sum(previous_scores) / len(previous_scores) if previous_scores else numeric_score + components = overview.get("sentiment_components") or {} + if isinstance(components, dict): + calculated["components"] = [ + {"key": key, **dict(component)} + for key, component in components.items() + if isinstance(component, dict) + ] + calculated.update( + { + "score": int(round(numeric_score)), + "label": str(overview.get("sentiment_label") or calculated["label"]), + "phase": str(overview["sentiment_phase"]), + "direction": str(overview.get("sentiment_direction") or calculated["direction"]), + "day_change": round(numeric_score - previous_score, 1), + "momentum": round(numeric_score - baseline, 1), + } + ) + return calculated + + +def _normalize_market_snapshot( + value: dict[str, Any], trade_date: str, history: list[dict[str, Any]] +) -> dict[str, Any]: + payload = _normalize_market_units(_normalize_identifiers(value)) + overview = dict(payload.get("overview") or {}) + for legacy, canonical, fallback in ( + ("limit_up_count", "limit_up", len(payload.get("limits") or [])), + ("limit_down_count", "limit_down", len(payload.get("down_limits") or [])), + ("broken_count", "broken", len(payload.get("broken") or [])), + ): + raw = overview.pop(legacy, None) + if canonical not in overview: + overview[canonical] = raw if raw is not None else fallback + if "amount" not in overview: + raw_amount = overview.pop("amount_billion", None) + overview["amount"] = float(raw_amount or 0) * 100_000_000 + meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {} + payload["trade_date"] = trade_date + previous_date = payload.get("previous_trade_date") or meta.get("previous_trade_date") + payload["previous_trade_date"] = _iso_date(str(previous_date)) if previous_date else "" + payload["overview"] = overview + sentiment = _legacy_sentiment(payload, history) + payload["sentiment"] = sentiment + payload["temperature"] = sentiment["score"] + for key in tuple(overview): + if key.startswith("sentiment_"): + overview.pop(key) + return payload + + +def _normalize_birth_profile(encrypted_payload: str, encryption_key: str | None) -> str | None: + if not encryption_key: + raise ValueError("legacy birth profiles require APP_ENCRYPTION_KEY") + fernet = Fernet(encryption_key.encode("ascii")) + decrypted = fernet.decrypt(encrypted_payload.encode("ascii")) + try: + payload = json.loads(decrypted) + gender = str(payload.get("gender") or "") + birth_date = str(payload.get("birth_date") or "") + birth_time = str(payload.get("birth_time") or "") + if (not birth_date or not birth_time) and payload.get("birth_datetime"): + stamp = datetime.fromisoformat(str(payload["birth_datetime"]).replace(" ", "T", 1)) + birth_date = stamp.date().isoformat() + birth_time = stamp.time().replace(tzinfo=None, second=0, microsecond=0).isoformat( + timespec="minutes" + ) + datetime.fromisoformat(f"{birth_date}T{birth_time}") + if gender not in {"male", "female"}: + return None + except (json.JSONDecodeError, TypeError, ValueError): + return None + normalized = json.dumps( + {"birth_date": birth_date, "birth_time": birth_time, "gender": gender}, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return fernet.encrypt(normalized.encode("utf-8")).decode("ascii") + + def _observed_at(payload: dict[str, Any], fallback: str) -> str: meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {} return str( @@ -295,6 +411,18 @@ class LegacyMigrator: self.admin_id = min(administrators or self.user_ids) if _table_exists(source, "user_birth_profiles"): for row in source.execute("SELECT * FROM user_birth_profiles"): + encrypted_profile = _normalize_birth_profile( + str(row["encrypted_payload"]), self.key + ) + if encrypted_profile is None: + self.row_skips["birth_profiles_invalid"] = { + "count": self.row_skips.get("birth_profiles_invalid", {}).get( + "count", 0 + ) + + 1, + "reason": "legacy profile is incomplete and must be configured again", + } + continue target.execute( """INSERT INTO birth_profiles (user_id,encrypted_payload,created_at,updated_at) VALUES (?,?,?,?) @@ -302,7 +430,7 @@ class LegacyMigrator: encrypted_payload=excluded.encrypted_payload,updated_at=excluded.updated_at""", ( row["user_id"], - row["encrypted_payload"], + encrypted_profile, row["updated_at"], row["updated_at"], ), @@ -425,9 +553,21 @@ class LegacyMigrator: ) previous = trade_date self.counts["trading_days"] = len(dates) - for row in source.execute("SELECT * FROM dashboard_snapshots"): - payload = _normalize_identifiers(_json(row["payload"], {})) - trade_date = _iso_date(row["trade_date"]) + snapshots: dict[str, tuple[sqlite3.Row, dict[str, Any]]] = {} + for row in source.execute( + "SELECT * FROM dashboard_snapshots ORDER BY trade_date,updated_at" + ): + payload = _json(row["payload"], {}) + meta = payload.get("meta") if isinstance(payload.get("meta"), dict) else {} + trade_date = _iso_date( + str(payload.get("trade_date") or meta.get("trade_date") or row["trade_date"]) + ) + snapshots[trade_date] = (row, payload) + history: list[dict[str, Any]] = [] + for trade_date in sorted(snapshots): + row, raw_payload = snapshots[trade_date] + payload = _normalize_market_snapshot(raw_payload, trade_date, history) + history.append(payload) events: dict[str, str] = {} for event_type, key in ( ("limit_up", "limits"), @@ -456,7 +596,7 @@ class LegacyMigrator: row["updated_at"], ), ) - self.counts["market_summaries"] += 1 + self.counts["market_summaries"] = len(snapshots) query = """WITH ranked AS ( SELECT *,row_number() OVER (PARTITION BY ts_code ORDER BY trade_date DESC) rank FROM daily_bars) SELECT * FROM ranked WHERE rank<=90 ORDER BY ts_code,trade_date"""