Compare commits

...
Author SHA1 Message Date
leefer 2ef31f6115 refactor: enforce canonical backend imports 2026-08-02 00:10:43 +08:00
leefer 159a9a6a8b refactor: centralize numeric normalization 2026-08-01 16:54:47 +08:00
leefer 7ed181e682 docs: record reduction acceptance 2026-08-01 15:40:31 +08:00
19 changed files with 283 additions and 77 deletions
+6 -2
View File
@@ -36,7 +36,7 @@ background scheduler
- `backend/features/<feature>/` owns the mechanically moved service, repository, HTTP, agent,
or deterministic calculation code for that product area.
- `backend/data/` owns provider construction, source policy, provenance, units, freshness,
coverage, and display-versus-calculation eligibility.
coverage, display-versus-calculation eligibility, and shared numeric normalization policies.
- `backend/database/` owns connection management, ordered migrations, and narrow repository
adapters. Root `database.py` remains the legacy schema/composition anchor and combines the
feature repository mixins; do not add feature queries to it.
@@ -55,7 +55,11 @@ background scheduler
Root modules such as `screener.py`, `tushare_client.py`, and `mentor_agent.py` are compatibility
aliases to canonical modules. They contain no second implementation and remain only because
the original public import surface is part of the preservation contract.
the original public import surface is part of the preservation contract. Canonical backend
modules must import other canonical modules directly rather than routing through these aliases.
The remaining `api_access` import in `backend/application.py` and preserved lazy
`sentiment_engine` import in the screener repository are registered transition boundaries;
the root `database.py` remains the documented schema/composition anchor.
## Non-negotiable maintenance rules
+1 -1
View File
@@ -9,10 +9,10 @@ from backend.database.repositories import RepositoryBundle, build_repository_bun
from backend.features.alerts import AlertService
from backend.features.mentor.agent import MentorSkillRegistry
from backend.features.review import TradeJournalService
from backend.features.screener.engine import ScreenerEngine
from backend.features.screener.tracking import StrategyTrackingService
from backend.jobs import InProcessJobRunner, JobRegistry, SQLiteJobRunRepository
from database import ReviewDatabase
from screener import ScreenerEngine
from backend.data.providers.ifind_client import IfindHttpClient
from backend.data.realtime import WebRealtimeAggregator
from backend.features.market.charts import MarketChartClient
+20
View File
@@ -0,0 +1,20 @@
from __future__ import annotations
import math
from typing import Any
def finite_number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if math.isfinite(number) else default
except (TypeError, ValueError):
return default
def non_nan_number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if number == number else default
except (TypeError, ValueError):
return default
+1 -8
View File
@@ -11,6 +11,7 @@ from datetime import datetime, time as dt_time, timedelta
from threading import Lock
from typing import Any, ClassVar
from backend.data.numbers import finite_number as _number
from backend.features.sentiment.engine import apply_sentiment_to_dashboard
@@ -1812,14 +1813,6 @@ class TushareClient:
}
def _number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if math.isfinite(number) else default
except (TypeError, ValueError):
return default
def _text(value: Any) -> str:
if isinstance(value, (list, tuple, set)):
return "".join(str(item).strip() for item in value if str(item).strip())
+1 -8
View File
@@ -6,6 +6,7 @@ from datetime import datetime, time as dt_time, timedelta, timezone
from statistics import median
from typing import TYPE_CHECKING, Any, Callable
from backend.data.numbers import non_nan_number as _number
from backend.data.providers.ifind_client import IfindError, IfindHttpClient
from backend.data.providers.tushare_client import TushareClient, TushareError
@@ -16,14 +17,6 @@ if TYPE_CHECKING:
CHINA_TIMEZONE = timezone(timedelta(hours=8))
def _number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if number == number else default
except (TypeError, ValueError):
return default
def _display_date(value: str) -> str:
text = str(value or "").replace("-", "")
if len(text) != 8:
+1 -1
View File
@@ -4,7 +4,7 @@ import json
from typing import Any
from backend.llm import transport as llm_transport
from screener import FACTOR_FIELDS, REGIMES
from backend.features.screener.engine import FACTOR_FIELDS, REGIMES
class LLMCompilerError(RuntimeError):
+3 -10
View File
@@ -8,10 +8,11 @@ from collections import defaultdict
from datetime import datetime, timedelta
from typing import Any
from advanced_strategies import ADVANCED_CURATED_STRATEGIES
from backend.data.numbers import finite_number as _number
from backend.data.providers.tushare_client import TushareClient, TushareError
from backend.features.screener.strategies import ADVANCED_CURATED_STRATEGIES
from database import ReviewDatabase
from backend.features.sentiment.engine import build_sentiment_history, latest_contiguous_history
from tushare_client import TushareClient, TushareError
REGIMES = {
@@ -2201,13 +2202,5 @@ def _regime_reason(regime: str) -> str:
}.get(regime, "市场阶段待确认。")
def _number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if math.isfinite(number) else default
except (TypeError, ValueError):
return default
def _display_date(value: str) -> str:
return f"{value[:4]}-{value[4:6]}-{value[6:8]}" if len(value) == 8 else value
+2 -8
View File
@@ -4,6 +4,8 @@ from copy import deepcopy
from statistics import mean, median
from typing import Any
from backend.data.numbers import non_nan_number as _number
COMPONENT_WEIGHTS = {
"breadth": 20,
@@ -16,14 +18,6 @@ COMPONENT_WEIGHTS = {
SENTIMENT_ENGINE_VERSION = 2
def _number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if number == number else default
except (TypeError, ValueError):
return default
def _clamp(value: float, lower: float = 0.0, upper: float = 100.0) -> float:
return min(upper, max(lower, value))
+16 -6
View File
@@ -220,6 +220,16 @@
"runtime_role": "index observation fallback"
}
],
"numeric_normalization": [
{
"function": "finite_number",
"path": "backend/data/numbers.py"
},
{
"function": "non_nan_number",
"path": "backend/data/numbers.py"
}
],
"llm_entrypoints": [
{
"function": "stream_with_mentor",
@@ -279,13 +289,13 @@
},
{
"path": "backend/features/screener/engine.py",
"bytes": 108552,
"lines": 2213
"bytes": 108434,
"lines": 2206
},
{
"path": "backend/data/providers/tushare_client.py",
"bytes": 94329,
"lines": 2175
"bytes": 94171,
"lines": 2168
},
{
"path": "frontend/app.js",
@@ -314,8 +324,8 @@
},
{
"path": "backend/features/market/insights.py",
"bytes": 58150,
"lines": 1314
"bytes": 57998,
"lines": 1307
},
{
"path": "frontend/pages/market/runtime.js",
+2 -1
View File
@@ -3,7 +3,8 @@ from __future__ import annotations
import argparse
from datetime import date
from server import SERVICE, normalize_date
from backend.application import SERVICE
from backend.bootstrap.config import normalize_date
def main() -> None:
+36
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import ast
import hashlib
import re
from pathlib import Path
@@ -33,6 +34,41 @@ def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def function_contract(path: Path, name: str) -> tuple[str, str]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
function = next(
node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name
)
body = ast.Module(body=function.body, type_ignores=[])
return (
ast.dump(function.args, include_attributes=False),
ast.dump(body, include_attributes=False),
)
def module_contract(
path: Path,
*,
excluded_definitions: set[str] | None = None,
excluded_import_modules: set[str] | None = None,
exclude_imports: bool = False,
) -> str:
excluded_definitions = excluded_definitions or set()
excluded_import_modules = excluded_import_modules or set()
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
tree.body = [
node
for node in tree.body
if not (exclude_imports and isinstance(node, (ast.Import, ast.ImportFrom)))
and not (
isinstance(node, ast.ImportFrom)
and node.module in excluded_import_modules
)
and getattr(node, "name", None) not in excluded_definitions
]
return ast.dump(tree, include_attributes=False)
def reassembled_frontend_runtime() -> str:
chunks: dict[tuple[int, int], str] = {}
for path in FRONTEND_ROOT.rglob("*.js"):
+47 -6
View File
@@ -20,12 +20,6 @@ class FeatureBoundaryTests(unittest.TestCase):
}
violations = []
for path in FEATURES.rglob("*.py"):
# The screener engine is an exact-preservation move of the legacy
# calculation module. Its provider dependency is covered by the
# slice equivalence tests and will be addressed only after the
# behavior-preserving migration is complete.
if path.relative_to(FEATURES).as_posix() == "screener/engine.py":
continue
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
for node in ast.walk(tree):
names = []
@@ -38,6 +32,53 @@ class FeatureBoundaryTests(unittest.TestCase):
violations.append(f"{path.relative_to(ROOT)} -> {name}")
self.assertEqual(violations, [])
def test_backend_uses_root_compatibility_modules_only_at_declared_boundaries(self) -> None:
compatibility_modules = {
"advanced_strategies",
"alert_service",
"api_access",
"app_config",
"assistant_agent",
"chart_data_provider",
"heaven_agent",
"heaven_engine",
"ifind_client",
"llm_strategy",
"llm_stream",
"market_insights",
"mentor_agent",
"realtime_aggregator",
"screener",
"security",
"sentiment_engine",
"server",
"strategy_tracking",
"trade_journal",
"tushare_client",
}
allowed = {
"backend/application.py": {"api_access"},
"backend/features/screener/repository.py": {"sentiment_engine"},
}
violations = []
for path in (ROOT / "backend").rglob("*.py"):
relative = path.relative_to(ROOT).as_posix()
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
for node in ast.walk(tree):
names = []
if isinstance(node, ast.Import):
names = [alias.name for alias in node.names]
elif isinstance(node, ast.ImportFrom) and node.module:
names = [node.module]
for name in names:
root_name = name.split(".")[0]
if (
root_name in compatibility_modules
and root_name not in allowed.get(relative, set())
):
violations.append(f"{relative} -> {name}")
self.assertEqual(violations, [])
def test_legacy_service_modules_are_compatibility_exports_only(self) -> None:
for filename in ("alert_service.py", "trade_journal.py", "strategy_tracking.py"):
tree = ast.parse((ROOT / filename).read_text(encoding="utf-8"))
+38
View File
@@ -0,0 +1,38 @@
from __future__ import annotations
import math
import unittest
from backend.data.numbers import finite_number, non_nan_number
from backend.data.providers import tushare_client
from backend.features.market import insights
from backend.features.screener import engine as screener_engine
from backend.features.sentiment import engine as sentiment_engine
class NumericNormalizationTests(unittest.TestCase):
def test_consumers_use_their_declared_shared_policy(self) -> None:
self.assertIs(tushare_client._number, finite_number)
self.assertIs(screener_engine._number, finite_number)
self.assertIs(insights._number, non_nan_number)
self.assertIs(sentiment_engine._number, non_nan_number)
def test_finite_policy_preserves_existing_results(self) -> None:
self.assertEqual(finite_number("12.5"), 12.5)
self.assertEqual(finite_number(None), 0.0)
self.assertEqual(finite_number("invalid", 7.0), 7.0)
self.assertEqual(finite_number(math.nan, 7.0), 7.0)
self.assertEqual(finite_number(math.inf, 7.0), 7.0)
self.assertEqual(finite_number(-math.inf, 7.0), 7.0)
def test_non_nan_policy_keeps_infinity_but_rejects_nan(self) -> None:
self.assertEqual(non_nan_number("12.5"), 12.5)
self.assertEqual(non_nan_number(None), 0.0)
self.assertEqual(non_nan_number("invalid", 7.0), 7.0)
self.assertEqual(non_nan_number(math.nan, 7.0), 7.0)
self.assertEqual(non_nan_number(math.inf, 7.0), math.inf)
self.assertEqual(non_nan_number(-math.inf, 7.0), -math.inf)
if __name__ == "__main__":
unittest.main()
+10 -13
View File
@@ -11,12 +11,14 @@ import realtime_aggregator
import tushare_client
from backend.bootstrap import config as bootstrap_config
from backend.data import realtime
from backend.data.numbers import finite_number
from backend.data.providers import ifind_client as canonical_ifind
from backend.data.providers import tushare_client as canonical_tushare
from backend.features.market import charts
from tests.preservation_helpers import (
assert_frontend_runtime_matches_audited_baseline,
assert_moved_asset_matches,
function_contract,
)
@@ -102,18 +104,6 @@ def top_level_definitions(path: Path) -> dict[str, str]:
}
def function_contract(path: Path, name: str) -> tuple[str, str]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
function = next(
node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name
)
body = ast.Module(body=function.body, type_ignores=[])
return (
ast.dump(function.args, include_attributes=False),
ast.dump(body, include_attributes=False),
)
class MarketSliceSourceEquivalenceTests(unittest.TestCase):
def test_market_service_methods_are_exact_original_ast(self) -> None:
original = class_methods(ORIGINAL_ROOT / "server.py", "DashboardService")
@@ -154,10 +144,17 @@ class MarketSliceSourceEquivalenceTests(unittest.TestCase):
)
for original, migrated in exact_moves:
self.assertEqual(sha256(ORIGINAL_ROOT / original), sha256(APP_ROOT / migrated))
original_tushare = top_level_definitions(ORIGINAL_ROOT / "tushare_client.py")
original_tushare.pop("_number")
self.assertEqual(
top_level_definitions(ORIGINAL_ROOT / "tushare_client.py"),
original_tushare,
top_level_definitions(APP_ROOT / "backend/data/providers/tushare_client.py"),
)
self.assertEqual(
function_contract(ORIGINAL_ROOT / "tushare_client.py", "_number"),
function_contract(APP_ROOT / "backend/data/numbers.py", "finite_number"),
)
self.assertIs(canonical_tushare._number, finite_number)
original_charts = top_level_definitions(ORIGINAL_ROOT / "chart_data_provider.py")
original_charts.pop("_stock_market_code")
self.assertEqual(
@@ -6,11 +6,13 @@ import unittest
from pathlib import Path
import market_insights
from backend.data.numbers import non_nan_number
from backend.features.market import insights as canonical_insights
from tests.preservation_helpers import (
assert_frontend_runtime_matches_audited_baseline,
assert_moved_asset_matches,
assert_page_prefix_matches,
function_contract,
)
@@ -106,6 +108,11 @@ class MarketInsightsSliceSourceEquivalenceTests(unittest.TestCase):
MARKET_INSIGHT_METHODS,
)
self.assertIs(market_insights.MarketInsightsService, canonical_insights.MarketInsightsService)
self.assertEqual(
function_contract(ORIGINAL_ROOT / "market_insights.py", "_number"),
function_contract(APP_ROOT / "backend/data/numbers.py", "non_nan_number"),
)
self.assertIs(canonical_insights._number, non_nan_number)
def test_dashboard_service_methods_are_exact_original_ast(self) -> None:
original = ORIGINAL_ROOT / "server.py"
+17 -11
View File
@@ -9,12 +9,15 @@ import advanced_strategies
import llm_strategy
import screener
import strategy_tracking
from backend.data.numbers import finite_number
from backend.features.screener import compiler, engine, strategies, tracking
from backend.features.screener import service as screener_service
from tests.preservation_helpers import (
assert_frontend_runtime_matches_audited_baseline,
assert_moved_asset_matches,
assert_page_prefix_matches,
function_contract,
module_contract,
)
@@ -91,14 +94,6 @@ def top_level_definition(path: Path, name: str) -> str:
return ast.dump(node, include_attributes=False)
def module_without_imports(path: Path) -> str:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
tree.body = [
node for node in tree.body if not isinstance(node, (ast.Import, ast.ImportFrom))
]
return ast.dump(tree, include_attributes=False)
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
@@ -154,11 +149,22 @@ class ScreenerSliceSourceEquivalenceTests(unittest.TestCase):
def test_engine_and_tracking_logic_match_the_original(self) -> None:
self.assertEqual(
module_without_imports(ORIGINAL_ROOT / "screener.py"),
module_without_imports(
APP_ROOT / "backend" / "features" / "screener" / "engine.py"
module_contract(
ORIGINAL_ROOT / "screener.py",
excluded_definitions={"_number"},
exclude_imports=True,
),
module_contract(
APP_ROOT / "backend" / "features" / "screener" / "engine.py",
excluded_definitions={"_number"},
exclude_imports=True,
),
)
self.assertEqual(
function_contract(ORIGINAL_ROOT / "screener.py", "_number"),
function_contract(APP_ROOT / "backend/data/numbers.py", "finite_number"),
)
self.assertIs(engine._number, finite_number)
self.assertEqual(
class_methods(
ORIGINAL_ROOT / "backend" / "features" / "screener" / "tracking.py",
@@ -6,11 +6,14 @@ import unittest
from pathlib import Path
import sentiment_engine
from backend.data.numbers import non_nan_number
from backend.features.sentiment import engine as canonical_engine
from tests.preservation_helpers import (
assert_frontend_runtime_matches_audited_baseline,
assert_moved_asset_matches,
assert_page_prefix_matches,
function_contract,
module_contract,
)
@@ -94,10 +97,22 @@ class SentimentPoolSliceSourceEquivalenceTests(unittest.TestCase):
def test_sentiment_engine_is_exact_original_with_legacy_alias(self) -> None:
self.assertEqual(
sha256(ORIGINAL_ROOT / "sentiment_engine.py"),
sha256(APP_ROOT / "backend" / "features" / "sentiment" / "engine.py"),
module_contract(
ORIGINAL_ROOT / "sentiment_engine.py",
excluded_definitions={"_number"},
),
module_contract(
APP_ROOT / "backend" / "features" / "sentiment" / "engine.py",
excluded_definitions={"_number"},
excluded_import_modules={"backend.data.numbers"},
),
)
self.assertEqual(
function_contract(ORIGINAL_ROOT / "sentiment_engine.py", "_number"),
function_contract(APP_ROOT / "backend/data/numbers.py", "non_nan_number"),
)
self.assertIs(sentiment_engine, canonical_engine)
self.assertIs(canonical_engine._number, non_nan_number)
def test_api_and_frontend_assets_are_unchanged(self) -> None:
self.assertEqual(
@@ -155,6 +155,10 @@ def build() -> dict[str, Any]:
{"provider": "eastmoney", "path": "backend/data/realtime.py", "runtime_role": "isolated realtime observation"},
{"provider": "tencent", "path": "backend/data/realtime.py", "runtime_role": "index observation fallback"},
],
"numeric_normalization": [
{"function": "finite_number", "path": "backend/data/numbers.py"},
{"function": "non_nan_number", "path": "backend/data/numbers.py"},
],
"llm_entrypoints": [
{"function": "stream_with_mentor", "path": "backend/features/mentor/agent.py"},
{"function": "interpret_heaven", "path": "backend/features/heaven/agent.py"},
+54
View File
@@ -21,6 +21,8 @@
| CR-01 | LLM供应商传输 | 问师、问天、复盘助手和策略编译各自构造HTTP请求、解析响应和读取错误 | 只保留`backend/llm/transport.py`一个网络出口 | 已完成 |
| CR-02 | HTTP精确POST委托 | 27个端点重复使用“比较路径、调用无参数处理器、返回”三行分支 | 用公开/受保护两张显式映射统一委托,同时保留复杂路由的原控制流 | 已完成 |
| CR-03 | 股票市场后缀转换 | Tushare业务与iFinD图表各保留一份完全相同的沪深京代码转换函数 | 图表复用`bootstrap/config.py::tushare_code`,只保留一份函数体 | 已完成 |
| CR-04 | 数值归一化策略 | 四个业务模块分别保留两组完全相同的数值转换函数体 | 由`backend/data/numbers.py`集中拥有两种既有语义,消费者保留原局部别名 | 已完成 |
| CR-05 | 根级兼容入口 | 正式后端仍有五处通过迁移兼容模块反向导入规范实现 | 正式代码改用规范路径;兼容入口只服务原公开导入契约 | 已完成 |
## CR-01验收口径
@@ -89,3 +91,55 @@
本批基线为`xiaobai-reduction-02-http-dispatch-20260801`;检查点为
`xiaobai-reduction-03-market-symbol-20260801`
## CR-04验收口径
- 只合并参数、函数体和运行结果完全一致的数值转换函数,不借本批改变任何业务计算或异常默认值。
- `finite_number`继续拒绝`NaN`与正负无穷;`non_nan_number`继续只拒绝`NaN`并保留正负无穷。
- Tushare与智能选股必须复用有限数策略;市场洞察与情绪引擎必须复用非NaN策略,并继续暴露原局部
`_number`名称以保持兼容。
- 实时行情和图表转换器的空值、默认值或参数签名语义不同,必须继续独立保留,不能因名称相同而合并。
- 原版函数参数和函数体分别与共享实现AST相等;四个消费者的局部别名必须指向对应的唯一函数对象。
## CR-04结果
- 删除Tushare、智能选股、市场洞察和情绪引擎中的四份重复函数体,新建两种明确命名的共享策略;生产
代码净减少约12行,全仓AST扫描不再发现完全相同的函数定义。
- 将可复用的函数及模块AST契约归入测试辅助层,原有迁移保持性测试改为“未改范围保持相等、被替换
函数与共享实现相等、运行时唯一对象”三重断言,没有降低门禁。
- 实时行情`backend/data/realtime.py::_number`与图表`backend/features/market/charts.py::_number`
被明确保留;它们不是本批重复实现,也未改变行为。
- 58项定向测试、41项保持性/治理测试、原版231项、候选320项、纯`app/`导出257项和45项
Playwright通过;24个JavaScript文件、API/架构注册表、Git空白检查和SQLite完整性检查通过。
- 本批不修改前端、CSS、接口、数据来源、行情口径、选股条件、数据库、LLM、权限或部署。
本批基线为`xiaobai-reduction-03-market-symbol-20260801`;检查点为
`xiaobai-reduction-04-numeric-normalization-20260801`
## CR-05验收口径
- 逐项扫描根级Python入口、生产代码、测试、工具和动态导入;没有消费者或兼容责任的入口才能删除。
- 规范后端不得经由`screener``advanced_strategies``tushare_client``server`兼容入口
间接访问已经归位的实现。
- 所有根级模块继续保持原导入名称、导出对象及模块对象身份,既有启动命令和第三方维护脚本不受影响。
- `api_access`、选股Repository的惰性`sentiment_engine`导入及根级`database.py`属于已登记边界,
分别留到HTTP、Repository阶段处理,不在本批跨边界修改。
## CR-05结果
- 审计确认21个根级兼容入口均有测试、工具、启动或原公开导入契约消费者,因此本批没有冒险删除文件。
- 容器、策略编译器、选股引擎及数据同步命令的五处导入改为规范模块路径,正式代码不再通过四个根级
兼容模块反向进入实现;运行代码行数未增加。
- 特性边界测试取消选股引擎旧例外,并新增全后端兼容导入门禁;只允许两项已登记过渡边界,后续代码
无法重新引入隐式根级依赖。
- 候选321项、纯`app/`导出258项、24个JavaScript文件、API/架构注册表、Git空白检查和SQLite
完整性检查通过;本批不涉及页面、CSS或浏览器行为。
- 本批不修改业务计算、策略公式、数据源、API、数据库、LLM、权限、前端或部署。
本批基线为`xiaobai-reduction-04-numeric-normalization-20260801`;检查点为
`xiaobai-reduction-05-compatibility-boundaries-20260801`
## 人工验收记录
- 2026-08-01:用户检查CR-02与CR-03运行结果,确认未发现明显异常。本记录仅表示本轮可见功能与
页面使用未发现明显回归,不替代后续批次各自的自动测试和人工抽查。