refactor: centralize numeric normalization

This commit is contained in:
leefer
2026-08-01 16:54:47 +08:00
parent 7ed181e682
commit 159a9a6a8b
15 changed files with 196 additions and 65 deletions
+1 -1
View File
@@ -36,7 +36,7 @@ background scheduler
- `backend/features/<feature>/` owns the mechanically moved service, repository, HTTP, agent, - `backend/features/<feature>/` owns the mechanically moved service, repository, HTTP, agent,
or deterministic calculation code for that product area. or deterministic calculation code for that product area.
- `backend/data/` owns provider construction, source policy, provenance, units, freshness, - `backend/data/` owns provider construction, source policy, provenance, units, freshness,
coverage, and display-versus-calculation eligibility. coverage, display-versus-calculation eligibility, and shared numeric normalization policies.
- `backend/database/` owns connection management, ordered migrations, and narrow repository - `backend/database/` owns connection management, ordered migrations, and narrow repository
adapters. Root `database.py` remains the legacy schema/composition anchor and combines the adapters. Root `database.py` remains the legacy schema/composition anchor and combines the
feature repository mixins; do not add feature queries to it. feature repository mixins; do not add feature queries to it.
+20
View File
@@ -0,0 +1,20 @@
from __future__ import annotations
import math
from typing import Any
def finite_number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if math.isfinite(number) else default
except (TypeError, ValueError):
return default
def non_nan_number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if number == number else default
except (TypeError, ValueError):
return default
+1 -8
View File
@@ -11,6 +11,7 @@ from datetime import datetime, time as dt_time, timedelta
from threading import Lock from threading import Lock
from typing import Any, ClassVar from typing import Any, ClassVar
from backend.data.numbers import finite_number as _number
from backend.features.sentiment.engine import apply_sentiment_to_dashboard from backend.features.sentiment.engine import apply_sentiment_to_dashboard
@@ -1812,14 +1813,6 @@ class TushareClient:
} }
def _number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if math.isfinite(number) else default
except (TypeError, ValueError):
return default
def _text(value: Any) -> str: def _text(value: Any) -> str:
if isinstance(value, (list, tuple, set)): if isinstance(value, (list, tuple, set)):
return "".join(str(item).strip() for item in value if str(item).strip()) return "".join(str(item).strip() for item in value if str(item).strip())
+1 -8
View File
@@ -6,6 +6,7 @@ from datetime import datetime, time as dt_time, timedelta, timezone
from statistics import median from statistics import median
from typing import TYPE_CHECKING, Any, Callable from typing import TYPE_CHECKING, Any, Callable
from backend.data.numbers import non_nan_number as _number
from backend.data.providers.ifind_client import IfindError, IfindHttpClient from backend.data.providers.ifind_client import IfindError, IfindHttpClient
from backend.data.providers.tushare_client import TushareClient, TushareError from backend.data.providers.tushare_client import TushareClient, TushareError
@@ -16,14 +17,6 @@ if TYPE_CHECKING:
CHINA_TIMEZONE = timezone(timedelta(hours=8)) CHINA_TIMEZONE = timezone(timedelta(hours=8))
def _number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if number == number else default
except (TypeError, ValueError):
return default
def _display_date(value: str) -> str: def _display_date(value: str) -> str:
text = str(value or "").replace("-", "") text = str(value or "").replace("-", "")
if len(text) != 8: if len(text) != 8:
+1 -8
View File
@@ -9,6 +9,7 @@ from datetime import datetime, timedelta
from typing import Any from typing import Any
from advanced_strategies import ADVANCED_CURATED_STRATEGIES from advanced_strategies import ADVANCED_CURATED_STRATEGIES
from backend.data.numbers import finite_number as _number
from database import ReviewDatabase from database import ReviewDatabase
from backend.features.sentiment.engine import build_sentiment_history, latest_contiguous_history from backend.features.sentiment.engine import build_sentiment_history, latest_contiguous_history
from tushare_client import TushareClient, TushareError from tushare_client import TushareClient, TushareError
@@ -2201,13 +2202,5 @@ def _regime_reason(regime: str) -> str:
}.get(regime, "市场阶段待确认。") }.get(regime, "市场阶段待确认。")
def _number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if math.isfinite(number) else default
except (TypeError, ValueError):
return default
def _display_date(value: str) -> str: def _display_date(value: str) -> str:
return f"{value[:4]}-{value[4:6]}-{value[6:8]}" if len(value) == 8 else value return f"{value[:4]}-{value[4:6]}-{value[6:8]}" if len(value) == 8 else value
+2 -8
View File
@@ -4,6 +4,8 @@ from copy import deepcopy
from statistics import mean, median from statistics import mean, median
from typing import Any from typing import Any
from backend.data.numbers import non_nan_number as _number
COMPONENT_WEIGHTS = { COMPONENT_WEIGHTS = {
"breadth": 20, "breadth": 20,
@@ -16,14 +18,6 @@ COMPONENT_WEIGHTS = {
SENTIMENT_ENGINE_VERSION = 2 SENTIMENT_ENGINE_VERSION = 2
def _number(value: Any, default: float = 0.0) -> float:
try:
number = float(value)
return number if number == number else default
except (TypeError, ValueError):
return default
def _clamp(value: float, lower: float = 0.0, upper: float = 100.0) -> float: def _clamp(value: float, lower: float = 0.0, upper: float = 100.0) -> float:
return min(upper, max(lower, value)) return min(upper, max(lower, value))
+16 -6
View File
@@ -220,6 +220,16 @@
"runtime_role": "index observation fallback" "runtime_role": "index observation fallback"
} }
], ],
"numeric_normalization": [
{
"function": "finite_number",
"path": "backend/data/numbers.py"
},
{
"function": "non_nan_number",
"path": "backend/data/numbers.py"
}
],
"llm_entrypoints": [ "llm_entrypoints": [
{ {
"function": "stream_with_mentor", "function": "stream_with_mentor",
@@ -279,13 +289,13 @@
}, },
{ {
"path": "backend/features/screener/engine.py", "path": "backend/features/screener/engine.py",
"bytes": 108552, "bytes": 108394,
"lines": 2213 "lines": 2206
}, },
{ {
"path": "backend/data/providers/tushare_client.py", "path": "backend/data/providers/tushare_client.py",
"bytes": 94329, "bytes": 94171,
"lines": 2175 "lines": 2168
}, },
{ {
"path": "frontend/app.js", "path": "frontend/app.js",
@@ -314,8 +324,8 @@
}, },
{ {
"path": "backend/features/market/insights.py", "path": "backend/features/market/insights.py",
"bytes": 58150, "bytes": 57998,
"lines": 1314 "lines": 1307
}, },
{ {
"path": "frontend/pages/market/runtime.js", "path": "frontend/pages/market/runtime.js",
+36
View File
@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import ast
import hashlib import hashlib
import re import re
from pathlib import Path from pathlib import Path
@@ -33,6 +34,41 @@ def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest() return hashlib.sha256(path.read_bytes()).hexdigest()
def function_contract(path: Path, name: str) -> tuple[str, str]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
function = next(
node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name
)
body = ast.Module(body=function.body, type_ignores=[])
return (
ast.dump(function.args, include_attributes=False),
ast.dump(body, include_attributes=False),
)
def module_contract(
path: Path,
*,
excluded_definitions: set[str] | None = None,
excluded_import_modules: set[str] | None = None,
exclude_imports: bool = False,
) -> str:
excluded_definitions = excluded_definitions or set()
excluded_import_modules = excluded_import_modules or set()
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
tree.body = [
node
for node in tree.body
if not (exclude_imports and isinstance(node, (ast.Import, ast.ImportFrom)))
and not (
isinstance(node, ast.ImportFrom)
and node.module in excluded_import_modules
)
and getattr(node, "name", None) not in excluded_definitions
]
return ast.dump(tree, include_attributes=False)
def reassembled_frontend_runtime() -> str: def reassembled_frontend_runtime() -> str:
chunks: dict[tuple[int, int], str] = {} chunks: dict[tuple[int, int], str] = {}
for path in FRONTEND_ROOT.rglob("*.js"): for path in FRONTEND_ROOT.rglob("*.js"):
+38
View File
@@ -0,0 +1,38 @@
from __future__ import annotations
import math
import unittest
from backend.data.numbers import finite_number, non_nan_number
from backend.data.providers import tushare_client
from backend.features.market import insights
from backend.features.screener import engine as screener_engine
from backend.features.sentiment import engine as sentiment_engine
class NumericNormalizationTests(unittest.TestCase):
def test_consumers_use_their_declared_shared_policy(self) -> None:
self.assertIs(tushare_client._number, finite_number)
self.assertIs(screener_engine._number, finite_number)
self.assertIs(insights._number, non_nan_number)
self.assertIs(sentiment_engine._number, non_nan_number)
def test_finite_policy_preserves_existing_results(self) -> None:
self.assertEqual(finite_number("12.5"), 12.5)
self.assertEqual(finite_number(None), 0.0)
self.assertEqual(finite_number("invalid", 7.0), 7.0)
self.assertEqual(finite_number(math.nan, 7.0), 7.0)
self.assertEqual(finite_number(math.inf, 7.0), 7.0)
self.assertEqual(finite_number(-math.inf, 7.0), 7.0)
def test_non_nan_policy_keeps_infinity_but_rejects_nan(self) -> None:
self.assertEqual(non_nan_number("12.5"), 12.5)
self.assertEqual(non_nan_number(None), 0.0)
self.assertEqual(non_nan_number("invalid", 7.0), 7.0)
self.assertEqual(non_nan_number(math.nan, 7.0), 7.0)
self.assertEqual(non_nan_number(math.inf, 7.0), math.inf)
self.assertEqual(non_nan_number(-math.inf, 7.0), -math.inf)
if __name__ == "__main__":
unittest.main()
+10 -13
View File
@@ -11,12 +11,14 @@ import realtime_aggregator
import tushare_client import tushare_client
from backend.bootstrap import config as bootstrap_config from backend.bootstrap import config as bootstrap_config
from backend.data import realtime from backend.data import realtime
from backend.data.numbers import finite_number
from backend.data.providers import ifind_client as canonical_ifind from backend.data.providers import ifind_client as canonical_ifind
from backend.data.providers import tushare_client as canonical_tushare from backend.data.providers import tushare_client as canonical_tushare
from backend.features.market import charts from backend.features.market import charts
from tests.preservation_helpers import ( from tests.preservation_helpers import (
assert_frontend_runtime_matches_audited_baseline, assert_frontend_runtime_matches_audited_baseline,
assert_moved_asset_matches, assert_moved_asset_matches,
function_contract,
) )
@@ -102,18 +104,6 @@ def top_level_definitions(path: Path) -> dict[str, str]:
} }
def function_contract(path: Path, name: str) -> tuple[str, str]:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
function = next(
node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name
)
body = ast.Module(body=function.body, type_ignores=[])
return (
ast.dump(function.args, include_attributes=False),
ast.dump(body, include_attributes=False),
)
class MarketSliceSourceEquivalenceTests(unittest.TestCase): class MarketSliceSourceEquivalenceTests(unittest.TestCase):
def test_market_service_methods_are_exact_original_ast(self) -> None: def test_market_service_methods_are_exact_original_ast(self) -> None:
original = class_methods(ORIGINAL_ROOT / "server.py", "DashboardService") original = class_methods(ORIGINAL_ROOT / "server.py", "DashboardService")
@@ -154,10 +144,17 @@ class MarketSliceSourceEquivalenceTests(unittest.TestCase):
) )
for original, migrated in exact_moves: for original, migrated in exact_moves:
self.assertEqual(sha256(ORIGINAL_ROOT / original), sha256(APP_ROOT / migrated)) self.assertEqual(sha256(ORIGINAL_ROOT / original), sha256(APP_ROOT / migrated))
original_tushare = top_level_definitions(ORIGINAL_ROOT / "tushare_client.py")
original_tushare.pop("_number")
self.assertEqual( self.assertEqual(
top_level_definitions(ORIGINAL_ROOT / "tushare_client.py"), original_tushare,
top_level_definitions(APP_ROOT / "backend/data/providers/tushare_client.py"), top_level_definitions(APP_ROOT / "backend/data/providers/tushare_client.py"),
) )
self.assertEqual(
function_contract(ORIGINAL_ROOT / "tushare_client.py", "_number"),
function_contract(APP_ROOT / "backend/data/numbers.py", "finite_number"),
)
self.assertIs(canonical_tushare._number, finite_number)
original_charts = top_level_definitions(ORIGINAL_ROOT / "chart_data_provider.py") original_charts = top_level_definitions(ORIGINAL_ROOT / "chart_data_provider.py")
original_charts.pop("_stock_market_code") original_charts.pop("_stock_market_code")
self.assertEqual( self.assertEqual(
@@ -6,11 +6,13 @@ import unittest
from pathlib import Path from pathlib import Path
import market_insights import market_insights
from backend.data.numbers import non_nan_number
from backend.features.market import insights as canonical_insights from backend.features.market import insights as canonical_insights
from tests.preservation_helpers import ( from tests.preservation_helpers import (
assert_frontend_runtime_matches_audited_baseline, assert_frontend_runtime_matches_audited_baseline,
assert_moved_asset_matches, assert_moved_asset_matches,
assert_page_prefix_matches, assert_page_prefix_matches,
function_contract,
) )
@@ -106,6 +108,11 @@ class MarketInsightsSliceSourceEquivalenceTests(unittest.TestCase):
MARKET_INSIGHT_METHODS, MARKET_INSIGHT_METHODS,
) )
self.assertIs(market_insights.MarketInsightsService, canonical_insights.MarketInsightsService) self.assertIs(market_insights.MarketInsightsService, canonical_insights.MarketInsightsService)
self.assertEqual(
function_contract(ORIGINAL_ROOT / "market_insights.py", "_number"),
function_contract(APP_ROOT / "backend/data/numbers.py", "non_nan_number"),
)
self.assertIs(canonical_insights._number, non_nan_number)
def test_dashboard_service_methods_are_exact_original_ast(self) -> None: def test_dashboard_service_methods_are_exact_original_ast(self) -> None:
original = ORIGINAL_ROOT / "server.py" original = ORIGINAL_ROOT / "server.py"
+17 -11
View File
@@ -9,12 +9,15 @@ import advanced_strategies
import llm_strategy import llm_strategy
import screener import screener
import strategy_tracking import strategy_tracking
from backend.data.numbers import finite_number
from backend.features.screener import compiler, engine, strategies, tracking from backend.features.screener import compiler, engine, strategies, tracking
from backend.features.screener import service as screener_service from backend.features.screener import service as screener_service
from tests.preservation_helpers import ( from tests.preservation_helpers import (
assert_frontend_runtime_matches_audited_baseline, assert_frontend_runtime_matches_audited_baseline,
assert_moved_asset_matches, assert_moved_asset_matches,
assert_page_prefix_matches, assert_page_prefix_matches,
function_contract,
module_contract,
) )
@@ -91,14 +94,6 @@ def top_level_definition(path: Path, name: str) -> str:
return ast.dump(node, include_attributes=False) return ast.dump(node, include_attributes=False)
def module_without_imports(path: Path) -> str:
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
tree.body = [
node for node in tree.body if not isinstance(node, (ast.Import, ast.ImportFrom))
]
return ast.dump(tree, include_attributes=False)
def sha256(path: Path) -> str: def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest() return hashlib.sha256(path.read_bytes()).hexdigest()
@@ -154,11 +149,22 @@ class ScreenerSliceSourceEquivalenceTests(unittest.TestCase):
def test_engine_and_tracking_logic_match_the_original(self) -> None: def test_engine_and_tracking_logic_match_the_original(self) -> None:
self.assertEqual( self.assertEqual(
module_without_imports(ORIGINAL_ROOT / "screener.py"), module_contract(
module_without_imports( ORIGINAL_ROOT / "screener.py",
APP_ROOT / "backend" / "features" / "screener" / "engine.py" excluded_definitions={"_number"},
exclude_imports=True,
),
module_contract(
APP_ROOT / "backend" / "features" / "screener" / "engine.py",
excluded_definitions={"_number"},
exclude_imports=True,
), ),
) )
self.assertEqual(
function_contract(ORIGINAL_ROOT / "screener.py", "_number"),
function_contract(APP_ROOT / "backend/data/numbers.py", "finite_number"),
)
self.assertIs(engine._number, finite_number)
self.assertEqual( self.assertEqual(
class_methods( class_methods(
ORIGINAL_ROOT / "backend" / "features" / "screener" / "tracking.py", ORIGINAL_ROOT / "backend" / "features" / "screener" / "tracking.py",
@@ -6,11 +6,14 @@ import unittest
from pathlib import Path from pathlib import Path
import sentiment_engine import sentiment_engine
from backend.data.numbers import non_nan_number
from backend.features.sentiment import engine as canonical_engine from backend.features.sentiment import engine as canonical_engine
from tests.preservation_helpers import ( from tests.preservation_helpers import (
assert_frontend_runtime_matches_audited_baseline, assert_frontend_runtime_matches_audited_baseline,
assert_moved_asset_matches, assert_moved_asset_matches,
assert_page_prefix_matches, assert_page_prefix_matches,
function_contract,
module_contract,
) )
@@ -94,10 +97,22 @@ class SentimentPoolSliceSourceEquivalenceTests(unittest.TestCase):
def test_sentiment_engine_is_exact_original_with_legacy_alias(self) -> None: def test_sentiment_engine_is_exact_original_with_legacy_alias(self) -> None:
self.assertEqual( self.assertEqual(
sha256(ORIGINAL_ROOT / "sentiment_engine.py"), module_contract(
sha256(APP_ROOT / "backend" / "features" / "sentiment" / "engine.py"), ORIGINAL_ROOT / "sentiment_engine.py",
excluded_definitions={"_number"},
),
module_contract(
APP_ROOT / "backend" / "features" / "sentiment" / "engine.py",
excluded_definitions={"_number"},
excluded_import_modules={"backend.data.numbers"},
),
)
self.assertEqual(
function_contract(ORIGINAL_ROOT / "sentiment_engine.py", "_number"),
function_contract(APP_ROOT / "backend/data/numbers.py", "non_nan_number"),
) )
self.assertIs(sentiment_engine, canonical_engine) self.assertIs(sentiment_engine, canonical_engine)
self.assertIs(canonical_engine._number, non_nan_number)
def test_api_and_frontend_assets_are_unchanged(self) -> None: def test_api_and_frontend_assets_are_unchanged(self) -> None:
self.assertEqual( self.assertEqual(
@@ -155,6 +155,10 @@ def build() -> dict[str, Any]:
{"provider": "eastmoney", "path": "backend/data/realtime.py", "runtime_role": "isolated realtime observation"}, {"provider": "eastmoney", "path": "backend/data/realtime.py", "runtime_role": "isolated realtime observation"},
{"provider": "tencent", "path": "backend/data/realtime.py", "runtime_role": "index observation fallback"}, {"provider": "tencent", "path": "backend/data/realtime.py", "runtime_role": "index observation fallback"},
], ],
"numeric_normalization": [
{"function": "finite_number", "path": "backend/data/numbers.py"},
{"function": "non_nan_number", "path": "backend/data/numbers.py"},
],
"llm_entrypoints": [ "llm_entrypoints": [
{"function": "stream_with_mentor", "path": "backend/features/mentor/agent.py"}, {"function": "stream_with_mentor", "path": "backend/features/mentor/agent.py"},
{"function": "interpret_heaven", "path": "backend/features/heaven/agent.py"}, {"function": "interpret_heaven", "path": "backend/features/heaven/agent.py"},
+25
View File
@@ -21,6 +21,7 @@
| CR-01 | LLM供应商传输 | 问师、问天、复盘助手和策略编译各自构造HTTP请求、解析响应和读取错误 | 只保留`backend/llm/transport.py`一个网络出口 | 已完成 | | CR-01 | LLM供应商传输 | 问师、问天、复盘助手和策略编译各自构造HTTP请求、解析响应和读取错误 | 只保留`backend/llm/transport.py`一个网络出口 | 已完成 |
| CR-02 | HTTP精确POST委托 | 27个端点重复使用“比较路径、调用无参数处理器、返回”三行分支 | 用公开/受保护两张显式映射统一委托,同时保留复杂路由的原控制流 | 已完成 | | CR-02 | HTTP精确POST委托 | 27个端点重复使用“比较路径、调用无参数处理器、返回”三行分支 | 用公开/受保护两张显式映射统一委托,同时保留复杂路由的原控制流 | 已完成 |
| CR-03 | 股票市场后缀转换 | Tushare业务与iFinD图表各保留一份完全相同的沪深京代码转换函数 | 图表复用`bootstrap/config.py::tushare_code`,只保留一份函数体 | 已完成 | | CR-03 | 股票市场后缀转换 | Tushare业务与iFinD图表各保留一份完全相同的沪深京代码转换函数 | 图表复用`bootstrap/config.py::tushare_code`,只保留一份函数体 | 已完成 |
| CR-04 | 数值归一化策略 | 四个业务模块分别保留两组完全相同的数值转换函数体 | 由`backend/data/numbers.py`集中拥有两种既有语义,消费者保留原局部别名 | 已完成 |
## CR-01验收口径 ## CR-01验收口径
@@ -90,6 +91,30 @@
本批基线为`xiaobai-reduction-02-http-dispatch-20260801`;检查点为 本批基线为`xiaobai-reduction-02-http-dispatch-20260801`;检查点为
`xiaobai-reduction-03-market-symbol-20260801` `xiaobai-reduction-03-market-symbol-20260801`
## CR-04验收口径
- 只合并参数、函数体和运行结果完全一致的数值转换函数,不借本批改变任何业务计算或异常默认值。
- `finite_number`继续拒绝`NaN`与正负无穷;`non_nan_number`继续只拒绝`NaN`并保留正负无穷。
- Tushare与智能选股必须复用有限数策略;市场洞察与情绪引擎必须复用非NaN策略,并继续暴露原局部
`_number`名称以保持兼容。
- 实时行情和图表转换器的空值、默认值或参数签名语义不同,必须继续独立保留,不能因名称相同而合并。
- 原版函数参数和函数体分别与共享实现AST相等;四个消费者的局部别名必须指向对应的唯一函数对象。
## CR-04结果
- 删除Tushare、智能选股、市场洞察和情绪引擎中的四份重复函数体,新建两种明确命名的共享策略;生产
代码净减少约12行,全仓AST扫描不再发现完全相同的函数定义。
- 将可复用的函数及模块AST契约归入测试辅助层,原有迁移保持性测试改为“未改范围保持相等、被替换
函数与共享实现相等、运行时唯一对象”三重断言,没有降低门禁。
- 实时行情`backend/data/realtime.py::_number`与图表`backend/features/market/charts.py::_number`
被明确保留;它们不是本批重复实现,也未改变行为。
- 58项定向测试、41项保持性/治理测试、原版231项、候选320项、纯`app/`导出257项和45项
Playwright通过;24个JavaScript文件、API/架构注册表、Git空白检查和SQLite完整性检查通过。
- 本批不修改前端、CSS、接口、数据来源、行情口径、选股条件、数据库、LLM、权限或部署。
本批基线为`xiaobai-reduction-03-market-symbol-20260801`;检查点为
`xiaobai-reduction-04-numeric-normalization-20260801`
## 人工验收记录 ## 人工验收记录
- 2026-08-01:用户检查CR-02与CR-03运行结果,确认未发现明显异常。本记录仅表示本轮可见功能与 - 2026-08-01:用户检查CR-02与CR-03运行结果,确认未发现明显异常。本记录仅表示本轮可见功能与