refactor: centralize numeric normalization
This commit is contained in:
+1
-1
@@ -36,7 +36,7 @@ background scheduler
|
||||
- `backend/features/<feature>/` owns the mechanically moved service, repository, HTTP, agent,
|
||||
or deterministic calculation code for that product area.
|
||||
- `backend/data/` owns provider construction, source policy, provenance, units, freshness,
|
||||
coverage, and display-versus-calculation eligibility.
|
||||
coverage, display-versus-calculation eligibility, and shared numeric normalization policies.
|
||||
- `backend/database/` owns connection management, ordered migrations, and narrow repository
|
||||
adapters. Root `database.py` remains the legacy schema/composition anchor and combines the
|
||||
feature repository mixins; do not add feature queries to it.
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
|
||||
def finite_number(value: Any, default: float = 0.0) -> float:
|
||||
try:
|
||||
number = float(value)
|
||||
return number if math.isfinite(number) else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def non_nan_number(value: Any, default: float = 0.0) -> float:
|
||||
try:
|
||||
number = float(value)
|
||||
return number if number == number else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
@@ -11,6 +11,7 @@ from datetime import datetime, time as dt_time, timedelta
|
||||
from threading import Lock
|
||||
from typing import Any, ClassVar
|
||||
|
||||
from backend.data.numbers import finite_number as _number
|
||||
from backend.features.sentiment.engine import apply_sentiment_to_dashboard
|
||||
|
||||
|
||||
@@ -1812,14 +1813,6 @@ class TushareClient:
|
||||
}
|
||||
|
||||
|
||||
def _number(value: Any, default: float = 0.0) -> float:
|
||||
try:
|
||||
number = float(value)
|
||||
return number if math.isfinite(number) else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return "、".join(str(item).strip() for item in value if str(item).strip())
|
||||
|
||||
@@ -6,6 +6,7 @@ from datetime import datetime, time as dt_time, timedelta, timezone
|
||||
from statistics import median
|
||||
from typing import TYPE_CHECKING, Any, Callable
|
||||
|
||||
from backend.data.numbers import non_nan_number as _number
|
||||
from backend.data.providers.ifind_client import IfindError, IfindHttpClient
|
||||
from backend.data.providers.tushare_client import TushareClient, TushareError
|
||||
|
||||
@@ -16,14 +17,6 @@ if TYPE_CHECKING:
|
||||
CHINA_TIMEZONE = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
def _number(value: Any, default: float = 0.0) -> float:
|
||||
try:
|
||||
number = float(value)
|
||||
return number if number == number else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _display_date(value: str) -> str:
|
||||
text = str(value or "").replace("-", "")
|
||||
if len(text) != 8:
|
||||
|
||||
@@ -9,6 +9,7 @@ from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from advanced_strategies import ADVANCED_CURATED_STRATEGIES
|
||||
from backend.data.numbers import finite_number as _number
|
||||
from database import ReviewDatabase
|
||||
from backend.features.sentiment.engine import build_sentiment_history, latest_contiguous_history
|
||||
from tushare_client import TushareClient, TushareError
|
||||
@@ -2201,13 +2202,5 @@ def _regime_reason(regime: str) -> str:
|
||||
}.get(regime, "市场阶段待确认。")
|
||||
|
||||
|
||||
def _number(value: Any, default: float = 0.0) -> float:
|
||||
try:
|
||||
number = float(value)
|
||||
return number if math.isfinite(number) else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _display_date(value: str) -> str:
|
||||
return f"{value[:4]}-{value[4:6]}-{value[6:8]}" if len(value) == 8 else value
|
||||
|
||||
@@ -4,6 +4,8 @@ from copy import deepcopy
|
||||
from statistics import mean, median
|
||||
from typing import Any
|
||||
|
||||
from backend.data.numbers import non_nan_number as _number
|
||||
|
||||
|
||||
COMPONENT_WEIGHTS = {
|
||||
"breadth": 20,
|
||||
@@ -16,14 +18,6 @@ COMPONENT_WEIGHTS = {
|
||||
SENTIMENT_ENGINE_VERSION = 2
|
||||
|
||||
|
||||
def _number(value: Any, default: float = 0.0) -> float:
|
||||
try:
|
||||
number = float(value)
|
||||
return number if number == number else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _clamp(value: float, lower: float = 0.0, upper: float = 100.0) -> float:
|
||||
return min(upper, max(lower, value))
|
||||
|
||||
|
||||
@@ -220,6 +220,16 @@
|
||||
"runtime_role": "index observation fallback"
|
||||
}
|
||||
],
|
||||
"numeric_normalization": [
|
||||
{
|
||||
"function": "finite_number",
|
||||
"path": "backend/data/numbers.py"
|
||||
},
|
||||
{
|
||||
"function": "non_nan_number",
|
||||
"path": "backend/data/numbers.py"
|
||||
}
|
||||
],
|
||||
"llm_entrypoints": [
|
||||
{
|
||||
"function": "stream_with_mentor",
|
||||
@@ -279,13 +289,13 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/features/screener/engine.py",
|
||||
"bytes": 108552,
|
||||
"lines": 2213
|
||||
"bytes": 108394,
|
||||
"lines": 2206
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_client.py",
|
||||
"bytes": 94329,
|
||||
"lines": 2175
|
||||
"bytes": 94171,
|
||||
"lines": 2168
|
||||
},
|
||||
{
|
||||
"path": "frontend/app.js",
|
||||
@@ -314,8 +324,8 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/insights.py",
|
||||
"bytes": 58150,
|
||||
"lines": 1314
|
||||
"bytes": 57998,
|
||||
"lines": 1307
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/market/runtime.js",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import hashlib
|
||||
import re
|
||||
from pathlib import Path
|
||||
@@ -33,6 +34,41 @@ def sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def function_contract(path: Path, name: str) -> tuple[str, str]:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
function = next(
|
||||
node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name
|
||||
)
|
||||
body = ast.Module(body=function.body, type_ignores=[])
|
||||
return (
|
||||
ast.dump(function.args, include_attributes=False),
|
||||
ast.dump(body, include_attributes=False),
|
||||
)
|
||||
|
||||
|
||||
def module_contract(
|
||||
path: Path,
|
||||
*,
|
||||
excluded_definitions: set[str] | None = None,
|
||||
excluded_import_modules: set[str] | None = None,
|
||||
exclude_imports: bool = False,
|
||||
) -> str:
|
||||
excluded_definitions = excluded_definitions or set()
|
||||
excluded_import_modules = excluded_import_modules or set()
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
tree.body = [
|
||||
node
|
||||
for node in tree.body
|
||||
if not (exclude_imports and isinstance(node, (ast.Import, ast.ImportFrom)))
|
||||
and not (
|
||||
isinstance(node, ast.ImportFrom)
|
||||
and node.module in excluded_import_modules
|
||||
)
|
||||
and getattr(node, "name", None) not in excluded_definitions
|
||||
]
|
||||
return ast.dump(tree, include_attributes=False)
|
||||
|
||||
|
||||
def reassembled_frontend_runtime() -> str:
|
||||
chunks: dict[tuple[int, int], str] = {}
|
||||
for path in FRONTEND_ROOT.rglob("*.js"):
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import unittest
|
||||
|
||||
from backend.data.numbers import finite_number, non_nan_number
|
||||
from backend.data.providers import tushare_client
|
||||
from backend.features.market import insights
|
||||
from backend.features.screener import engine as screener_engine
|
||||
from backend.features.sentiment import engine as sentiment_engine
|
||||
|
||||
|
||||
class NumericNormalizationTests(unittest.TestCase):
|
||||
def test_consumers_use_their_declared_shared_policy(self) -> None:
|
||||
self.assertIs(tushare_client._number, finite_number)
|
||||
self.assertIs(screener_engine._number, finite_number)
|
||||
self.assertIs(insights._number, non_nan_number)
|
||||
self.assertIs(sentiment_engine._number, non_nan_number)
|
||||
|
||||
def test_finite_policy_preserves_existing_results(self) -> None:
|
||||
self.assertEqual(finite_number("12.5"), 12.5)
|
||||
self.assertEqual(finite_number(None), 0.0)
|
||||
self.assertEqual(finite_number("invalid", 7.0), 7.0)
|
||||
self.assertEqual(finite_number(math.nan, 7.0), 7.0)
|
||||
self.assertEqual(finite_number(math.inf, 7.0), 7.0)
|
||||
self.assertEqual(finite_number(-math.inf, 7.0), 7.0)
|
||||
|
||||
def test_non_nan_policy_keeps_infinity_but_rejects_nan(self) -> None:
|
||||
self.assertEqual(non_nan_number("12.5"), 12.5)
|
||||
self.assertEqual(non_nan_number(None), 0.0)
|
||||
self.assertEqual(non_nan_number("invalid", 7.0), 7.0)
|
||||
self.assertEqual(non_nan_number(math.nan, 7.0), 7.0)
|
||||
self.assertEqual(non_nan_number(math.inf, 7.0), math.inf)
|
||||
self.assertEqual(non_nan_number(-math.inf, 7.0), -math.inf)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -11,12 +11,14 @@ import realtime_aggregator
|
||||
import tushare_client
|
||||
from backend.bootstrap import config as bootstrap_config
|
||||
from backend.data import realtime
|
||||
from backend.data.numbers import finite_number
|
||||
from backend.data.providers import ifind_client as canonical_ifind
|
||||
from backend.data.providers import tushare_client as canonical_tushare
|
||||
from backend.features.market import charts
|
||||
from tests.preservation_helpers import (
|
||||
assert_frontend_runtime_matches_audited_baseline,
|
||||
assert_moved_asset_matches,
|
||||
function_contract,
|
||||
)
|
||||
|
||||
|
||||
@@ -102,18 +104,6 @@ def top_level_definitions(path: Path) -> dict[str, str]:
|
||||
}
|
||||
|
||||
|
||||
def function_contract(path: Path, name: str) -> tuple[str, str]:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
function = next(
|
||||
node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name
|
||||
)
|
||||
body = ast.Module(body=function.body, type_ignores=[])
|
||||
return (
|
||||
ast.dump(function.args, include_attributes=False),
|
||||
ast.dump(body, include_attributes=False),
|
||||
)
|
||||
|
||||
|
||||
class MarketSliceSourceEquivalenceTests(unittest.TestCase):
|
||||
def test_market_service_methods_are_exact_original_ast(self) -> None:
|
||||
original = class_methods(ORIGINAL_ROOT / "server.py", "DashboardService")
|
||||
@@ -154,10 +144,17 @@ class MarketSliceSourceEquivalenceTests(unittest.TestCase):
|
||||
)
|
||||
for original, migrated in exact_moves:
|
||||
self.assertEqual(sha256(ORIGINAL_ROOT / original), sha256(APP_ROOT / migrated))
|
||||
original_tushare = top_level_definitions(ORIGINAL_ROOT / "tushare_client.py")
|
||||
original_tushare.pop("_number")
|
||||
self.assertEqual(
|
||||
top_level_definitions(ORIGINAL_ROOT / "tushare_client.py"),
|
||||
original_tushare,
|
||||
top_level_definitions(APP_ROOT / "backend/data/providers/tushare_client.py"),
|
||||
)
|
||||
self.assertEqual(
|
||||
function_contract(ORIGINAL_ROOT / "tushare_client.py", "_number"),
|
||||
function_contract(APP_ROOT / "backend/data/numbers.py", "finite_number"),
|
||||
)
|
||||
self.assertIs(canonical_tushare._number, finite_number)
|
||||
original_charts = top_level_definitions(ORIGINAL_ROOT / "chart_data_provider.py")
|
||||
original_charts.pop("_stock_market_code")
|
||||
self.assertEqual(
|
||||
|
||||
@@ -6,11 +6,13 @@ import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import market_insights
|
||||
from backend.data.numbers import non_nan_number
|
||||
from backend.features.market import insights as canonical_insights
|
||||
from tests.preservation_helpers import (
|
||||
assert_frontend_runtime_matches_audited_baseline,
|
||||
assert_moved_asset_matches,
|
||||
assert_page_prefix_matches,
|
||||
function_contract,
|
||||
)
|
||||
|
||||
|
||||
@@ -106,6 +108,11 @@ class MarketInsightsSliceSourceEquivalenceTests(unittest.TestCase):
|
||||
MARKET_INSIGHT_METHODS,
|
||||
)
|
||||
self.assertIs(market_insights.MarketInsightsService, canonical_insights.MarketInsightsService)
|
||||
self.assertEqual(
|
||||
function_contract(ORIGINAL_ROOT / "market_insights.py", "_number"),
|
||||
function_contract(APP_ROOT / "backend/data/numbers.py", "non_nan_number"),
|
||||
)
|
||||
self.assertIs(canonical_insights._number, non_nan_number)
|
||||
|
||||
def test_dashboard_service_methods_are_exact_original_ast(self) -> None:
|
||||
original = ORIGINAL_ROOT / "server.py"
|
||||
|
||||
@@ -9,12 +9,15 @@ import advanced_strategies
|
||||
import llm_strategy
|
||||
import screener
|
||||
import strategy_tracking
|
||||
from backend.data.numbers import finite_number
|
||||
from backend.features.screener import compiler, engine, strategies, tracking
|
||||
from backend.features.screener import service as screener_service
|
||||
from tests.preservation_helpers import (
|
||||
assert_frontend_runtime_matches_audited_baseline,
|
||||
assert_moved_asset_matches,
|
||||
assert_page_prefix_matches,
|
||||
function_contract,
|
||||
module_contract,
|
||||
)
|
||||
|
||||
|
||||
@@ -91,14 +94,6 @@ def top_level_definition(path: Path, name: str) -> str:
|
||||
return ast.dump(node, include_attributes=False)
|
||||
|
||||
|
||||
def module_without_imports(path: Path) -> str:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
tree.body = [
|
||||
node for node in tree.body if not isinstance(node, (ast.Import, ast.ImportFrom))
|
||||
]
|
||||
return ast.dump(tree, include_attributes=False)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
@@ -154,11 +149,22 @@ class ScreenerSliceSourceEquivalenceTests(unittest.TestCase):
|
||||
|
||||
def test_engine_and_tracking_logic_match_the_original(self) -> None:
|
||||
self.assertEqual(
|
||||
module_without_imports(ORIGINAL_ROOT / "screener.py"),
|
||||
module_without_imports(
|
||||
APP_ROOT / "backend" / "features" / "screener" / "engine.py"
|
||||
module_contract(
|
||||
ORIGINAL_ROOT / "screener.py",
|
||||
excluded_definitions={"_number"},
|
||||
exclude_imports=True,
|
||||
),
|
||||
module_contract(
|
||||
APP_ROOT / "backend" / "features" / "screener" / "engine.py",
|
||||
excluded_definitions={"_number"},
|
||||
exclude_imports=True,
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
function_contract(ORIGINAL_ROOT / "screener.py", "_number"),
|
||||
function_contract(APP_ROOT / "backend/data/numbers.py", "finite_number"),
|
||||
)
|
||||
self.assertIs(engine._number, finite_number)
|
||||
self.assertEqual(
|
||||
class_methods(
|
||||
ORIGINAL_ROOT / "backend" / "features" / "screener" / "tracking.py",
|
||||
|
||||
@@ -6,11 +6,14 @@ import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import sentiment_engine
|
||||
from backend.data.numbers import non_nan_number
|
||||
from backend.features.sentiment import engine as canonical_engine
|
||||
from tests.preservation_helpers import (
|
||||
assert_frontend_runtime_matches_audited_baseline,
|
||||
assert_moved_asset_matches,
|
||||
assert_page_prefix_matches,
|
||||
function_contract,
|
||||
module_contract,
|
||||
)
|
||||
|
||||
|
||||
@@ -94,10 +97,22 @@ class SentimentPoolSliceSourceEquivalenceTests(unittest.TestCase):
|
||||
|
||||
def test_sentiment_engine_is_exact_original_with_legacy_alias(self) -> None:
|
||||
self.assertEqual(
|
||||
sha256(ORIGINAL_ROOT / "sentiment_engine.py"),
|
||||
sha256(APP_ROOT / "backend" / "features" / "sentiment" / "engine.py"),
|
||||
module_contract(
|
||||
ORIGINAL_ROOT / "sentiment_engine.py",
|
||||
excluded_definitions={"_number"},
|
||||
),
|
||||
module_contract(
|
||||
APP_ROOT / "backend" / "features" / "sentiment" / "engine.py",
|
||||
excluded_definitions={"_number"},
|
||||
excluded_import_modules={"backend.data.numbers"},
|
||||
),
|
||||
)
|
||||
self.assertEqual(
|
||||
function_contract(ORIGINAL_ROOT / "sentiment_engine.py", "_number"),
|
||||
function_contract(APP_ROOT / "backend/data/numbers.py", "non_nan_number"),
|
||||
)
|
||||
self.assertIs(sentiment_engine, canonical_engine)
|
||||
self.assertIs(canonical_engine._number, non_nan_number)
|
||||
|
||||
def test_api_and_frontend_assets_are_unchanged(self) -> None:
|
||||
self.assertEqual(
|
||||
|
||||
@@ -155,6 +155,10 @@ def build() -> dict[str, Any]:
|
||||
{"provider": "eastmoney", "path": "backend/data/realtime.py", "runtime_role": "isolated realtime observation"},
|
||||
{"provider": "tencent", "path": "backend/data/realtime.py", "runtime_role": "index observation fallback"},
|
||||
],
|
||||
"numeric_normalization": [
|
||||
{"function": "finite_number", "path": "backend/data/numbers.py"},
|
||||
{"function": "non_nan_number", "path": "backend/data/numbers.py"},
|
||||
],
|
||||
"llm_entrypoints": [
|
||||
{"function": "stream_with_mentor", "path": "backend/features/mentor/agent.py"},
|
||||
{"function": "interpret_heaven", "path": "backend/features/heaven/agent.py"},
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
| CR-01 | LLM供应商传输 | 问师、问天、复盘助手和策略编译各自构造HTTP请求、解析响应和读取错误 | 只保留`backend/llm/transport.py`一个网络出口 | 已完成 |
|
||||
| CR-02 | HTTP精确POST委托 | 27个端点重复使用“比较路径、调用无参数处理器、返回”三行分支 | 用公开/受保护两张显式映射统一委托,同时保留复杂路由的原控制流 | 已完成 |
|
||||
| CR-03 | 股票市场后缀转换 | Tushare业务与iFinD图表各保留一份完全相同的沪深京代码转换函数 | 图表复用`bootstrap/config.py::tushare_code`,只保留一份函数体 | 已完成 |
|
||||
| CR-04 | 数值归一化策略 | 四个业务模块分别保留两组完全相同的数值转换函数体 | 由`backend/data/numbers.py`集中拥有两种既有语义,消费者保留原局部别名 | 已完成 |
|
||||
|
||||
## CR-01验收口径
|
||||
|
||||
@@ -90,6 +91,30 @@
|
||||
本批基线为`xiaobai-reduction-02-http-dispatch-20260801`;检查点为
|
||||
`xiaobai-reduction-03-market-symbol-20260801`。
|
||||
|
||||
## CR-04验收口径
|
||||
|
||||
- 只合并参数、函数体和运行结果完全一致的数值转换函数,不借本批改变任何业务计算或异常默认值。
|
||||
- `finite_number`继续拒绝`NaN`与正负无穷;`non_nan_number`继续只拒绝`NaN`并保留正负无穷。
|
||||
- Tushare与智能选股必须复用有限数策略;市场洞察与情绪引擎必须复用非NaN策略,并继续暴露原局部
|
||||
`_number`名称以保持兼容。
|
||||
- 实时行情和图表转换器的空值、默认值或参数签名语义不同,必须继续独立保留,不能因名称相同而合并。
|
||||
- 原版函数参数和函数体分别与共享实现AST相等;四个消费者的局部别名必须指向对应的唯一函数对象。
|
||||
|
||||
## CR-04结果
|
||||
|
||||
- 删除Tushare、智能选股、市场洞察和情绪引擎中的四份重复函数体,新建两种明确命名的共享策略;生产
|
||||
代码净减少约12行,全仓AST扫描不再发现完全相同的函数定义。
|
||||
- 将可复用的函数及模块AST契约归入测试辅助层,原有迁移保持性测试改为“未改范围保持相等、被替换
|
||||
函数与共享实现相等、运行时唯一对象”三重断言,没有降低门禁。
|
||||
- 实时行情`backend/data/realtime.py::_number`与图表`backend/features/market/charts.py::_number`
|
||||
被明确保留;它们不是本批重复实现,也未改变行为。
|
||||
- 58项定向测试、41项保持性/治理测试、原版231项、候选320项、纯`app/`导出257项和45项
|
||||
Playwright通过;24个JavaScript文件、API/架构注册表、Git空白检查和SQLite完整性检查通过。
|
||||
- 本批不修改前端、CSS、接口、数据来源、行情口径、选股条件、数据库、LLM、权限或部署。
|
||||
|
||||
本批基线为`xiaobai-reduction-03-market-symbol-20260801`;检查点为
|
||||
`xiaobai-reduction-04-numeric-normalization-20260801`。
|
||||
|
||||
## 人工验收记录
|
||||
|
||||
- 2026-08-01:用户检查CR-02与CR-03运行结果,确认未发现明显异常。本记录仅表示本轮可见功能与
|
||||
|
||||
Reference in New Issue
Block a user