From 7afc9ac23b493c704ad0d724cec8438d04c42fac Mon Sep 17 00:00:00 2001 From: leefer Date: Sat, 1 Aug 2026 16:54:47 +0800 Subject: [PATCH] refactor: centralize numeric normalization --- ARCHITECTURE.md | 2 +- backend/data/numbers.py | 20 ++++++++++ backend/data/providers/tushare_client.py | 9 +---- backend/features/market/insights.py | 9 +---- backend/features/screener/engine.py | 9 +---- backend/features/sentiment/engine.py | 10 +---- config/architecture-inventory.json | 22 ++++++++--- tests/preservation_helpers.py | 36 ++++++++++++++++++ tests/test_numeric_normalization.py | 38 +++++++++++++++++++ tests/test_preservation_slice_market.py | 23 +++++------ ...test_preservation_slice_market_insights.py | 7 ++++ tests/test_preservation_slice_screener.py | 28 ++++++++------ ...test_preservation_slice_sentiment_pools.py | 19 +++++++++- tools/build_architecture_inventory.py | 4 ++ 14 files changed, 171 insertions(+), 65 deletions(-) create mode 100644 backend/data/numbers.py create mode 100644 tests/test_numeric_normalization.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 33cdec8..2cfb529 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -36,7 +36,7 @@ background scheduler - `backend/features//` owns the mechanically moved service, repository, HTTP, agent, or deterministic calculation code for that product area. - `backend/data/` owns provider construction, source policy, provenance, units, freshness, - coverage, and display-versus-calculation eligibility. + coverage, display-versus-calculation eligibility, and shared numeric normalization policies. - `backend/database/` owns connection management, ordered migrations, and narrow repository adapters. Root `database.py` remains the legacy schema/composition anchor and combines the feature repository mixins; do not add feature queries to it. diff --git a/backend/data/numbers.py b/backend/data/numbers.py new file mode 100644 index 0000000..f3b239b --- /dev/null +++ b/backend/data/numbers.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import math +from typing import Any + + +def finite_number(value: Any, default: float = 0.0) -> float: + try: + number = float(value) + return number if math.isfinite(number) else default + except (TypeError, ValueError): + return default + + +def non_nan_number(value: Any, default: float = 0.0) -> float: + try: + number = float(value) + return number if number == number else default + except (TypeError, ValueError): + return default diff --git a/backend/data/providers/tushare_client.py b/backend/data/providers/tushare_client.py index 8e3c71e..752708a 100644 --- a/backend/data/providers/tushare_client.py +++ b/backend/data/providers/tushare_client.py @@ -11,6 +11,7 @@ from datetime import datetime, time as dt_time, timedelta from threading import Lock from typing import Any, ClassVar +from backend.data.numbers import finite_number as _number from backend.features.sentiment.engine import apply_sentiment_to_dashboard @@ -1812,14 +1813,6 @@ class TushareClient: } -def _number(value: Any, default: float = 0.0) -> float: - try: - number = float(value) - return number if math.isfinite(number) else default - except (TypeError, ValueError): - return default - - def _text(value: Any) -> str: if isinstance(value, (list, tuple, set)): return "、".join(str(item).strip() for item in value if str(item).strip()) diff --git a/backend/features/market/insights.py b/backend/features/market/insights.py index 226521d..4ec8274 100644 --- a/backend/features/market/insights.py +++ b/backend/features/market/insights.py @@ -6,6 +6,7 @@ from datetime import datetime, time as dt_time, timedelta, timezone from statistics import median from typing import TYPE_CHECKING, Any, Callable +from backend.data.numbers import non_nan_number as _number from backend.data.providers.ifind_client import IfindError, IfindHttpClient from backend.data.providers.tushare_client import TushareClient, TushareError @@ -16,14 +17,6 @@ if TYPE_CHECKING: CHINA_TIMEZONE = timezone(timedelta(hours=8)) -def _number(value: Any, default: float = 0.0) -> float: - try: - number = float(value) - return number if number == number else default - except (TypeError, ValueError): - return default - - def _display_date(value: str) -> str: text = str(value or "").replace("-", "") if len(text) != 8: diff --git a/backend/features/screener/engine.py b/backend/features/screener/engine.py index d0796db..e56ceb2 100644 --- a/backend/features/screener/engine.py +++ b/backend/features/screener/engine.py @@ -9,6 +9,7 @@ from datetime import datetime, timedelta from typing import Any from advanced_strategies import ADVANCED_CURATED_STRATEGIES +from backend.data.numbers import finite_number as _number from database import ReviewDatabase from backend.features.sentiment.engine import build_sentiment_history, latest_contiguous_history from tushare_client import TushareClient, TushareError @@ -2201,13 +2202,5 @@ def _regime_reason(regime: str) -> str: }.get(regime, "市场阶段待确认。") -def _number(value: Any, default: float = 0.0) -> float: - try: - number = float(value) - return number if math.isfinite(number) else default - except (TypeError, ValueError): - return default - - def _display_date(value: str) -> str: return f"{value[:4]}-{value[4:6]}-{value[6:8]}" if len(value) == 8 else value diff --git a/backend/features/sentiment/engine.py b/backend/features/sentiment/engine.py index 9345cd5..31934e3 100644 --- a/backend/features/sentiment/engine.py +++ b/backend/features/sentiment/engine.py @@ -4,6 +4,8 @@ from copy import deepcopy from statistics import mean, median from typing import Any +from backend.data.numbers import non_nan_number as _number + COMPONENT_WEIGHTS = { "breadth": 20, @@ -16,14 +18,6 @@ COMPONENT_WEIGHTS = { SENTIMENT_ENGINE_VERSION = 2 -def _number(value: Any, default: float = 0.0) -> float: - try: - number = float(value) - return number if number == number else default - except (TypeError, ValueError): - return default - - def _clamp(value: float, lower: float = 0.0, upper: float = 100.0) -> float: return min(upper, max(lower, value)) diff --git a/config/architecture-inventory.json b/config/architecture-inventory.json index fa818a2..1e0b183 100644 --- a/config/architecture-inventory.json +++ b/config/architecture-inventory.json @@ -220,6 +220,16 @@ "runtime_role": "index observation fallback" } ], + "numeric_normalization": [ + { + "function": "finite_number", + "path": "backend/data/numbers.py" + }, + { + "function": "non_nan_number", + "path": "backend/data/numbers.py" + } + ], "llm_entrypoints": [ { "function": "stream_with_mentor", @@ -279,13 +289,13 @@ }, { "path": "backend/features/screener/engine.py", - "bytes": 108552, - "lines": 2213 + "bytes": 108394, + "lines": 2206 }, { "path": "backend/data/providers/tushare_client.py", - "bytes": 94329, - "lines": 2175 + "bytes": 94171, + "lines": 2168 }, { "path": "frontend/app.js", @@ -314,8 +324,8 @@ }, { "path": "backend/features/market/insights.py", - "bytes": 58150, - "lines": 1314 + "bytes": 57998, + "lines": 1307 }, { "path": "frontend/pages/market/runtime.js", diff --git a/tests/preservation_helpers.py b/tests/preservation_helpers.py index 308dfeb..664b099 100644 --- a/tests/preservation_helpers.py +++ b/tests/preservation_helpers.py @@ -1,5 +1,6 @@ from __future__ import annotations +import ast import hashlib import re from pathlib import Path @@ -33,6 +34,41 @@ def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() +def function_contract(path: Path, name: str) -> tuple[str, str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + function = next( + node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name + ) + body = ast.Module(body=function.body, type_ignores=[]) + return ( + ast.dump(function.args, include_attributes=False), + ast.dump(body, include_attributes=False), + ) + + +def module_contract( + path: Path, + *, + excluded_definitions: set[str] | None = None, + excluded_import_modules: set[str] | None = None, + exclude_imports: bool = False, +) -> str: + excluded_definitions = excluded_definitions or set() + excluded_import_modules = excluded_import_modules or set() + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + tree.body = [ + node + for node in tree.body + if not (exclude_imports and isinstance(node, (ast.Import, ast.ImportFrom))) + and not ( + isinstance(node, ast.ImportFrom) + and node.module in excluded_import_modules + ) + and getattr(node, "name", None) not in excluded_definitions + ] + return ast.dump(tree, include_attributes=False) + + def reassembled_frontend_runtime() -> str: chunks: dict[tuple[int, int], str] = {} for path in FRONTEND_ROOT.rglob("*.js"): diff --git a/tests/test_numeric_normalization.py b/tests/test_numeric_normalization.py new file mode 100644 index 0000000..f31a801 --- /dev/null +++ b/tests/test_numeric_normalization.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import math +import unittest + +from backend.data.numbers import finite_number, non_nan_number +from backend.data.providers import tushare_client +from backend.features.market import insights +from backend.features.screener import engine as screener_engine +from backend.features.sentiment import engine as sentiment_engine + + +class NumericNormalizationTests(unittest.TestCase): + def test_consumers_use_their_declared_shared_policy(self) -> None: + self.assertIs(tushare_client._number, finite_number) + self.assertIs(screener_engine._number, finite_number) + self.assertIs(insights._number, non_nan_number) + self.assertIs(sentiment_engine._number, non_nan_number) + + def test_finite_policy_preserves_existing_results(self) -> None: + self.assertEqual(finite_number("12.5"), 12.5) + self.assertEqual(finite_number(None), 0.0) + self.assertEqual(finite_number("invalid", 7.0), 7.0) + self.assertEqual(finite_number(math.nan, 7.0), 7.0) + self.assertEqual(finite_number(math.inf, 7.0), 7.0) + self.assertEqual(finite_number(-math.inf, 7.0), 7.0) + + def test_non_nan_policy_keeps_infinity_but_rejects_nan(self) -> None: + self.assertEqual(non_nan_number("12.5"), 12.5) + self.assertEqual(non_nan_number(None), 0.0) + self.assertEqual(non_nan_number("invalid", 7.0), 7.0) + self.assertEqual(non_nan_number(math.nan, 7.0), 7.0) + self.assertEqual(non_nan_number(math.inf, 7.0), math.inf) + self.assertEqual(non_nan_number(-math.inf, 7.0), -math.inf) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_preservation_slice_market.py b/tests/test_preservation_slice_market.py index cc370a6..eae887a 100644 --- a/tests/test_preservation_slice_market.py +++ b/tests/test_preservation_slice_market.py @@ -11,12 +11,14 @@ import realtime_aggregator import tushare_client from backend.bootstrap import config as bootstrap_config from backend.data import realtime +from backend.data.numbers import finite_number from backend.data.providers import ifind_client as canonical_ifind from backend.data.providers import tushare_client as canonical_tushare from backend.features.market import charts from tests.preservation_helpers import ( assert_frontend_runtime_matches_audited_baseline, assert_moved_asset_matches, + function_contract, ) @@ -102,18 +104,6 @@ def top_level_definitions(path: Path) -> dict[str, str]: } -def function_contract(path: Path, name: str) -> tuple[str, str]: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - function = next( - node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == name - ) - body = ast.Module(body=function.body, type_ignores=[]) - return ( - ast.dump(function.args, include_attributes=False), - ast.dump(body, include_attributes=False), - ) - - class MarketSliceSourceEquivalenceTests(unittest.TestCase): def test_market_service_methods_are_exact_original_ast(self) -> None: original = class_methods(ORIGINAL_ROOT / "server.py", "DashboardService") @@ -154,10 +144,17 @@ class MarketSliceSourceEquivalenceTests(unittest.TestCase): ) for original, migrated in exact_moves: self.assertEqual(sha256(ORIGINAL_ROOT / original), sha256(APP_ROOT / migrated)) + original_tushare = top_level_definitions(ORIGINAL_ROOT / "tushare_client.py") + original_tushare.pop("_number") self.assertEqual( - top_level_definitions(ORIGINAL_ROOT / "tushare_client.py"), + original_tushare, top_level_definitions(APP_ROOT / "backend/data/providers/tushare_client.py"), ) + self.assertEqual( + function_contract(ORIGINAL_ROOT / "tushare_client.py", "_number"), + function_contract(APP_ROOT / "backend/data/numbers.py", "finite_number"), + ) + self.assertIs(canonical_tushare._number, finite_number) original_charts = top_level_definitions(ORIGINAL_ROOT / "chart_data_provider.py") original_charts.pop("_stock_market_code") self.assertEqual( diff --git a/tests/test_preservation_slice_market_insights.py b/tests/test_preservation_slice_market_insights.py index b2e2973..7242c94 100644 --- a/tests/test_preservation_slice_market_insights.py +++ b/tests/test_preservation_slice_market_insights.py @@ -6,11 +6,13 @@ import unittest from pathlib import Path import market_insights +from backend.data.numbers import non_nan_number from backend.features.market import insights as canonical_insights from tests.preservation_helpers import ( assert_frontend_runtime_matches_audited_baseline, assert_moved_asset_matches, assert_page_prefix_matches, + function_contract, ) @@ -106,6 +108,11 @@ class MarketInsightsSliceSourceEquivalenceTests(unittest.TestCase): MARKET_INSIGHT_METHODS, ) self.assertIs(market_insights.MarketInsightsService, canonical_insights.MarketInsightsService) + self.assertEqual( + function_contract(ORIGINAL_ROOT / "market_insights.py", "_number"), + function_contract(APP_ROOT / "backend/data/numbers.py", "non_nan_number"), + ) + self.assertIs(canonical_insights._number, non_nan_number) def test_dashboard_service_methods_are_exact_original_ast(self) -> None: original = ORIGINAL_ROOT / "server.py" diff --git a/tests/test_preservation_slice_screener.py b/tests/test_preservation_slice_screener.py index 478ceae..2788981 100644 --- a/tests/test_preservation_slice_screener.py +++ b/tests/test_preservation_slice_screener.py @@ -9,12 +9,15 @@ import advanced_strategies import llm_strategy import screener import strategy_tracking +from backend.data.numbers import finite_number from backend.features.screener import compiler, engine, strategies, tracking from backend.features.screener import service as screener_service from tests.preservation_helpers import ( assert_frontend_runtime_matches_audited_baseline, assert_moved_asset_matches, assert_page_prefix_matches, + function_contract, + module_contract, ) @@ -91,14 +94,6 @@ def top_level_definition(path: Path, name: str) -> str: return ast.dump(node, include_attributes=False) -def module_without_imports(path: Path) -> str: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - tree.body = [ - node for node in tree.body if not isinstance(node, (ast.Import, ast.ImportFrom)) - ] - return ast.dump(tree, include_attributes=False) - - def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -154,11 +149,22 @@ class ScreenerSliceSourceEquivalenceTests(unittest.TestCase): def test_engine_and_tracking_logic_match_the_original(self) -> None: self.assertEqual( - module_without_imports(ORIGINAL_ROOT / "screener.py"), - module_without_imports( - APP_ROOT / "backend" / "features" / "screener" / "engine.py" + module_contract( + ORIGINAL_ROOT / "screener.py", + excluded_definitions={"_number"}, + exclude_imports=True, + ), + module_contract( + APP_ROOT / "backend" / "features" / "screener" / "engine.py", + excluded_definitions={"_number"}, + exclude_imports=True, ), ) + self.assertEqual( + function_contract(ORIGINAL_ROOT / "screener.py", "_number"), + function_contract(APP_ROOT / "backend/data/numbers.py", "finite_number"), + ) + self.assertIs(engine._number, finite_number) self.assertEqual( class_methods( ORIGINAL_ROOT / "backend" / "features" / "screener" / "tracking.py", diff --git a/tests/test_preservation_slice_sentiment_pools.py b/tests/test_preservation_slice_sentiment_pools.py index bf3c3d2..7d261ca 100644 --- a/tests/test_preservation_slice_sentiment_pools.py +++ b/tests/test_preservation_slice_sentiment_pools.py @@ -6,11 +6,14 @@ import unittest from pathlib import Path import sentiment_engine +from backend.data.numbers import non_nan_number from backend.features.sentiment import engine as canonical_engine from tests.preservation_helpers import ( assert_frontend_runtime_matches_audited_baseline, assert_moved_asset_matches, assert_page_prefix_matches, + function_contract, + module_contract, ) @@ -94,10 +97,22 @@ class SentimentPoolSliceSourceEquivalenceTests(unittest.TestCase): def test_sentiment_engine_is_exact_original_with_legacy_alias(self) -> None: self.assertEqual( - sha256(ORIGINAL_ROOT / "sentiment_engine.py"), - sha256(APP_ROOT / "backend" / "features" / "sentiment" / "engine.py"), + module_contract( + ORIGINAL_ROOT / "sentiment_engine.py", + excluded_definitions={"_number"}, + ), + module_contract( + APP_ROOT / "backend" / "features" / "sentiment" / "engine.py", + excluded_definitions={"_number"}, + excluded_import_modules={"backend.data.numbers"}, + ), + ) + self.assertEqual( + function_contract(ORIGINAL_ROOT / "sentiment_engine.py", "_number"), + function_contract(APP_ROOT / "backend/data/numbers.py", "non_nan_number"), ) self.assertIs(sentiment_engine, canonical_engine) + self.assertIs(canonical_engine._number, non_nan_number) def test_api_and_frontend_assets_are_unchanged(self) -> None: self.assertEqual( diff --git a/tools/build_architecture_inventory.py b/tools/build_architecture_inventory.py index 5a1a172..8dbb448 100644 --- a/tools/build_architecture_inventory.py +++ b/tools/build_architecture_inventory.py @@ -155,6 +155,10 @@ def build() -> dict[str, Any]: {"provider": "eastmoney", "path": "backend/data/realtime.py", "runtime_role": "isolated realtime observation"}, {"provider": "tencent", "path": "backend/data/realtime.py", "runtime_role": "index observation fallback"}, ], + "numeric_normalization": [ + {"function": "finite_number", "path": "backend/data/numbers.py"}, + {"function": "non_nan_number", "path": "backend/data/numbers.py"}, + ], "llm_entrypoints": [ {"function": "stream_with_mentor", "path": "backend/features/mentor/agent.py"}, {"function": "interpret_heaven", "path": "backend/features/heaven/agent.py"},