feat(HEL-402): 接通网站首批只读 datahub 并建立双路对比

默认全部读取/影子开关关闭,网站继续走旧 Tushare 链路;开启单项时只替换该类原料并在失败时回旧,问天保持旧路径。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
2026-09-02 17:09:28 +08:00
co-authored by Cursor multica-agent
parent f5dc0f8076
commit 0d13066386
19 changed files with 1412 additions and 1 deletions
+3
View File
@@ -67,6 +67,9 @@ class DataGatewayTests(unittest.TestCase):
"TushareClient": {"backend/features/market/service.py"},
"TushareProvider": {"backend/data/gateway.py"},
"WebRealtimeAggregator": {"backend/data/gateway.py"},
"DatahubClient": {"backend/data/gateway.py"},
"DatahubAwareTushareClient": {"backend/data/gateway.py"},
"DatahubBridge": {"backend/data/gateway.py"},
}
found = {name: set() for name in owners}
for path in (root / "backend").rglob("*.py"):
+254
View File
@@ -0,0 +1,254 @@
from __future__ import annotations
import ast
import json
import unittest
from pathlib import Path
from typing import Any
from backend.data import build_data_gateway
from backend.data.datahub.bridge import DatahubAwareTushareClient, DatahubBridge, looks_like_heaven
from backend.data.datahub.client import DatahubClient, DatahubResponse
from backend.data.datahub.compare import compare_rows
from backend.data.datahub.errors import DatahubError
from backend.data.datahub.native import to_canonical_row, to_native_row
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
ROOT = Path(__file__).resolve().parents[1]
TOKEN = "super-secret-datahub-token"
LEGACY_DAILY = {
"ts_code": "600000.SH",
"trade_date": "20240902",
"open": 10.11,
"high": 10.25,
"low": 10.01,
"close": 10.20,
"pct_chg": 1.2345,
"vol": 1000.0,
"amount": 2000.0,
}
HUB_DAILY = {
"ts_code": "600000.SH",
"trade_date": "20240902",
"open": 10.11,
"high": 10.25,
"low": 10.01,
"close": 10.20,
"pct_chg": 1.2345,
"volume": 100000.0,
"amount": 2000000.0,
}
class FakeLegacy:
def __init__(self, rows: list[dict[str, Any]] | Exception | None = None) -> None:
self.token = "legacy-token"
self.timeout = 30
self.rows = [] if rows is None else rows
self.calls: list[tuple[str, dict[str, Any] | None, str]] = []
def query(self, api_name: str, params: dict[str, Any] | None = None, fields: str = "") -> list[dict[str, Any]]:
self.calls.append((api_name, params, fields))
if isinstance(self.rows, Exception):
raise self.rows
return [dict(row) for row in self.rows]
class FakeClient(DatahubClient):
def __init__(self, error: DatahubError | None = None, response: DatahubResponse | None = None) -> None:
super().__init__(DatahubSettings(base_url="http://127.0.0.1:9", token=TOKEN))
self.error = error
self.response = response or DatahubResponse(
data=[dict(HUB_DAILY)],
meta={"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0},
)
self.paths: list[str] = []
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
self.paths.append(path)
if TOKEN in json.dumps(params or {}) or TOKEN in path:
raise AssertionError("token leaked into url")
if self.error:
raise self.error
return self.response
def flags(**enabled: tuple[bool, bool]) -> DatahubSettings:
datasets = {name: DatasetFlags(name) for name in DATASETS}
for name, pair in enabled.items():
datasets[name] = DatasetFlags(name, read=pair[0], shadow=pair[1])
return DatahubSettings(base_url="http://127.0.0.1:9", token=TOKEN, datasets=datasets)
class DatahubBridgeTests(unittest.TestCase):
def test_default_config_keeps_legacy_and_does_not_call_datahub(self) -> None:
settings = DatahubSettings.load(environ={}, credentials={})
self.assertFalse(settings.any_enabled())
self.assertTrue(all(not settings.flags(name).read and not settings.flags(name).shadow for name in DATASETS))
client = FakeClient(error=DatahubError("INTERNAL", "should not be called"))
legacy = FakeLegacy([LEGACY_DAILY])
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(settings, client))
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,close,vol,amount")
self.assertEqual(rows[0]["amount"], 2000.0)
self.assertEqual(client.paths, [])
self.assertEqual(len(legacy.calls), 1)
def test_each_dataset_has_independent_read_flag(self) -> None:
settings = flags(daily=(True, False), auction=(False, False))
self.assertTrue(settings.flags("daily").read)
self.assertFalse(settings.flags("auction").read)
self.assertFalse(any(settings.flags(name).read for name in DATASETS if name != "daily"))
source = (ROOT / "config" / "datahub.config.json").read_text(encoding="utf-8")
self.assertNotIn("master", source)
self.assertNotIn("DATAHUB_READ_ALL", source)
def test_read_flag_replaces_only_that_dataset_and_converts_units(self) -> None:
shadows: list[dict[str, Any]] = []
client = FakeClient()
legacy = FakeLegacy([LEGACY_DAILY])
wrapped = DatahubAwareTushareClient(
legacy,
DatahubBridge(flags(daily=(True, False)), client, shadow_sink=shadows.append),
)
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,vol,amount")
self.assertEqual(rows[0]["vol"], 1000.0)
self.assertEqual(rows[0]["amount"], 2000.0)
self.assertEqual(legacy.calls, [])
self.assertEqual(client.paths, ["/v1/bars/daily"])
calendar_legacy = FakeLegacy([{"cal_date": "20240902", "is_open": 1}])
calendar_client = FakeClient(error=DatahubError("INTERNAL", "nope"))
calendar_wrapped = DatahubAwareTushareClient(
calendar_legacy,
DatahubBridge(flags(daily=(True, False)), calendar_client),
)
calendar = calendar_wrapped.query("trade_cal", {"start_date": "20240902", "end_date": "20240902"}, "")
self.assertEqual(calendar[0]["is_open"], 1)
self.assertEqual(calendar_client.paths, [])
def test_fallback_on_down_401_timeout_empty_unpublished_and_stale(self) -> None:
cases = [
DatahubError("UNAVAILABLE", "down"),
DatahubError("UNAUTHORIZED", "401"),
DatahubError("TIMEOUT", "late"),
DatahubError("EMPTY", "no rows"),
DatahubError("DATASET_NOT_PUBLISHED", "not ready"),
DatahubError("STALE", "old"),
]
for error in cases:
with self.subTest(error=error.code):
if error.code == "EMPTY":
client = FakeClient(response=DatahubResponse(data=[], meta={"stale": False, "staleness_seconds": 0}))
elif error.code == "STALE":
client = FakeClient(response=DatahubResponse(
data=[dict(HUB_DAILY)],
meta={"stale": True, "staleness_seconds": 999999},
))
else:
client = FakeClient(error=error)
legacy = FakeLegacy([LEGACY_DAILY])
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(flags(daily=(True, False)), client))
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
self.assertEqual(rows[0]["amount"], 2000.0)
self.assertEqual(len(legacy.calls), 1)
def test_shadow_compares_without_replacing_and_survives_hub_failure(self) -> None:
reports: list[dict[str, Any]] = []
client = FakeClient()
legacy = FakeLegacy([LEGACY_DAILY])
wrapped = DatahubAwareTushareClient(
legacy,
DatahubBridge(flags(daily=(False, True)), client, shadow_sink=reports.append),
)
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount,vol")
self.assertEqual(rows[0]["amount"], 2000.0)
self.assertEqual(len(legacy.calls), 1)
self.assertEqual(reports[0]["equal"], True)
self.assertEqual(reports[0]["matched"], 1)
failed = FakeClient(error=DatahubError("UNAVAILABLE", TOKEN))
fail_reports: list[dict[str, Any]] = []
fail_legacy = FakeLegacy([LEGACY_DAILY])
fail_wrapped = DatahubAwareTushareClient(
fail_legacy,
DatahubBridge(flags(daily=(False, True)), failed, shadow_sink=fail_reports.append),
)
again = fail_wrapped.query("daily", {"trade_date": "20240902"}, "amount")
self.assertEqual(again[0]["amount"], 2000.0)
self.assertTrue(fail_reports[0]["hub_error"])
self.assertNotIn(TOKEN, json.dumps(fail_reports[0]))
def test_compare_classifies_unit_conversion_missing_row_and_value_diff(self) -> None:
equal = compare_rows("daily", [LEGACY_DAILY], [HUB_DAILY], {"stale": False, "staleness_seconds": 0})
self.assertTrue(equal["equal"])
unit = compare_rows("daily", [LEGACY_DAILY], [{**HUB_DAILY, "amount": 2000.0, "volume": 1000.0}])
self.assertGreater(unit["unit_conversion_count"], 0)
missing = compare_rows("daily", [LEGACY_DAILY], [])
self.assertEqual(missing["missing_hub_count"], 1)
value = compare_rows("daily", [LEGACY_DAILY], [{**HUB_DAILY, "close": 99.0}])
self.assertEqual(value["value_diff_count"], 1)
skew = compare_rows("daily", [LEGACY_DAILY], [HUB_DAILY], {"stale": False, "staleness_seconds": 12})
self.assertTrue(skew["time_skew"])
def test_native_roundtrip_matches_known_scales(self) -> None:
native = to_native_row("daily", HUB_DAILY)
self.assertEqual(native["vol"], 1000.0)
self.assertEqual(native["amount"], 2000.0)
canonical = to_canonical_row("daily", native)
self.assertEqual(canonical["vol"], 100000.0)
self.assertEqual(canonical["amount"], 2000000.0)
def test_heaven_keeps_legacy_even_when_read_flag_is_on(self) -> None:
self.assertTrue(looks_like_heaven("backend.features.heaven.market_context", "backend/features/heaven/market_context.py"))
self.assertFalse(looks_like_heaven("backend.features.market.service", "backend/features/market/service.py"))
client = FakeClient()
legacy = FakeLegacy([LEGACY_DAILY])
wrapped = DatahubAwareTushareClient(
legacy,
DatahubBridge(flags(daily=(True, False)), client, heaven_guard=lambda: True),
)
rows = wrapped.query("daily", {"trade_date": "20240902"}, "amount")
self.assertEqual(rows[0]["amount"], 2000.0)
self.assertEqual(client.paths, [])
def test_status_flag_does_not_run_when_off_and_falls_back_when_on(self) -> None:
off = DatahubBridge(flags(), FakeClient(error=DatahubError("UNAVAILABLE", "down")))
self.assertIsNone(off.dataset_status("20240902"))
reports: list[dict[str, Any]] = []
failed = DatahubBridge(
flags(status=(True, True)),
FakeClient(error=DatahubError("UNAUTHORIZED", "nope")),
shadow_sink=reports.append,
)
self.assertIsNone(failed.dataset_status("20240902"))
self.assertTrue(reports[0]["hub_error"])
ok = DatahubBridge(
flags(status=(True, False)),
FakeClient(response=DatahubResponse(data=[{"dataset": "daily", "state": "published"}], meta={"stale": False, "staleness_seconds": 0})),
)
self.assertEqual(ok.dataset_status("20240902")[0]["state"], "published")
def test_default_gateway_wraps_tushare_without_calling_datahub(self) -> None:
gateway = build_data_gateway({}, datahub_settings=flags())
client = gateway.tushare()
self.assertIsInstance(client, DatahubAwareTushareClient)
self.assertFalse(gateway.datahub.settings.any_enabled())
def test_features_do_not_import_datahub_client(self) -> None:
violations = []
for path in (ROOT / "backend" / "features").rglob("*.py"):
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
for node in ast.walk(tree):
names = []
if isinstance(node, ast.Import):
names = [alias.name for alias in node.names]
elif isinstance(node, ast.ImportFrom) and node.module:
names = [node.module]
for name in names:
if "datahub" in name.split("."):
violations.append(f"{path.relative_to(ROOT)} -> {name}")
self.assertEqual(violations, [])
if __name__ == "__main__":
unittest.main()
+185
View File
@@ -0,0 +1,185 @@
from __future__ import annotations
import io
import json
import logging
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlparse
from backend.data.datahub.client import DatahubClient
from backend.data.datahub.errors import DatahubError
from backend.data.datahub.redact import redact_text
from backend.data.datahub.settings import DatahubSettings
TOKEN = "super-secret-datahub-token"
class FakeHubState:
def __init__(self) -> None:
self.mode = "ok"
self.hits = 0
self.paths: list[str] = []
STATE = FakeHubState()
class FakeHubHandler(BaseHTTPRequestHandler):
def log_message(self, format: str, *args: object) -> None:
return
def do_GET(self) -> None: # noqa: N802
STATE.hits += 1
parsed = urlparse(self.path)
STATE.paths.append(parsed.path)
token = self.headers.get("X-Datahub-Token", "")
if STATE.mode == "timeout":
raise TimeoutError("simulated timeout")
if token != TOKEN:
self._json(401, {"error": {"code": "UNAUTHORIZED", "message": "missing or invalid X-Datahub-Token"}})
return
if STATE.mode == "unpublished":
self._json(404, {"error": {"code": "DATASET_NOT_PUBLISHED", "message": "daily 19990101 尚未发布", "expected_at": "15:05+08:00"}})
return
if STATE.mode == "empty":
self._json(200, {"schema_version": 1, "data": [], "meta": {"tier": "official", "stale": False, "staleness_seconds": 0}})
return
if STATE.mode == "stale":
self._json(200, {"schema_version": 1, "data": [{"ts_code": "600000.SH", "trade_date": "20240902", "close": 10.2, "volume": 100000, "amount": 2000000}], "meta": {"tier": "official", "stale": True, "staleness_seconds": 999999}})
return
if STATE.mode == "invalid":
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(b"not-json")
return
if parsed.path == "/v1/health":
self._json(200, {"schema_version": 1, "data": {"status": "ok"}, "meta": {"tier": "official", "source": "datahub", "stale": False, "staleness_seconds": 0}})
return
if parsed.path == "/v1/calendar":
self._json(200, {"schema_version": 1, "data": [{"cal_date": "20240902", "is_open": True, "pretrade_date": "20240830"}], "meta": {"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0}})
return
if parsed.path == "/v1/bars/daily":
query = {key: values[-1] for key, values in parse_qs(parsed.query).items()}
self._json(200, {
"schema_version": 1,
"data": [{
"ts_code": "600000.SH",
"trade_date": query.get("date") or "20240902",
"open": 10.11, "high": 10.25, "low": 10.01, "close": 10.20,
"pct_chg": 1.2345, "volume": 100000.0, "amount": 2000000.0, "adj_factor": 1.1,
}],
"meta": {"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0, "source": "tushare:daily"},
})
return
if parsed.path == "/v1/datasets/status":
self._json(200, {"schema_version": 1, "data": [{"dataset": "daily", "state": "published", "trade_date": "20240902"}], "meta": {"tier": "official", "stale": False, "staleness_seconds": 0}})
return
self._json(400, {"error": {"code": "INVALID_ARGUMENT", "message": f"unknown endpoint: {parsed.path}"}})
def _json(self, status: int, payload: dict) -> None:
body = json.dumps(payload).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
class DatahubClientTests(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
cls.server = ThreadingHTTPServer(("127.0.0.1", 0), FakeHubHandler)
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
cls.thread.start()
cls.base = f"http://127.0.0.1:{cls.server.server_address[1]}"
@classmethod
def tearDownClass(cls) -> None:
cls.server.shutdown()
cls.server.server_close()
def setUp(self) -> None:
STATE.mode = "ok"
STATE.hits = 0
STATE.paths = []
self.client = DatahubClient(DatahubSettings(base_url=self.base, token=TOKEN, retries=1, timeout_seconds=2))
def test_health_envelope(self) -> None:
response = self.client.health()
self.assertEqual(response.schema_version, 1)
self.assertEqual(response.data["status"], "ok")
self.assertIn("stale", response.meta)
def test_missing_and_bad_token_401(self) -> None:
missing = DatahubClient(DatahubSettings(base_url=self.base, token=""))
with self.assertRaises(DatahubError) as raised:
missing.health()
self.assertEqual(raised.exception.code, "NOT_CONFIGURED")
bad = DatahubClient(DatahubSettings(base_url=self.base, token="wrong"))
with self.assertRaises(DatahubError) as raised:
bad.health()
self.assertEqual(raised.exception.code, "UNAUTHORIZED")
self.assertNotIn(TOKEN, str(raised.exception))
def test_unpublished_and_empty_and_stale_codes(self) -> None:
STATE.mode = "unpublished"
with self.assertRaises(DatahubError) as raised:
self.client.daily_bars(date="19990101")
self.assertEqual(raised.exception.code, "DATASET_NOT_PUBLISHED")
STATE.mode = "empty"
response = self.client.daily_bars(date="20240902")
self.assertEqual(response.data, [])
STATE.mode = "stale"
stale = self.client.daily_bars(date="20240902")
self.assertTrue(stale.meta["stale"])
def test_invalid_json_maps_to_internal(self) -> None:
STATE.mode = "invalid"
with self.assertRaises(DatahubError) as raised:
self.client.health()
self.assertEqual(raised.exception.code, "INTERNAL")
def test_timeout_maps_and_retries(self) -> None:
hits = {"n": 0}
def boom(_request, timeout=None):
hits["n"] += 1
raise TimeoutError("late")
client = DatahubClient(
DatahubSettings(base_url=self.base, token=TOKEN, retries=1, timeout_seconds=1),
urlopen=boom,
)
with self.assertRaises(DatahubError) as raised:
client.health()
self.assertEqual(raised.exception.code, "TIMEOUT")
self.assertEqual(hits["n"], 2)
def test_token_never_appears_in_error_text_or_logs(self) -> None:
stream = io.StringIO()
logger = logging.getLogger("xiaobai.datahub")
handler = logging.StreamHandler(stream)
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
try:
with self.assertRaises(DatahubError):
DatahubClient(DatahubSettings(base_url=self.base, token="wrong")).health()
blob = stream.getvalue() + redact_text("header " + TOKEN, (TOKEN,))
self.assertNotIn(TOKEN, blob)
self.assertIn("***", redact_text(TOKEN, (TOKEN,)))
finally:
logger.removeHandler(handler)
def test_calendar_and_status_contract(self) -> None:
calendar = self.client.calendar("20240901", "20240902")
self.assertEqual(calendar.data[0]["cal_date"], "20240902")
status = self.client.dataset_status("20240902")
self.assertEqual(status.data[0]["dataset"], "daily")
if __name__ == "__main__":
unittest.main()