fix(HEL-494): 数据中枢独占调度，主网站不再回退旧接口
主网站只向中枢要业务数据；来源选择、切源、补数全部在中枢内部完成，失败不再走东财/腾讯/Tushare 保底。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
co-authored by
Cursor
multica-agent
Cursor
multica-agent
parent
ef13d6feb5
commit
0b8419abca
@@ -13,6 +13,7 @@ from backend.data.datahub.compare import compare_rows
|
||||
from backend.data.datahub.errors import DatahubError
|
||||
from backend.data.datahub.native import to_canonical_row, to_native_row
|
||||
from backend.data.datahub.route_state import LEDGER
|
||||
from backend.data.providers.tushare_transport import TushareError
|
||||
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
@@ -68,9 +69,16 @@ class FakeClient(DatahubClient):
|
||||
self.calls: list[tuple[str, dict[str, Any]]] = []
|
||||
|
||||
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
|
||||
return self._record(path, params)
|
||||
|
||||
def post(self, path: str, body: dict[str, Any] | None = None) -> DatahubResponse:
|
||||
return self._record(path, body)
|
||||
|
||||
def _record(self, path: str, payload: dict[str, Any] | None) -> DatahubResponse:
|
||||
self.paths.append(path)
|
||||
self.calls.append((path, {key: value for key, value in (params or {}).items()}))
|
||||
if TOKEN in json.dumps(params or {}) or TOKEN in path:
|
||||
self.calls.append((path, {key: value for key, value in (payload or {}).items()}))
|
||||
packed = json.dumps(payload or {})
|
||||
if TOKEN in packed or TOKEN in path:
|
||||
raise AssertionError("token leaked into url")
|
||||
if self.error:
|
||||
raise self.error
|
||||
@@ -131,16 +139,22 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
self.assertEqual(legacy.calls, [])
|
||||
self.assertEqual(client.paths, ["/v1/bars/daily"])
|
||||
calendar_legacy = FakeLegacy([{"cal_date": "20240902", "is_open": 1}])
|
||||
calendar_client = FakeClient(error=DatahubError("INTERNAL", "nope"))
|
||||
calendar_client = FakeClient(
|
||||
response=DatahubResponse(
|
||||
data=[{"cal_date": "20240902", "is_open": 1, "pretrade_date": "20240830"}],
|
||||
meta={"source": "datahub", "stale": False, "staleness_seconds": 0},
|
||||
)
|
||||
)
|
||||
calendar_wrapped = DatahubAwareTushareClient(
|
||||
calendar_legacy,
|
||||
DatahubBridge(flags(daily=(True, False)), calendar_client),
|
||||
)
|
||||
calendar = calendar_wrapped.query("trade_cal", {"start_date": "20240902", "end_date": "20240902"}, "")
|
||||
self.assertEqual(calendar[0]["is_open"], 1)
|
||||
self.assertEqual(calendar_client.paths, [])
|
||||
self.assertEqual(calendar_legacy.calls, [])
|
||||
self.assertEqual(calendar_client.paths, ["/v1/query"])
|
||||
|
||||
def test_fallback_on_down_401_timeout_empty_unpublished_stale_and_incomplete(self) -> None:
|
||||
def test_hub_failure_does_not_call_website_legacy(self) -> None:
|
||||
cases = [
|
||||
DatahubError("UNAVAILABLE", "down"),
|
||||
DatahubError("UNAUTHORIZED", "401"),
|
||||
@@ -152,34 +166,21 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
]
|
||||
for error in cases:
|
||||
with self.subTest(error=error.code):
|
||||
if error.code == "EMPTY":
|
||||
client = FakeClient(response=DatahubResponse(data=[], meta={"stale": False, "staleness_seconds": 0}))
|
||||
elif error.code == "STALE":
|
||||
client = FakeClient(response=DatahubResponse(
|
||||
data=[dict(HUB_DAILY)],
|
||||
meta={"stale": True, "staleness_seconds": 999999},
|
||||
))
|
||||
elif error.code == "INCOMPLETE":
|
||||
client = FakeClient(response=DatahubResponse(
|
||||
data=[dict(HUB_DAILY)],
|
||||
meta={
|
||||
"stale": False,
|
||||
"staleness_seconds": 0,
|
||||
"incomplete": True,
|
||||
"coverage": {"complete": False, "missing_count": 80},
|
||||
},
|
||||
))
|
||||
else:
|
||||
client = FakeClient(error=error)
|
||||
client = FakeClient(error=error)
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(flags(daily=(True, False)), client))
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(len(legacy.calls), 1)
|
||||
with self.assertRaises(TushareError):
|
||||
wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
|
||||
self.assertEqual(legacy.calls, [])
|
||||
|
||||
def test_shadow_compares_without_replacing_and_survives_hub_failure(self) -> None:
|
||||
def test_shadow_mode_no_longer_calls_website_tushare(self) -> None:
|
||||
reports: list[dict[str, Any]] = []
|
||||
client = FakeClient()
|
||||
client = FakeClient(
|
||||
response=DatahubResponse(
|
||||
data=[dict(LEGACY_DAILY)],
|
||||
meta={"source": "tushare", "stale": False, "staleness_seconds": 0, "row_shape": "tushare"},
|
||||
)
|
||||
)
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(
|
||||
legacy,
|
||||
@@ -187,21 +188,19 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
)
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount,vol")
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(len(legacy.calls), 1)
|
||||
self.assertEqual(reports[0]["equal"], True)
|
||||
self.assertEqual(reports[0]["matched"], 1)
|
||||
self.assertEqual(legacy.calls, [])
|
||||
self.assertEqual(client.paths, ["/v1/query"])
|
||||
|
||||
failed = FakeClient(error=DatahubError("UNAVAILABLE", TOKEN))
|
||||
fail_reports: list[dict[str, Any]] = []
|
||||
fail_legacy = FakeLegacy([LEGACY_DAILY])
|
||||
fail_wrapped = DatahubAwareTushareClient(
|
||||
fail_legacy,
|
||||
DatahubBridge(flags(daily=(False, True)), failed, shadow_sink=fail_reports.append),
|
||||
DatahubBridge(flags(daily=(False, True)), failed, shadow_sink=reports.append),
|
||||
)
|
||||
again = fail_wrapped.query("daily", {"trade_date": "20240902"}, "amount")
|
||||
self.assertEqual(again[0]["amount"], 2000.0)
|
||||
self.assertTrue(fail_reports[0]["hub_error"])
|
||||
self.assertNotIn(TOKEN, json.dumps(fail_reports[0]))
|
||||
with self.assertRaises(TushareError):
|
||||
fail_wrapped.query("daily", {"trade_date": "20240902"}, "amount")
|
||||
self.assertEqual(fail_legacy.calls, [])
|
||||
self.assertNotIn(TOKEN, str(failed.calls))
|
||||
|
||||
def test_compare_classifies_unit_conversion_missing_row_and_value_diff(self) -> None:
|
||||
equal = compare_rows("daily", [LEGACY_DAILY], [HUB_DAILY], {"stale": False, "staleness_seconds": 0})
|
||||
@@ -288,13 +287,12 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
)
|
||||
wrapped = DatahubAwareTushareClient(
|
||||
FakeLegacy([legacy_close_only]),
|
||||
DatahubBridge(flags(daily=(False, True)), client, shadow_sink=reports.append),
|
||||
DatahubBridge(flags(daily=(True, False)), client, shadow_sink=reports.append),
|
||||
)
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,trade_date,close,vol,amount")
|
||||
self.assertEqual(rows[0]["close"], 10.20)
|
||||
self.assertEqual(rows[0]["vol"], 1000.0)
|
||||
self.assertTrue(reports[0]["equal"])
|
||||
self.assertEqual(reports[0]["matched"], 1)
|
||||
self.assertEqual(client.paths, ["/v1/bars/daily"])
|
||||
|
||||
def test_native_roundtrip_matches_known_scales(self) -> None:
|
||||
native = to_native_row("daily", HUB_DAILY)
|
||||
@@ -347,21 +345,17 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
self.assertIn('"daily"', source)
|
||||
self.assertIn("start_date", source)
|
||||
self.assertIn("end_date", source)
|
||||
client = FakeClient(
|
||||
response=DatahubResponse(
|
||||
data=[dict(HUB_DAILY)],
|
||||
meta={"stale": False, "staleness_seconds": 0, "incomplete": True, "coverage": {"complete": False, "missing_count": 89}},
|
||||
)
|
||||
)
|
||||
client = FakeClient(error=DatahubError("INCOMPLETE", "truncated"))
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(flags(daily=(True, False)), client))
|
||||
rows = wrapped.query(
|
||||
"daily",
|
||||
{"ts_code": "600000.SH", "start_date": "20240301", "end_date": "20240902"},
|
||||
"ts_code,amount",
|
||||
)
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(len(legacy.calls), 1)
|
||||
with self.assertRaises(TushareError):
|
||||
wrapped.query(
|
||||
"daily",
|
||||
{"ts_code": "600000.SH", "start_date": "20240301", "end_date": "20240902"},
|
||||
"ts_code,amount",
|
||||
)
|
||||
self.assertEqual(legacy.calls, [])
|
||||
self.assertIn("/v1/query", client.paths)
|
||||
|
||||
def test_try_intraday_respects_switch_and_falls_back_on_bad_payload(self) -> None:
|
||||
closed = DatahubBridge(flags(), FakeClient(error=DatahubError("INTERNAL", "should not run")))
|
||||
@@ -462,17 +456,15 @@ class DatahubBridgeTests(unittest.TestCase):
|
||||
FakeClient(error=DatahubError("UNAVAILABLE", "down")),
|
||||
)
|
||||
self.assertIsNone(failed.try_market_quotes("20240902"))
|
||||
failed.record_legacy("quotes", "tencent_qt", "down")
|
||||
snap = next(item for item in LEDGER.snapshot() if item["dataset"] == "quotes")
|
||||
self.assertEqual(snap["route"], "legacy")
|
||||
self.assertEqual(snap["source"], "tencent_qt")
|
||||
self.assertIn("备用", "备用")
|
||||
self.assertEqual(snap["route"], "datahub")
|
||||
self.assertEqual(snap["source"], "unavailable")
|
||||
|
||||
gateway = build_data_gateway({}, datahub_settings=flags(quotes=(True, False)))
|
||||
status = gateway.datahub_status()
|
||||
self.assertEqual(status["enabled_reads"], 1)
|
||||
self.assertEqual(status["total_reads"], len(DATASETS))
|
||||
self.assertGreaterEqual(status["fallback_count"], 1)
|
||||
self.assertEqual(status["fallback_count"], 0)
|
||||
|
||||
def test_try_daily_chart_converts_hub_bars(self) -> None:
|
||||
rows = [
|
||||
|
||||
Reference in New Issue
Block a user