435 lines
18 KiB
Python
435 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import ast
|
|
import json
|
|
import re
|
|
import threading
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from backend.data import build_data_gateway
|
|
from backend.data.datahub.bridge import DatahubAwareTushareClient, DatahubBridge
|
|
from backend.data.datahub.client import DatahubClient
|
|
from backend.data.datahub.ifind_proxy import HubIfindProxy
|
|
from backend.data.datahub.realtime_proxy import HubRealtimeProxy
|
|
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
|
from backend.data.providers.tushare_transport import TushareError
|
|
from backend.features.market.charts import MarketChartClient
|
|
from backend.features.system.service import SystemServiceMixin
|
|
from tests.test_datahub_bridge import FakeClient, FakeLegacy, flags
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
BLOCKED_HOSTS = (
|
|
"api.tushare.pro",
|
|
"push2.eastmoney.com",
|
|
"push2delay.eastmoney.com",
|
|
"push2his.eastmoney.com",
|
|
"push2ex.eastmoney.com",
|
|
"qt.gtimg.cn",
|
|
"hq.sinajs.cn",
|
|
"10jqka.com.cn",
|
|
"xuangubao.cn",
|
|
"quantapi.51ifind.com",
|
|
"51ifind.com",
|
|
)
|
|
LEFTOVER_WEBSITE_FILES = {
|
|
"backend/data/providers/ifind_client.py",
|
|
"backend/data/realtime.py",
|
|
"backend/features/market/charts.py",
|
|
"backend/data/providers/tushare_transport.py",
|
|
}
|
|
HUB_BASE = "http://127.0.0.1:8766"
|
|
|
|
|
|
def _enabled_settings() -> DatahubSettings:
|
|
return DatahubSettings(
|
|
base_url=HUB_BASE,
|
|
token="hub-token",
|
|
datasets={name: DatasetFlags(name, read=True) for name in DATASETS},
|
|
)
|
|
|
|
|
|
class _Resp:
|
|
def __init__(self, payload: dict) -> None:
|
|
self.status = 200
|
|
self._raw = json.dumps(payload).encode("utf-8")
|
|
|
|
def read(self):
|
|
return self._raw
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
return False
|
|
|
|
|
|
def hub_payload(request) -> dict:
|
|
url = str(getattr(request, "full_url", None) or request)
|
|
if any(host in url for host in BLOCKED_HOSTS):
|
|
raise AssertionError(f"website opened blocked host: {url}")
|
|
if HUB_BASE not in url:
|
|
raise AssertionError(f"unexpected url: {url}")
|
|
path = url.split(HUB_BASE, 1)[1].split("?", 1)[0]
|
|
if path == "/v1/bars/daily":
|
|
return {
|
|
"schema_version": 1,
|
|
"data": [
|
|
{
|
|
"ts_code": "600000.SH",
|
|
"trade_date": "20240902",
|
|
"open": 10.0,
|
|
"high": 10.4,
|
|
"low": 9.9,
|
|
"close": 10.2,
|
|
"volume": 100000,
|
|
"amount": 2000000,
|
|
}
|
|
],
|
|
"meta": {"stale": False, "staleness_seconds": 0, "source": "datahub"},
|
|
}
|
|
if path == "/v1/quotes/latest":
|
|
return {
|
|
"schema_version": 1,
|
|
"data": [
|
|
{
|
|
"ts_code": "600000.SH",
|
|
"code": "600000",
|
|
"name": "浦发银行",
|
|
"close": 10.2,
|
|
"price": 10.2,
|
|
"pre_close": 10.0,
|
|
"open": 10.1,
|
|
"high": 10.3,
|
|
"low": 9.9,
|
|
"vol": 1000,
|
|
"amount": 2000000,
|
|
"quote_date": "20240902",
|
|
"source": "datahub",
|
|
}
|
|
],
|
|
"meta": {"stale": False, "staleness_seconds": 0, "source": "eastmoney"},
|
|
}
|
|
if path == "/v1/indexes/quotes":
|
|
return {
|
|
"schema_version": 1,
|
|
"data": [
|
|
{
|
|
"ts_code": "000001.SH",
|
|
"code": "000001",
|
|
"name": "上证指数",
|
|
"price": 3000,
|
|
"pct_chg": 1.2,
|
|
"quote_time_epoch": 1725249600,
|
|
"source": "datahub",
|
|
},
|
|
{
|
|
"ts_code": "399001.SZ",
|
|
"code": "399001",
|
|
"name": "深证成指",
|
|
"price": 9000,
|
|
"pct_chg": 0.8,
|
|
"quote_time_epoch": 1725249600,
|
|
"source": "datahub",
|
|
},
|
|
{
|
|
"ts_code": "399006.SZ",
|
|
"code": "399006",
|
|
"name": "创业板指",
|
|
"price": 1800,
|
|
"pct_chg": 0.5,
|
|
"quote_time_epoch": 1725249600,
|
|
"source": "datahub",
|
|
},
|
|
],
|
|
"meta": {"stale": False, "staleness_seconds": 0, "source": "tencent"},
|
|
}
|
|
if path == "/v1/auction":
|
|
return {
|
|
"schema_version": 1,
|
|
"data": [
|
|
{
|
|
"ts_code": "600000.SH",
|
|
"trade_date": "20240902",
|
|
"close": 10.2,
|
|
"vol": 1000.0,
|
|
"amount": 2000.0,
|
|
}
|
|
],
|
|
"meta": {"stale": False, "staleness_seconds": 0, "source": "datahub"},
|
|
}
|
|
if path == "/v1/credentials/ifind":
|
|
return {
|
|
"schema_version": 1,
|
|
"data": {"configured": True, "access_ready": True, "access_expires_at": ""},
|
|
"meta": {"source": "ifind"},
|
|
}
|
|
if path == "/v1/intraday/points":
|
|
return {
|
|
"schema_version": 1,
|
|
"data": {
|
|
"entity_type": "stock",
|
|
"identifier": "600000",
|
|
"code": "600000",
|
|
"trade_date": "2024-09-02",
|
|
"previous_close": 10.0,
|
|
"points": [
|
|
{"date": "2024-09-02", "time": "09:30", "close": 10.2, "average": 10.1, "open": 10.1, "high": 10.2, "low": 10.0, "volume": 100, "amount": 1000}
|
|
],
|
|
},
|
|
"meta": {"stale": False, "source": "datahub"},
|
|
}
|
|
if path == "/v1/query":
|
|
body = json.loads(request.data.decode("utf-8") if request.data else "{}")
|
|
api_name = body.get("api_name")
|
|
if api_name == "ifind_status":
|
|
return {"schema_version": 1, "data": [{"configured": True, "access_ready": True, "access_expires_at": ""}], "meta": {"source": "ifind"}}
|
|
if api_name == "ifind_wencai":
|
|
return {
|
|
"schema_version": 1,
|
|
"data": [{"股票代码": "000001.SZ", "涨停原因": "重组", "首次涨停时间": "09:42:00"}],
|
|
"meta": {"source": "ifind"},
|
|
}
|
|
if api_name == "ifind_snapshots":
|
|
return {
|
|
"schema_version": 1,
|
|
"data": [
|
|
{
|
|
"time": "2024-09-02 09:21:00",
|
|
"thscode": "000001.SZ",
|
|
"latest": 10.5,
|
|
"preClose": 10,
|
|
"volume": 2000,
|
|
"amount": 21000,
|
|
}
|
|
],
|
|
"meta": {"source": "ifind"},
|
|
}
|
|
if api_name in {"daily", "rt_k", "stk_auction"}:
|
|
return {
|
|
"schema_version": 1,
|
|
"data": [{"ts_code": "600000.SH", "trade_date": "20240902", "close": 10.2, "amount": 2000.0}],
|
|
"meta": {"source": "datahub", "stale": False, "row_shape": "tushare"},
|
|
}
|
|
raise AssertionError(f"unexpected query api: {api_name}")
|
|
raise AssertionError(f"unexpected path: {path}")
|
|
|
|
|
|
def blocked_urlopen(request, timeout=None):
|
|
return _Resp(hub_payload(request))
|
|
|
|
|
|
class HubExclusiveWebsiteTests(unittest.TestCase):
|
|
def test_website_availability_depends_on_hub_not_provider_credentials(self) -> None:
|
|
service = SystemServiceMixin()
|
|
service._system_credentials = {}
|
|
service.data_gateway = build_data_gateway({}, datahub_settings=_enabled_settings())
|
|
self.assertTrue(service.configured)
|
|
self.assertEqual(service.token, "datahub")
|
|
|
|
def test_website_vault_scrubs_historical_provider_credentials(self) -> None:
|
|
class Vault:
|
|
@staticmethod
|
|
def decrypt_json(payload):
|
|
if isinstance(payload, str):
|
|
return json.loads(payload)
|
|
return dict(payload)
|
|
|
|
@staticmethod
|
|
def encrypt_json(payload):
|
|
return dict(payload)
|
|
|
|
class Database:
|
|
system = {
|
|
"tushare_token": "old-tushare",
|
|
"ifind_refresh_token": "old-refresh",
|
|
"ifind_access_token": "old-access",
|
|
}
|
|
personal = {"tushare_token": "old-personal", "llm_primary_model": "model"}
|
|
|
|
def get_system_setting(self, _name):
|
|
return dict(self.system)
|
|
|
|
@staticmethod
|
|
def first_user_id():
|
|
return 1
|
|
|
|
def get_user_credentials(self, _user_id):
|
|
return dict(self.personal)
|
|
|
|
@staticmethod
|
|
def list_user_credentials():
|
|
return [{"user_id": 1, "encrypted_payload": '{"tushare_token":"old-personal"}'}]
|
|
|
|
def save_system_setting(self, _name, payload):
|
|
self.system = dict(payload)
|
|
|
|
def save_user_credentials(self, _user_id, payload):
|
|
self.personal = dict(payload)
|
|
|
|
service = SystemServiceMixin()
|
|
service.database = Database()
|
|
service.vault = Vault()
|
|
service.system_lock = threading.Lock()
|
|
loaded = service._load_system_credentials({})
|
|
for key in ("tushare_token", "ifind_refresh_token", "ifind_access_token"):
|
|
self.assertNotIn(key, loaded)
|
|
self.assertNotIn(key, service.database.system)
|
|
self.assertNotIn(key, service.database.personal)
|
|
|
|
def test_query_never_calls_website_tushare_transport(self) -> None:
|
|
client = FakeClient()
|
|
legacy = FakeLegacy(TushareError("website tushare must stay dark"))
|
|
wrapped = DatahubAwareTushareClient(
|
|
legacy,
|
|
DatahubBridge(flags(daily=(True, False)), client),
|
|
)
|
|
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
|
|
self.assertEqual(rows[0]["amount"], 2000.0)
|
|
self.assertEqual(legacy.calls, [])
|
|
|
|
def test_blocked_external_hosts_still_read_hub(self) -> None:
|
|
settings = _enabled_settings()
|
|
hub_client = DatahubClient(settings, urlopen=blocked_urlopen)
|
|
legacy = FakeLegacy(TushareError("blocked"))
|
|
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(settings, hub_client))
|
|
with patch("urllib.request.urlopen", blocked_urlopen):
|
|
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,close,amount")
|
|
self.assertEqual(rows[0]["close"], 10.2)
|
|
self.assertEqual(rows[0]["amount"], 2000.0)
|
|
self.assertEqual(legacy.calls, [])
|
|
|
|
def test_site_features_only_reach_hub_when_external_hosts_blocked(self) -> None:
|
|
settings = _enabled_settings()
|
|
hub_client = DatahubClient(settings, urlopen=blocked_urlopen)
|
|
bridge = DatahubBridge(settings, hub_client)
|
|
with patch("urllib.request.urlopen", blocked_urlopen):
|
|
quotes = bridge.try_quotes(["600000.SH"])
|
|
indices = bridge.try_index_quotes()
|
|
chart = MarketChartClient(bridge).stock_daily("600000", "20240902")
|
|
intraday = MarketChartClient(bridge).stock_intraday("600000")
|
|
ifind = HubIfindProxy(bridge)
|
|
wencai = ifind.wencai("2024年9月2日涨停股票")
|
|
snapshots = ifind.snapshots(["000001.SZ"], ["latest"], "2024-09-02 09:15:00", "2024-09-02 09:22:00")
|
|
realtime = HubRealtimeProxy(bridge)
|
|
index_rows = realtime.tencent_indices()
|
|
stock = realtime.tencent_stock_quote("600000.SH", "20240902")
|
|
health = realtime.health_snapshot()
|
|
|
|
self.assertEqual(quotes[0]["close"], 10.2)
|
|
self.assertEqual(len(indices), 3)
|
|
self.assertEqual(chart[-1]["close"], 10.2)
|
|
self.assertEqual(intraday["source"], "datahub")
|
|
self.assertEqual(wencai[0]["股票代码"], "000001.SZ")
|
|
self.assertEqual(snapshots[0]["latest"], 10.5)
|
|
self.assertEqual([row["code"] for row in index_rows], ["000001", "399001", "399006"])
|
|
self.assertEqual(stock["close"], 10.2)
|
|
self.assertTrue(health["ready"])
|
|
self.assertTrue(ifind.configured)
|
|
|
|
def test_production_gateway_does_not_construct_external_clients(self) -> None:
|
|
source = (ROOT / "backend" / "data" / "gateway.py").read_text(encoding="utf-8")
|
|
self.assertNotIn("IfindHttpClient", source)
|
|
self.assertNotIn("EastmoneyChartClient", source)
|
|
self.assertNotIn("WebRealtimeAggregator", source)
|
|
self.assertNotIn("TushareProvider", source)
|
|
self.assertIsNone(re.search(r"(?<![A-Za-z])TushareClient\(", source))
|
|
self.assertIn("HubIfindProxy", source)
|
|
self.assertIn("HubRealtimeProxy", source)
|
|
self.assertIn("DatahubAwareTushareClient", source)
|
|
facade = (ROOT / "backend" / "data" / "datahub" / "bridge.py").read_text(encoding="utf-8")
|
|
tree = ast.parse(facade)
|
|
cls = next(
|
|
node
|
|
for node in tree.body
|
|
if isinstance(node, ast.ClassDef) and node.name == "DatahubAwareTushareClient"
|
|
)
|
|
methods = {item.name for item in cls.body if isinstance(item, ast.FunctionDef)}
|
|
self.assertNotIn("__getattr__", methods)
|
|
self.assertIn("query", methods)
|
|
self.assertTrue(any(base.id == "DashboardMixin" for base in cls.bases if isinstance(base, ast.Name)))
|
|
|
|
def test_production_python_does_not_embed_blocked_hosts(self) -> None:
|
|
violations = []
|
|
for path in (ROOT / "backend").rglob("*.py"):
|
|
relative = path.relative_to(ROOT).as_posix()
|
|
if relative in LEFTOVER_WEBSITE_FILES:
|
|
continue
|
|
text = path.read_text(encoding="utf-8")
|
|
for host in BLOCKED_HOSTS:
|
|
if host in text:
|
|
violations.append(f"{relative} -> {host}")
|
|
self.assertEqual(violations, [])
|
|
|
|
def test_website_runtime_does_not_call_blocked_hosts_from_gateway(self) -> None:
|
|
gateway_src = (ROOT / "backend" / "data" / "gateway.py").read_text(encoding="utf-8")
|
|
self.assertNotIn("TushareProvider", gateway_src)
|
|
self.assertIsNone(re.search(r"(?<![A-Za-z])TushareClient\(", gateway_src))
|
|
self.assertIn("DatahubAwareTushareClient", gateway_src)
|
|
|
|
def test_bridge_query_has_no_legacy_call(self) -> None:
|
|
source = (ROOT / "backend" / "data" / "datahub" / "bridge.py").read_text(encoding="utf-8")
|
|
tree = ast.parse(source)
|
|
query_fn = next(
|
|
node
|
|
for node in tree.body
|
|
if isinstance(node, ast.ClassDef) and node.name == "DatahubBridge"
|
|
for item in node.body
|
|
if isinstance(item, ast.FunctionDef) and item.name == "query"
|
|
)
|
|
called = [
|
|
ast.unparse(item.func) if hasattr(ast, "unparse") else ""
|
|
for item in ast.walk(query_fn)
|
|
if isinstance(item, ast.Call)
|
|
]
|
|
self.assertTrue(any("query_api" in text for text in called))
|
|
self.assertFalse(any("legacy_query" in text for text in called))
|
|
|
|
def test_build_gateway_uses_hub_proxies_without_opening_external_hosts(self) -> None:
|
|
settings = _enabled_settings()
|
|
with patch("urllib.request.urlopen", blocked_urlopen):
|
|
gateway = build_data_gateway({"tushare_token": "tok"}, datahub_settings=settings)
|
|
hub_client = DatahubClient(settings, urlopen=blocked_urlopen)
|
|
gateway.datahub.client = hub_client
|
|
rows = gateway.ifind.wencai("涨停")
|
|
quotes = gateway.realtime_observer.tencent_indices()
|
|
chart = gateway.chart_data.stock_daily("600000", "20240902")
|
|
market = gateway.tushare()
|
|
market_quotes = market.try_quotes(["600000.SH"])
|
|
auction = market.query("stk_auction", {"trade_date": "20240902"}, "")
|
|
self.assertEqual(rows[0]["涨停原因"], "重组")
|
|
self.assertEqual(len(quotes), 3)
|
|
self.assertEqual(chart[-1]["close"], 10.2)
|
|
self.assertEqual(market_quotes[0]["close"], 10.2)
|
|
self.assertEqual(auction[0]["close"], 10.2)
|
|
self.assertIsNone(market.realtime_aggregator)
|
|
self.assertEqual(market.token, "datahub")
|
|
|
|
def test_website_does_not_accept_or_forward_provider_credentials(self) -> None:
|
|
proxy = HubIfindProxy(DatahubBridge(_enabled_settings(), FakeClient()))
|
|
self.assertFalse(hasattr(proxy, "set_credentials"))
|
|
client_source = (ROOT / "backend" / "data" / "datahub" / "client.py").read_text(encoding="utf-8")
|
|
application_source = (ROOT / "backend" / "application.py").read_text(encoding="utf-8")
|
|
self.assertNotIn("/v1/credentials", client_source)
|
|
self.assertNotIn("ifind_refresh_token", application_source)
|
|
|
|
def test_site_configuration_depends_on_hub_not_provider_credentials(self) -> None:
|
|
service = SystemServiceMixin()
|
|
service._system_credentials = {}
|
|
service.data_gateway = build_data_gateway({}, datahub_settings=_enabled_settings())
|
|
self.assertTrue(service.configured)
|
|
self.assertEqual(service.token, "datahub")
|
|
|
|
def test_compose_passes_ifind_env_to_hub(self) -> None:
|
|
overlay = (ROOT / "compose.datahub.yaml").read_text(encoding="utf-8")
|
|
standalone = (ROOT / "xiaobai-datahub" / "compose.yaml").read_text(encoding="utf-8")
|
|
for text in (overlay, standalone):
|
|
self.assertIn('IFIND_REFRESH_TOKEN: "${IFIND_REFRESH_TOKEN:-}"', text)
|
|
self.assertIn('IFIND_ACCESS_TOKEN: "${IFIND_ACCESS_TOKEN:-}"', text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|