feat(HEL-382): 搭建 datahub 底座和盘后正式数据链路
新增独立 xiaobai-datahub 服务（SQLite WAL、Tushare 盘后发布、/v1 契约和管理后台），不改现站页面与数据链路。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
co-authored by
Cursor
multica-agent
Cursor
multica-agent
parent
c2ebc0ab91
commit
3498dd7a4b
@@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
TRADE_DATE = "20240902"
|
||||
|
||||
RAW = {
|
||||
"trade_cal": [
|
||||
{"exchange": "SSE", "cal_date": "20240902", "is_open": 1, "pretrade_date": "20240830"},
|
||||
{"exchange": "SSE", "cal_date": "20240903", "is_open": 1, "pretrade_date": "20240902"},
|
||||
{"exchange": "SSE", "cal_date": "20240907", "is_open": 0, "pretrade_date": "20240906"},
|
||||
],
|
||||
"stock_basic": [
|
||||
{"ts_code": "600000.SH", "symbol": "600000", "name": "浦发银行", "area": "上海", "industry": "银行", "market": "主板", "list_status": "L", "list_date": "19991110"},
|
||||
{"ts_code": "000001.SZ", "symbol": "000001", "name": "平安银行", "area": "深圳", "industry": "银行", "market": "主板", "list_status": "L", "list_date": "19910403"},
|
||||
],
|
||||
"daily": [
|
||||
{"ts_code": "600000.SH", "trade_date": "20240902", "open": 10.11, "high": 10.25, "low": 10.01, "close": 10.20, "pct_chg": 1.2345, "vol": 1000.0, "amount": 2000.0},
|
||||
{"ts_code": "000001.SZ", "trade_date": "20240902", "open": 11.00, "high": 11.20, "low": 10.90, "close": 11.10, "pct_chg": -0.5, "vol": 2000.0, "amount": 4000.0},
|
||||
],
|
||||
"daily_basic": [
|
||||
{"ts_code": "600000.SH", "trade_date": "20240902", "turnover_rate": 1.2, "volume_ratio": 0.8, "total_mv": 1000.0, "circ_mv": 800.0, "pe_ttm": 5.1, "pb": 0.6, "ps_ttm": 1.1, "dv_ttm": 4.0},
|
||||
{"ts_code": "000001.SZ", "trade_date": "20240902", "turnover_rate": 2.2, "volume_ratio": 1.1, "total_mv": 2000.0, "circ_mv": 1500.0, "pe_ttm": 6.2, "pb": 0.7, "ps_ttm": 1.2, "dv_ttm": 3.0},
|
||||
],
|
||||
"adj_factor": [
|
||||
{"ts_code": "600000.SH", "trade_date": "20240902", "adj_factor": 1.1},
|
||||
{"ts_code": "000001.SZ", "trade_date": "20240902", "adj_factor": 2.0},
|
||||
],
|
||||
"index_daily": [
|
||||
{"ts_code": "000001.SH", "trade_date": "20240902", "open": 2700, "high": 2750, "low": 2690, "close": 2740, "pct_chg": 0.5, "vol": 3000.0, "amount": 500000.0},
|
||||
{"ts_code": "399001.SZ", "trade_date": "20240902", "open": 8000, "high": 8100, "low": 7900, "close": 8050, "pct_chg": 0.4, "vol": 2000.0, "amount": 300000.0},
|
||||
{"ts_code": "399006.SZ", "trade_date": "20240902", "open": 1600, "high": 1620, "low": 1580, "close": 1610, "pct_chg": 0.3, "vol": 1000.0, "amount": 100000.0},
|
||||
{"ts_code": "000300.SH", "trade_date": "20240902", "open": 3500, "high": 3550, "low": 3480, "close": 3520, "pct_chg": 0.2, "vol": 1500.0, "amount": 200000.0},
|
||||
],
|
||||
"moneyflow": [
|
||||
{"ts_code": "600000.SH", "trade_date": "20240902", "buy_sm_amount": 10, "sell_sm_amount": 8, "buy_md_amount": 20, "sell_md_amount": 15, "buy_lg_amount": 30, "sell_lg_amount": 25, "buy_elg_amount": 40, "sell_elg_amount": 35, "net_mf_amount": 17},
|
||||
{"ts_code": "000001.SZ", "trade_date": "20240902", "buy_sm_amount": 11, "sell_sm_amount": 9, "buy_md_amount": 21, "sell_md_amount": 16, "buy_lg_amount": 31, "sell_lg_amount": 26, "buy_elg_amount": 41, "sell_elg_amount": 36, "net_mf_amount": 18},
|
||||
],
|
||||
"stk_auction": [
|
||||
{"ts_code": "600000.SH", "trade_date": "20240902", "vol": 100, "price": 10.15, "amount": 1500000, "pre_close": 10.00, "turnover_rate": 0.1, "volume_ratio": 1.2, "float_share": 2000},
|
||||
{"ts_code": "000001.SZ", "trade_date": "20240902", "vol": 80, "price": 11.05, "amount": 1200000, "pre_close": 11.10, "turnover_rate": 0.2, "volume_ratio": 0.9, "float_share": 1800},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def fake_transport(api_name: str, params: dict, fields: str):
|
||||
if api_name == "index_daily":
|
||||
code = params.get("ts_code")
|
||||
return [row for row in RAW["index_daily"] if row["ts_code"] == code]
|
||||
if api_name == "trade_cal":
|
||||
start = str(params.get("start_date") or "")
|
||||
end = str(params.get("end_date") or "99999999")
|
||||
return [row for row in RAW["trade_cal"] if start <= row["cal_date"] <= end]
|
||||
return list(RAW.get(api_name) or [])
|
||||
@@ -0,0 +1,96 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from datahub.adapters.tushare import TushareAdapter
|
||||
from datahub.crypto import SecretVault
|
||||
from datahub.httpapp import make_handler
|
||||
from datahub.hub import Hub
|
||||
from datahub.settings import Settings
|
||||
from tests.fixtures import fake_transport
|
||||
|
||||
|
||||
class AdminTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
settings = Settings(
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="z" * 32,
|
||||
admin_password="StartPass1",
|
||||
tushare_token="real-tushare-token-abcdef",
|
||||
db_path=Path(self.tmp.name) / "hub.db",
|
||||
scheduler_enabled=False,
|
||||
)
|
||||
self.hub = Hub(settings, adapter=TushareAdapter("real-tushare-token-abcdef", transport=fake_transport))
|
||||
handler = make_handler(self.hub)
|
||||
self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
threading.Thread(target=self.server.serve_forever, daemon=True).start()
|
||||
self.base = f"http://127.0.0.1:{self.server.server_address[1]}"
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _json(self, path, method="GET", body=None, cookie="", csrf=""):
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if cookie:
|
||||
headers["Cookie"] = cookie
|
||||
if csrf:
|
||||
headers["X-CSRF-Token"] = csrf
|
||||
req = Request(self.base + path, data=data, headers=headers, method=method)
|
||||
with urlopen(req, timeout=5) as resp:
|
||||
set_cookie = resp.headers.get("Set-Cookie", "")
|
||||
return resp.status, json.loads(resp.read().decode()), set_cookie
|
||||
|
||||
def test_login_change_password_and_secret_masking(self) -> None:
|
||||
status, body, cookie_header = self._json(
|
||||
"/admin/api/login", "POST", {"username": "hub_admin", "password": "StartPass1"}
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertTrue(body["must_change"])
|
||||
cookie = cookie_header.split(";")[0]
|
||||
csrf = body["csrf"]
|
||||
status, _, _ = self._json(
|
||||
"/admin/api/change-password",
|
||||
"POST",
|
||||
{"current": "StartPass1", "new_password": "NewPass123"},
|
||||
cookie=cookie,
|
||||
csrf=csrf,
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
_, sources, _ = self._json("/admin/api/sources", cookie=cookie, csrf=csrf)
|
||||
blob = json.dumps(sources)
|
||||
self.assertNotIn("real-tushare-token-abcdef", blob)
|
||||
self.assertTrue(sources["items"][0]["credential"]["configured"])
|
||||
self.assertTrue(str(sources["items"][0]["credential"]["last4"]).endswith("cdef") or "****" in str(sources["items"][0]["credential"]["last4"]))
|
||||
|
||||
def test_rollback_requires_password_and_confirm(self) -> None:
|
||||
_, body, cookie_header = self._json(
|
||||
"/admin/api/login", "POST", {"username": "hub_admin", "password": "StartPass1"}
|
||||
)
|
||||
cookie = cookie_header.split(";")[0]
|
||||
csrf = body["csrf"]
|
||||
self._json("/admin/api/change-password", "POST", {"current": "StartPass1", "new_password": "NewPass123"}, cookie, csrf)
|
||||
from urllib.error import HTTPError
|
||||
|
||||
with self.assertRaises(HTTPError) as ctx:
|
||||
self._json(
|
||||
"/admin/api/rollback",
|
||||
"POST",
|
||||
{"dataset": "daily", "trade_date": "20240902", "password": "wrong", "confirm": "daily:20240902"},
|
||||
cookie,
|
||||
csrf,
|
||||
)
|
||||
self.assertEqual(ctx.exception.code, 401)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,147 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from datahub.adapters.tushare import TushareAdapter
|
||||
from datahub.crypto import SecretVault
|
||||
from datahub.httpapp import make_handler
|
||||
from datahub.hub import Hub
|
||||
from datahub.settings import Settings
|
||||
from tests.fixtures import TRADE_DATE, fake_transport
|
||||
|
||||
ERROR_CODES = {
|
||||
"UNAUTHORIZED",
|
||||
"INVALID_ARGUMENT",
|
||||
"RATE_LIMITED",
|
||||
"SOURCE_UNAVAILABLE",
|
||||
"DATASET_NOT_PUBLISHED",
|
||||
"STALE_DATA",
|
||||
"INTERNAL",
|
||||
}
|
||||
|
||||
|
||||
class ApiContractTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
key = SecretVault.generate_key()
|
||||
self.token = "k" * 32
|
||||
settings = Settings(
|
||||
host="127.0.0.1",
|
||||
port=0,
|
||||
encryption_key=key,
|
||||
api_token=self.token,
|
||||
admin_password="StartPass1",
|
||||
tushare_token="tushare-secret-token-xyz",
|
||||
db_path=Path(self.tmp.name) / "hub.db",
|
||||
backup_dir=Path(self.tmp.name) / "backups",
|
||||
scheduler_enabled=False,
|
||||
quality={"daily_row_ratio": 0.5, "null_rate_max": 0.5, "list_limit_default": 5000, "list_limit_max": 5000},
|
||||
)
|
||||
adapter = TushareAdapter("tushare-secret-token-xyz", transport=fake_transport)
|
||||
self.hub = Hub(settings, adapter=adapter)
|
||||
self.hub.pipeline.ingest_reference(TRADE_DATE)
|
||||
for dataset in ("daily", "valuation", "moneyflow", "auction", "index_daily"):
|
||||
self.hub.pipeline.run_dataset(dataset, TRADE_DATE)
|
||||
handler = make_handler(self.hub)
|
||||
self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
self.base = f"http://127.0.0.1:{self.server.server_address[1]}"
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.hub.stop()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _get(self, path: str, token: str | None = None) -> tuple[int, dict]:
|
||||
headers = {}
|
||||
if token is not None:
|
||||
headers["X-Datahub-Token"] = token
|
||||
req = Request(self.base + path, headers=headers)
|
||||
try:
|
||||
with urlopen(req, timeout=5) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except HTTPError as exc:
|
||||
return exc.code, json.loads(exc.read().decode())
|
||||
|
||||
def test_livez_no_token(self) -> None:
|
||||
status, body = self._get("/livez", token=None)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body["status"], "ok")
|
||||
|
||||
def test_missing_and_bad_token_401(self) -> None:
|
||||
status, body = self._get("/v1/health", token=None)
|
||||
self.assertEqual(status, 401)
|
||||
self.assertEqual(body["error"]["code"], "UNAUTHORIZED")
|
||||
status, body = self._get("/v1/health", token="wrong")
|
||||
self.assertEqual(status, 401)
|
||||
self.assertNotIn("tushare-secret-token-xyz", json.dumps(body))
|
||||
self.assertNotIn(self.token, json.dumps(body))
|
||||
|
||||
def test_core_endpoints_schema(self) -> None:
|
||||
paths = [
|
||||
"/v1/health",
|
||||
f"/v1/calendar?from=20240901&to=20240907",
|
||||
"/v1/stocks",
|
||||
f"/v1/bars/daily?date={TRADE_DATE}&code=600000.SH&adjust=none",
|
||||
f"/v1/bars/daily?date={TRADE_DATE}&code=600000.SH&adjust=qfq",
|
||||
f"/v1/indexes/bars?date={TRADE_DATE}&code=000001.SH",
|
||||
f"/v1/valuation?date={TRADE_DATE}&code=600000.SH",
|
||||
f"/v1/moneyflow?date={TRADE_DATE}&code=600000.SH",
|
||||
f"/v1/auction?date={TRADE_DATE}",
|
||||
f"/v1/datasets/status?date={TRADE_DATE}",
|
||||
f"/v1/batches?date={TRADE_DATE}",
|
||||
]
|
||||
for path in paths:
|
||||
status, body = self._get(path, token=self.token)
|
||||
self.assertEqual(status, 200, path)
|
||||
self.assertEqual(body["schema_version"], 1)
|
||||
self.assertIn("data", body)
|
||||
self.assertIn("meta", body)
|
||||
self.assertIn("tier", body["meta"])
|
||||
|
||||
def test_qfq_matches_formula(self) -> None:
|
||||
_, none = self._get(f"/v1/bars/daily?date={TRADE_DATE}&code=600000.SH&adjust=none", token=self.token)
|
||||
_, qfq = self._get(f"/v1/bars/daily?date={TRADE_DATE}&code=600000.SH&adjust=qfq", token=self.token)
|
||||
raw = none["data"][0]
|
||||
adj = qfq["data"][0]
|
||||
expected = round(raw["close"] * raw["adj_factor"] / raw["adj_factor"], 4)
|
||||
self.assertEqual(adj["close"], expected)
|
||||
|
||||
def test_unpublished_code(self) -> None:
|
||||
status, body = self._get("/v1/bars/daily?date=19990101", token=self.token)
|
||||
self.assertEqual(status, 404)
|
||||
self.assertEqual(body["error"]["code"], "DATASET_NOT_PUBLISHED")
|
||||
self.assertIn("expected_at", body["error"])
|
||||
|
||||
def test_error_code_set_documented(self) -> None:
|
||||
self.assertGreaterEqual(ERROR_CODES, {"UNAUTHORIZED", "DATASET_NOT_PUBLISHED", "INVALID_ARGUMENT"})
|
||||
|
||||
def test_six_digit_code(self) -> None:
|
||||
status, body = self._get(f"/v1/bars/daily?date={TRADE_DATE}&code=600000", token=self.token)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body["data"][0]["ts_code"], "600000.SH")
|
||||
|
||||
def test_amount_unit_is_yuan(self) -> None:
|
||||
_, body = self._get(f"/v1/bars/daily?date={TRADE_DATE}&code=600000.SH", token=self.token)
|
||||
self.assertEqual(body["data"][0]["amount"], 2_000_000.0)
|
||||
_, flow = self._get(f"/v1/moneyflow?date={TRADE_DATE}&code=600000.SH", token=self.token)
|
||||
self.assertEqual(flow["data"][0]["net_mf_amount"], 170000.0)
|
||||
|
||||
def test_token_never_in_health_or_admin_sources(self) -> None:
|
||||
_, health = self._get("/v1/health", token=self.token)
|
||||
blob = json.dumps(health)
|
||||
self.assertNotIn("tushare-secret-token-xyz", blob)
|
||||
self.assertNotIn(self.token, blob)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from datahub.governance.circuit import CircuitBreaker
|
||||
from datahub.governance.ratelimit import TokenBucket
|
||||
from datahub.governance.retry import RetryError, retry_call
|
||||
|
||||
|
||||
class FakeClock:
|
||||
def __init__(self) -> None:
|
||||
self.value = 0.0
|
||||
|
||||
def __call__(self) -> float:
|
||||
return self.value
|
||||
|
||||
|
||||
class GovernanceTests(unittest.TestCase):
|
||||
def test_token_bucket_caps_burst_at_capacity(self) -> None:
|
||||
clock = FakeClock()
|
||||
bucket = TokenBucket(rate_per_minute=300, capacity=300, clock=clock)
|
||||
ok = 0
|
||||
for _ in range(400):
|
||||
if bucket.acquire(block=False):
|
||||
ok += 1
|
||||
self.assertEqual(ok, 300)
|
||||
clock.value = 60
|
||||
self.assertTrue(bucket.acquire(block=False))
|
||||
|
||||
def test_circuit_opens_after_five_failures_and_half_opens(self) -> None:
|
||||
clock = FakeClock()
|
||||
breaker = CircuitBreaker(clock=clock, open_seconds=120)
|
||||
for _ in range(5):
|
||||
breaker.record_failure("boom")
|
||||
self.assertEqual(breaker.snapshot().state, "open")
|
||||
self.assertFalse(breaker.allow())
|
||||
clock.value = 120
|
||||
self.assertEqual(breaker.snapshot().state, "half_open")
|
||||
self.assertTrue(breaker.allow())
|
||||
breaker.record_success()
|
||||
self.assertEqual(breaker.snapshot().state, "closed")
|
||||
|
||||
def test_retry_exhausts(self) -> None:
|
||||
calls = {"n": 0}
|
||||
|
||||
def fail():
|
||||
calls["n"] += 1
|
||||
raise RuntimeError("no")
|
||||
|
||||
with self.assertRaises(RetryError):
|
||||
retry_call(fail, attempts=3, sleeper=lambda _d: None)
|
||||
self.assertEqual(calls["n"], 3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
class LayoutTests(unittest.TestCase):
|
||||
def test_dockerfile_and_compose_exist(self) -> None:
|
||||
self.assertTrue((ROOT / "Dockerfile").is_file())
|
||||
self.assertTrue((ROOT / "compose.yaml").is_file())
|
||||
self.assertTrue((ROOT / "requirements.txt").read_text(encoding="utf-8").startswith("cryptography=="))
|
||||
dockerfile = (ROOT / "Dockerfile").read_text(encoding="utf-8")
|
||||
self.assertIn("10002", dockerfile)
|
||||
self.assertIn("8766", dockerfile)
|
||||
self.assertIn("livez", dockerfile)
|
||||
|
||||
def test_reserved_adapters_present(self) -> None:
|
||||
from datahub.adapters import RESERVED
|
||||
|
||||
for name in ("eastmoney", "tencent", "ths", "xgb", "akshare", "ifind"):
|
||||
self.assertIn(name, RESERVED)
|
||||
probe = RESERVED[name].probe()
|
||||
self.assertEqual(probe["state"], "reserved")
|
||||
self.assertFalse(probe["configured"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from datahub.normalize import (
|
||||
AMOUNT_THOUSAND_YUAN,
|
||||
AMOUNT_WAN_YUAN,
|
||||
VOLUME_LOT,
|
||||
apply_qfq,
|
||||
normalize_auction,
|
||||
normalize_daily,
|
||||
normalize_index_daily,
|
||||
normalize_moneyflow,
|
||||
normalize_valuation,
|
||||
review_daily_to_canonical,
|
||||
)
|
||||
from tests.fixtures import RAW
|
||||
|
||||
|
||||
class NormalizeTests(unittest.TestCase):
|
||||
def test_daily_matches_architecture_and_review_native_conversion(self) -> None:
|
||||
raw = RAW["daily"][0]
|
||||
hub = normalize_daily(raw, adj_factor=1.1)
|
||||
# review stores Tushare native units; canonical = review * factor
|
||||
review_native = dict(raw)
|
||||
converted = review_daily_to_canonical(review_native)
|
||||
self.assertEqual(hub["amount"], converted["amount"])
|
||||
self.assertEqual(hub["amount"], raw["amount"] * AMOUNT_THOUSAND_YUAN)
|
||||
self.assertEqual(hub["volume"], raw["vol"] * VOLUME_LOT)
|
||||
self.assertEqual(hub["close"], 10.2)
|
||||
self.assertEqual(hub["adj_factor"], 1.1)
|
||||
self.assertEqual(hub["ts_code"], "600000.SH")
|
||||
|
||||
def test_moneyflow_wan_to_yuan(self) -> None:
|
||||
raw = RAW["moneyflow"][0]
|
||||
hub = normalize_moneyflow(raw)
|
||||
self.assertEqual(hub["net_mf_amount"], raw["net_mf_amount"] * AMOUNT_WAN_YUAN)
|
||||
self.assertEqual(hub["buy_lg_amount"], 30 * AMOUNT_WAN_YUAN)
|
||||
|
||||
def test_valuation_mv_wan_to_yuan(self) -> None:
|
||||
raw = RAW["daily_basic"][0]
|
||||
hub = normalize_valuation(raw)
|
||||
self.assertEqual(hub["total_mv"], 1000 * AMOUNT_WAN_YUAN)
|
||||
self.assertEqual(hub["circ_mv"], 800 * AMOUNT_WAN_YUAN)
|
||||
|
||||
def test_index_daily_amount_thousand_yuan(self) -> None:
|
||||
raw = RAW["index_daily"][0]
|
||||
hub = normalize_index_daily(raw)
|
||||
self.assertEqual(hub["amount"], raw["amount"] * AMOUNT_THOUSAND_YUAN)
|
||||
self.assertEqual(hub["volume"], raw["vol"] * VOLUME_LOT)
|
||||
|
||||
def test_auction_amount_already_yuan(self) -> None:
|
||||
raw = RAW["stk_auction"][0]
|
||||
hub = normalize_auction(raw)
|
||||
self.assertEqual(hub["amount"], raw["amount"])
|
||||
self.assertEqual(hub["volume"], raw["vol"] * VOLUME_LOT)
|
||||
|
||||
def test_field_diff_against_review_native_is_explained(self) -> None:
|
||||
"""Golden: every non-zero diff vs review-native daily is a documented unit factor."""
|
||||
raw = RAW["daily"][0]
|
||||
hub = normalize_daily(raw)
|
||||
diffs = {}
|
||||
for key in ("open", "high", "low", "close", "pct_chg"):
|
||||
if hub[key] != raw[key]:
|
||||
diffs[key] = (raw[key], hub[key])
|
||||
self.assertEqual(diffs, {})
|
||||
self.assertNotEqual(hub["amount"], raw["amount"])
|
||||
self.assertEqual(hub["amount"] / raw["amount"], AMOUNT_THOUSAND_YUAN)
|
||||
self.assertEqual(hub["volume"] / raw["vol"], VOLUME_LOT)
|
||||
|
||||
def test_qfq_formula(self) -> None:
|
||||
self.assertEqual(apply_qfq(10.0, 1.1, 2.2), 5.0)
|
||||
none_price = apply_qfq(None, 1.1, 2.2)
|
||||
self.assertIsNone(none_price)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,108 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from datahub.adapters.tushare import TushareAdapter
|
||||
from datahub.crypto import SecretVault
|
||||
from datahub.db import HubDB
|
||||
from datahub.pipeline import Pipeline, QualityError
|
||||
from datahub.settings import Settings
|
||||
from tests.fixtures import TRADE_DATE, fake_transport
|
||||
|
||||
|
||||
def make_pipeline(before_commit=None) -> tuple[Pipeline, HubDB]:
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
db = HubDB(Path(tmp.name) / "hub.db")
|
||||
adapter = TushareAdapter("test-token", transport=fake_transport)
|
||||
settings = Settings(
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="t" * 32,
|
||||
admin_password="admin-pass",
|
||||
tushare_token="test-token",
|
||||
db_path=db.path,
|
||||
quality={"daily_row_ratio": 0.98, "null_rate_max": 0.01, "max_publish_attempts": 3, "publication_generations": 3},
|
||||
scheduler_enabled=False,
|
||||
)
|
||||
pipe = Pipeline(db, adapter, settings, before_commit=before_commit)
|
||||
pipe._tmp = tmp # keep alive
|
||||
return pipe, db
|
||||
|
||||
|
||||
class PipelineTests(unittest.TestCase):
|
||||
def test_reference_and_daily_publish(self) -> None:
|
||||
pipe, db = make_pipeline()
|
||||
ref = pipe.ingest_reference(TRADE_DATE)
|
||||
self.assertEqual(ref["stocks"], 2)
|
||||
result = pipe.run_dataset("daily", TRADE_DATE)
|
||||
self.assertEqual(result["state"], "published")
|
||||
self.assertEqual(result["rows"], 2)
|
||||
pub = db.fetchone("SELECT * FROM publications WHERE dataset='daily' AND trade_date=?", (TRADE_DATE,))
|
||||
self.assertEqual(pub["active_batch"], result["batch_id"])
|
||||
rows = db.fetchall("SELECT * FROM eod_bars WHERE batch_id=?", (result["batch_id"],))
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertEqual(rows[0]["amount"] if rows[0]["ts_code"] == "600000.SH" else rows[1]["amount"], 2_000_000.0)
|
||||
|
||||
def test_atomic_publish_abort_leaves_no_half_batch(self) -> None:
|
||||
pipe, db = make_pipeline()
|
||||
pipe.ingest_reference(TRADE_DATE)
|
||||
first = pipe.run_dataset("daily", TRADE_DATE)
|
||||
boom = {"n": 0}
|
||||
|
||||
def explode() -> None:
|
||||
boom["n"] += 1
|
||||
raise RuntimeError("killed")
|
||||
|
||||
pipe.before_commit = explode
|
||||
with self.assertRaises(RuntimeError):
|
||||
pipe.run_dataset("daily", TRADE_DATE)
|
||||
pub = db.fetchone("SELECT * FROM publications WHERE dataset='daily' AND trade_date=?", (TRADE_DATE,))
|
||||
self.assertEqual(pub["active_batch"], first["batch_id"])
|
||||
visible = db.fetchall(
|
||||
"SELECT DISTINCT batch_id FROM eod_bars WHERE trade_date=? AND batch_id=?",
|
||||
(TRADE_DATE, pub["active_batch"]),
|
||||
)
|
||||
self.assertEqual(len(visible), 1)
|
||||
|
||||
def test_rollback_switches_active_batch(self) -> None:
|
||||
pipe, _db = make_pipeline()
|
||||
pipe.ingest_reference(TRADE_DATE)
|
||||
first = pipe.run_dataset("daily", TRADE_DATE)
|
||||
second = pipe.run_dataset("daily", TRADE_DATE)
|
||||
self.assertNotEqual(first["batch_id"], second["batch_id"])
|
||||
rolled = pipe.rollback("daily", TRADE_DATE, actor="test")
|
||||
self.assertEqual(rolled["active_batch"], first["batch_id"])
|
||||
from datahub.serving import V1API
|
||||
|
||||
api = V1API(pipe.db, pipe, pipe.settings)
|
||||
payload = api.handle("/v1/bars/daily", {"date": [TRADE_DATE], "code": ["600000.SH"]})
|
||||
self.assertEqual(payload["meta"]["batch_id"], first["batch_id"])
|
||||
|
||||
def test_row_ratio_gate_rejects_short_batch(self) -> None:
|
||||
pipe, _db = make_pipeline()
|
||||
pipe.ingest_reference(TRADE_DATE)
|
||||
original = fake_transport
|
||||
|
||||
def short(api_name, params, fields):
|
||||
rows = original(api_name, params, fields)
|
||||
if api_name == "daily":
|
||||
return rows[:1]
|
||||
return rows
|
||||
|
||||
pipe.adapter._transport = short
|
||||
with self.assertRaises(QualityError) as ctx:
|
||||
pipe.run_dataset("daily", TRADE_DATE)
|
||||
self.assertTrue(ctx.exception.report["hard_fail"])
|
||||
pub = pipe.db.fetchone("SELECT * FROM publications WHERE dataset='daily'")
|
||||
self.assertIsNone(pub)
|
||||
|
||||
def test_wal_mode(self) -> None:
|
||||
pipe, db = make_pipeline()
|
||||
with db.connect() as connection:
|
||||
mode = connection.execute("PRAGMA journal_mode").fetchone()[0]
|
||||
self.assertEqual(str(mode).lower(), "wal")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
from datahub.adapters.tushare import TushareAdapter
|
||||
from datahub.crypto import SecretVault
|
||||
from datahub.db import HubDB
|
||||
from datahub.pipeline import Pipeline
|
||||
from datahub.scheduler import Scheduler
|
||||
from datahub.settings import Settings
|
||||
from datahub.timeutil import SHANGHAI
|
||||
from tests.fixtures import fake_transport
|
||||
|
||||
|
||||
class SchedulerTests(unittest.TestCase):
|
||||
def test_skips_eod_on_closed_day(self) -> None:
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
db = HubDB(Path(tmp.name) / "hub.db")
|
||||
adapter = TushareAdapter("x", transport=fake_transport)
|
||||
settings = Settings(encryption_key=SecretVault.generate_key(), scheduler_enabled=False, db_path=db.path)
|
||||
pipe = Pipeline(db, adapter, settings)
|
||||
pipe.ingest_reference("20240902")
|
||||
# 20240907 is closed in fixture
|
||||
ran = {"eod_a": 0}
|
||||
|
||||
def fake_eod(_date: str):
|
||||
ran["eod_a"] += 1
|
||||
return {}
|
||||
|
||||
sched = Scheduler(db, pipe, jobs={"precheck": lambda d: {}, "eod_a": fake_eod, "eod_b": lambda d: {}, "cleanup": lambda d: {}, "backup": lambda d: {}})
|
||||
clock = datetime(2024, 9, 7, 16, 0, tzinfo=SHANGHAI)
|
||||
fired = sched.tick(clock)
|
||||
self.assertNotIn("eod_a", fired)
|
||||
self.assertEqual(ran["eod_a"], 0)
|
||||
tmp.cleanup()
|
||||
|
||||
def test_fires_eod_on_open_day(self) -> None:
|
||||
tmp = tempfile.TemporaryDirectory()
|
||||
db = HubDB(Path(tmp.name) / "hub.db")
|
||||
adapter = TushareAdapter("x", transport=fake_transport)
|
||||
settings = Settings(encryption_key=SecretVault.generate_key(), scheduler_enabled=False, db_path=db.path)
|
||||
pipe = Pipeline(db, adapter, settings)
|
||||
pipe.ingest_reference("20240902")
|
||||
ran = {"eod_a": 0}
|
||||
|
||||
def fake_eod(_date: str):
|
||||
ran["eod_a"] += 1
|
||||
return {"rows": 1}
|
||||
|
||||
sched = Scheduler(db, pipe, jobs={"precheck": lambda d: {}, "eod_a": fake_eod, "eod_b": lambda d: {}, "cleanup": lambda d: {}, "backup": lambda d: {}})
|
||||
clock = datetime(2024, 9, 2, 16, 0, tzinfo=SHANGHAI)
|
||||
fired = sched.tick(clock)
|
||||
self.assertIn("eod_a", fired)
|
||||
self.assertEqual(ran["eod_a"], 1)
|
||||
tmp.cleanup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user