82 lines
3.6 KiB
Python
82 lines
3.6 KiB
Python
from __future__ import annotations
|
|
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from datahub.adapters.tushare import TushareAdapter
|
|
from datahub.crypto import SecretVault
|
|
from datahub.hub import Hub
|
|
from datahub.settings import Settings
|
|
from tests.fixtures import TRADE_DATE, fake_transport
|
|
|
|
|
|
class ExtendedEodTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
key = SecretVault.generate_key()
|
|
settings = Settings(
|
|
host="127.0.0.1",
|
|
port=0,
|
|
encryption_key=key,
|
|
api_token="k" * 32,
|
|
admin_password="StartPass1",
|
|
tushare_token="tushare-secret",
|
|
db_path=Path(self.tmp.name) / "hub.db",
|
|
backup_dir=Path(self.tmp.name) / "backups",
|
|
scheduler_enabled=False,
|
|
quality={"daily_row_ratio": 0.5, "null_rate_max": 0.5, "list_limit_default": 5000, "list_limit_max": 5000},
|
|
)
|
|
adapter = TushareAdapter("tushare-secret", transport=fake_transport)
|
|
self.hub = Hub(settings, adapter=adapter)
|
|
self.hub.pipeline.ingest_reference(TRADE_DATE)
|
|
for dataset in ("daily", "valuation", "moneyflow", "auction", "index_daily"):
|
|
self.hub.pipeline.run_dataset(dataset, TRADE_DATE)
|
|
|
|
def tearDown(self) -> None:
|
|
self.hub.stop()
|
|
self.tmp.cleanup()
|
|
|
|
def test_extended_soft_datasets_publish_and_serve(self) -> None:
|
|
results = self.hub.pipeline.run_extended_soft(
|
|
("limit_events", "popularity", "dragon_tiger", "sector_daily"),
|
|
TRADE_DATE,
|
|
)
|
|
for name in ("limit_events", "popularity", "dragon_tiger", "sector_daily"):
|
|
self.assertEqual(results[name]["state"], "published", results[name])
|
|
api = self.hub.api
|
|
limits = api.handle("/v1/limit-events", {"date": [TRADE_DATE]})
|
|
self.assertGreaterEqual(len(limits["data"]), 1)
|
|
self.assertEqual(limits["meta"]["tier"], "official")
|
|
pop = api.handle("/v1/popularity", {"date": [TRADE_DATE], "source": ["ths"]})
|
|
self.assertEqual(pop["data"][0]["source"], "ths")
|
|
lhb = api.handle("/v1/dragon-tiger", {"date": [TRADE_DATE]})
|
|
self.assertEqual(lhb["data"][0]["hm_name"], "测试游资")
|
|
# hub stores 万元→元
|
|
self.assertEqual(lhb["data"][0]["buy_amount"], 10_000_000.0)
|
|
sectors = api.handle("/v1/sectors", {"date": [TRADE_DATE], "family": ["ths"]})
|
|
self.assertEqual(sectors["data"][0]["family"], "ths")
|
|
status = api.handle("/v1/datasets/status", {"date": [TRADE_DATE]})
|
|
names = {item["dataset"] for item in status["data"]}
|
|
self.assertTrue({"limit_events", "popularity", "dragon_tiger", "sector_daily"} <= names)
|
|
|
|
def test_partial_sector_families_are_retried_instead_of_skipped(self) -> None:
|
|
first = self.hub.pipeline.run_eod_batch_e(TRADE_DATE)
|
|
self.assertEqual(first["sector_daily"]["state"], "published")
|
|
batch_id = self.hub.pipeline.active_batch("sector_daily", TRADE_DATE)
|
|
with self.hub.db.write() as connection:
|
|
connection.execute(
|
|
"DELETE FROM eod_sector_daily WHERE trade_date = ? AND batch_id = ? AND family = 'sw'",
|
|
(TRADE_DATE, batch_id),
|
|
)
|
|
self.assertFalse(self.hub.pipeline.extended_publication_complete("sector_daily", TRADE_DATE))
|
|
|
|
retried = self.hub.pipeline.run_eod_batch_e(TRADE_DATE)
|
|
|
|
self.assertEqual(retried["sector_daily"]["state"], "published")
|
|
self.assertTrue(self.hub.pipeline.extended_publication_complete("sector_daily", TRADE_DATE))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|