leefer/xiaobai-review
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(HEL-461): 整批发布按完整边界重暂存，主档与快照同事务

Browse Source 
边界内任有缺失则整组重暂存后统一切换，避免旧新批次混发；
refresh_stocks 失败时主档保持旧值，并补齐回归测试。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
2026-09-05 08:52:50 +08:00
co-authored by Cursor multica-agent
parent 16841e9ae3
commit 32f565ecb9
3 changed files with 167 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
xiaobai-datahub/tests/test_atomic_release.py
+88 -8
View File
@@ -21,14 +21,14 @@ class GroupTransport:
def __init__(self) -> None:
self.empty: set[str] = set()
self.keep_rows: dict[str, int] = {}
self.stocks: list[dict] = []
self.stocks: list[dict] | None = None
self.calls: list[str] = []
def __call__(self, api_name: str, params: dict, fields: str):
self.calls.append(api_name)
if api_name in self.empty:
return []
if api_name == "stock_basic" and self.stocks:
if api_name == "stock_basic" and self.stocks is not None:
return [dict(row) for row in self.stocks]
rows = fake_transport(api_name, params, fields)
keep = self.keep_rows.get(api_name)
@@ -113,19 +113,48 @@ class ReleaseGroupSwitchTests(unittest.TestCase):
first = self.pipe.run_dataset("daily", TRADE_DATE)
self.transport.empty = {"daily_basic"}
results = self.pipe.run_eod_missing(TRADE_DATE)
self.assertEqual(results["daily"]["state"], "skipped")
# incomplete A-group restages daily with the others; valuation fails → no A switch
self.assertEqual(results["daily"]["state"], "failed")
self.assertEqual(results["valuation"]["state"], "failed")
# the already-published complete batch is untouched and keeps serving
# the already-published daily batch is untouched and keeps serving
self.assertEqual(self.pipe.active_batch("daily", TRADE_DATE), first["batch_id"])
self.assertEqual(
publications_map(self.db, TRADE_DATE),
{"daily": first["batch_id"]},
)
pubs = publications_map(self.db, TRADE_DATE)
self.assertEqual(pubs["daily"], first["batch_id"])
self.assertNotIn("valuation", pubs)
self.assertNotIn("moneyflow", pubs)
self.assertNotIn("auction", pubs)
# B-group is an independent boundary and may still publish
self.assertEqual(results["index_daily"]["state"], "published")
payload = V1API(self.db, self.pipe, self.pipe.settings).handle(
"/v1/bars/daily", {"date": [TRADE_DATE], "code": ["600000.SH"]}
)
self.assertEqual(payload["meta"]["batch_id"], first["batch_id"])
def test_partial_group_retry_does_not_mix_batches(self) -> None:
"""Already-published A members must be restaged with missing ones."""
first_daily = self.pipe.run_dataset("daily", TRADE_DATE)
first_moneyflow = self.pipe.run_dataset("moneyflow", TRADE_DATE)
results = self.pipe.run_eod_missing(TRADE_DATE)
# A-group switched as one boundary; B-group (index) also published
for name in (*GROUP_A, "stocks"):
self.assertEqual(results[name]["state"], "published", name)
self.assertEqual(results["index_daily"]["state"], "published")
pubs = self.db.fetchall(
"SELECT dataset, active_batch, published_at FROM publications WHERE trade_date = ?",
(TRADE_DATE,),
)
by_ds = {str(row["dataset"]): row for row in pubs}
# old partial batches replaced — no cross-batch mix of the first wave
self.assertNotEqual(by_ds["daily"]["active_batch"], first_daily["batch_id"])
self.assertNotEqual(by_ds["moneyflow"]["active_batch"], first_moneyflow["batch_id"])
a_times = {by_ds[name]["published_at"] for name in (*GROUP_A, "stocks")}
self.assertEqual(len(a_times), 1)
# serving resolves the new complete A-group batches
api = V1API(self.db, self.pipe, self.pipe.settings)
daily = api.handle("/v1/bars/daily", {"date": [TRADE_DATE], "code": ["600000.SH"]})
self.assertEqual(daily["meta"]["batch_id"], results["daily"]["batch_id"])
self.assertEqual(daily["meta"]["batch_id"], by_ds["daily"]["active_batch"])
def test_reads_during_switch_see_old_state_until_commit(self) -> None:
snapshots: list[dict] = []
@@ -231,5 +260,56 @@ class ReleaseGroupSwitchTests(unittest.TestCase):
self.assertIsNone(stocks_pub)
class StocksRefreshAtomicTests(unittest.TestCase):
def setUp(self) -> None:
self.transport = GroupTransport()
self.pipe, self.db = make_pipe(self.transport)
self.pipe.ingest_reference(TRADE_DATE)
self.transport.stocks = [
{"ts_code": "600000.SH", "symbol": "600000", "name": "浦发银行", "area": "上海",
"industry": "银行", "market": "主板", "list_status": "L", "list_date": "19991110"},
{"ts_code": "920071.BJ", "symbol": "920071", "name": "N金钛", "area": "辽宁",
"industry": "小金属", "market": "北交所", "list_status": "L", "list_date": "20240901"},
]
first = self.pipe.refresh_stocks(TRADE_DATE)
self.assertEqual(first["state"], "published")
self.first_batch = first["batch_id"]
def test_refresh_keeps_master_when_snapshot_publish_fails(self) -> None:
self.transport.stocks = [
{"ts_code": "600000.SH", "symbol": "600000", "name": "浦发银行", "area": "上海",
"industry": "银行", "market": "主板", "list_status": "L", "list_date": "19991110"},
{"ts_code": "920071.BJ", "symbol": "920071", "name": "金钛股份", "area": "辽宁",
"industry": "小金属", "market": "北交所", "list_status": "L", "list_date": "20240901"},
]
def explode() -> None:
raise RuntimeError("snapshot switch killed")
self.pipe.before_commit = explode
with self.assertRaises(RuntimeError):
self.pipe.refresh_stocks(TRADE_DATE)
master = self.db.fetchone("SELECT name FROM stock_master WHERE ts_code = '920071.BJ'")
self.assertEqual(master["name"], "N金钛") # rename not applied
self.assertEqual(self.pipe.active_batch("stocks", TRADE_DATE), self.first_batch)
audit = self.db.fetchone(
"SELECT * FROM audit_log WHERE action = 'stocks-refresh' ORDER BY id DESC"
)
self.assertIn("failed", str(audit["detail"]))
self.assertIn("snapshot switch killed", str(audit["detail"]))
def test_refresh_keeps_master_when_quality_gate_rejects(self) -> None:
self.transport.stocks = [] # empty → hard fail before publish
with self.assertRaises(Exception):
self.pipe.refresh_stocks(TRADE_DATE)
master = self.db.fetchone("SELECT name FROM stock_master WHERE ts_code = '920071.BJ'")
self.assertEqual(master["name"], "N金钛")
self.assertEqual(self.pipe.active_batch("stocks", TRADE_DATE), self.first_batch)
audit = self.db.fetchone(
"SELECT * FROM audit_log WHERE action = 'stocks-refresh' ORDER BY id DESC"
)
self.assertIn("failed", str(audit["detail"]))
if __name__ == "__main__":
unittest.main()