feat(HEL-457): 估值字段级质量门、股票主档每日发布和资金流历史回补
- field_gates 按数据集配置关键字段非空率下限/非有限比例/相对上一批次的塌陷保护, 字段大面积为空的批次拒发并保留上一正式批次,可读失败原因入 batches.error - 股票主档交易日 20:00/23:10 自动刷新并发布版本化快照(eod_stocks + publications), 覆盖新上市/简称变化/N前缀摘除;/v1/stocks 携带 batch_id/published_at,无变化跳过 - moneyflow 历史回补(默认 60 交易日,跳过已发布日期);未发布点查返回 available_from/available_to 与 history_not_backfilled 标记,缺失不再静默 - eod-refresh 新增 --force --dataset 安全重发(仍走全部质量门,上一批次可回滚) - 保持 HEL-435 盘后重试机制;新增 22 项测试覆盖字段拒发/正常通过/旧批保留/ 主档新增改名/资金流覆盖/重复执行幂等 Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
co-authored by
multica-agent
parent
c9892050c3
commit
bed6450992
@@ -60,7 +60,11 @@ class EodRetryTests(unittest.TestCase):
|
||||
return db.fetchall("SELECT * FROM job_runs WHERE job_id = ? ORDER BY id", (job_id,))
|
||||
|
||||
def _batches(self, db: HubDB, day: str) -> list[dict]:
|
||||
return db.fetchall("SELECT * FROM batches WHERE trade_date = ?", (day,))
|
||||
placeholders = ",".join("?" for _ in OFFICIAL)
|
||||
return db.fetchall(
|
||||
f"SELECT * FROM batches WHERE trade_date = ? AND dataset IN ({placeholders})",
|
||||
(day, *sorted(OFFICIAL)),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _batch_ids(db: HubDB, day: str) -> set[str]:
|
||||
@@ -71,7 +75,11 @@ class EodRetryTests(unittest.TestCase):
|
||||
return [name for name in transport.calls if name in DelayedTransport.DATE_APIS]
|
||||
|
||||
def _published(self, db: HubDB, day: str) -> set[str]:
|
||||
rows = db.fetchall("SELECT dataset FROM publications WHERE trade_date = ?", (day,))
|
||||
placeholders = ",".join("?" for _ in OFFICIAL)
|
||||
rows = db.fetchall(
|
||||
f"SELECT dataset FROM publications WHERE trade_date = ? AND dataset IN ({placeholders})",
|
||||
(day, *sorted(OFFICIAL)),
|
||||
)
|
||||
return {str(row["dataset"]) for row in rows}
|
||||
|
||||
def test_first_empty_then_retry_succeeds(self) -> None:
|
||||
@@ -105,12 +113,12 @@ class EodRetryTests(unittest.TestCase):
|
||||
|
||||
# success stops all further same-day requests
|
||||
batches_before = len(self._batches(db, day))
|
||||
calls_before = len(transport.calls)
|
||||
eod_calls_before = len(self._eod_calls(transport))
|
||||
sched.tick(clock_at(day, 17, 0))
|
||||
sched.tick(clock_at(day, 23, 0))
|
||||
self.assertEqual(len(self._job_runs(db, "eod_retry")), 2)
|
||||
self.assertEqual(len(self._batches(db, day)), batches_before)
|
||||
self.assertEqual(len(transport.calls), calls_before)
|
||||
self.assertEqual(len(self._eod_calls(transport)), eod_calls_before)
|
||||
|
||||
def test_never_ready_marks_cutoff_failed_and_stops(self) -> None:
|
||||
day = "20240902"
|
||||
@@ -144,9 +152,20 @@ class EodRetryTests(unittest.TestCase):
|
||||
sched.tick(clock_at(day, 15, 5)) # eod_a publishes 4 datasets
|
||||
sched.tick(clock_at(day, 15, 10)) # eod_b publishes index
|
||||
self.assertEqual(self._published(db, day), OFFICIAL)
|
||||
|
||||
def official_batches() -> list[str]:
|
||||
placeholders = ",".join("?" for _ in OFFICIAL)
|
||||
return [
|
||||
str(row["batch_id"])
|
||||
for row in db.fetchall(
|
||||
f"SELECT batch_id FROM batches WHERE trade_date = ? AND dataset IN ({placeholders})",
|
||||
(day, *sorted(OFFICIAL)),
|
||||
)
|
||||
]
|
||||
|
||||
active = db.fetchall("SELECT dataset, active_batch FROM publications WHERE trade_date = ?", (day,))
|
||||
active_map = {row["dataset"]: row["active_batch"] for row in active}
|
||||
batches_before = self._batch_ids(db, day)
|
||||
active_map = {row["dataset"]: row["active_batch"] for row in active if row["dataset"] in OFFICIAL}
|
||||
batches_before = set(official_batches())
|
||||
calls_before = self._eod_calls(transport)
|
||||
|
||||
# container restart: fresh scheduler, missed-time catch-up fires eod_a/eod_b
|
||||
@@ -157,8 +176,11 @@ class EodRetryTests(unittest.TestCase):
|
||||
self.assertNotIn("eod_retry", ran)
|
||||
self.assertEqual(self._published(db, day), OFFICIAL)
|
||||
after = db.fetchall("SELECT dataset, active_batch FROM publications WHERE trade_date = ?", (day,))
|
||||
self.assertEqual({row["dataset"]: row["active_batch"] for row in after}, active_map)
|
||||
self.assertEqual(self._batch_ids(db, day), batches_before) # no duplicate batches
|
||||
self.assertEqual(
|
||||
{row["dataset"]: row["active_batch"] for row in after if row["dataset"] in OFFICIAL},
|
||||
active_map,
|
||||
)
|
||||
self.assertEqual(set(official_batches()), batches_before) # no duplicate batches
|
||||
self.assertEqual(self._eod_calls(transport), calls_before) # no duplicate upstream EOD calls
|
||||
self.assertEqual(sched2.eod_status(day, clock=clock_at(day, 21, 0))["state"], "done")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user