feat(HEL-457): 估值字段级质量门、股票主档每日发布和资金流历史回补

- field_gates 按数据集配置关键字段非空率下限/非有限比例/相对上一批次的塌陷保护,
  字段大面积为空的批次拒发并保留上一正式批次,可读失败原因入 batches.error
- 股票主档交易日 20:00/23:10 自动刷新并发布版本化快照(eod_stocks + publications),
  覆盖新上市/简称变化/N前缀摘除;/v1/stocks 携带 batch_id/published_at,无变化跳过
- moneyflow 历史回补(默认 60 交易日,跳过已发布日期);未发布点查返回
  available_from/available_to 与 history_not_backfilled 标记,缺失不再静默
- eod-refresh 新增 --force --dataset 安全重发(仍走全部质量门,上一批次可回滚)
- 保持 HEL-435 盘后重试机制;新增 22 项测试覆盖字段拒发/正常通过/旧批保留/
  主档新增改名/资金流覆盖/重复执行幂等

Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
multica-agent
2026-09-04 21:36:20 +08:00
co-authored by multica-agent
parent c9892050c3
commit bed6450992
14 changed files with 1058 additions and 37 deletions
+30 -8
View File
@@ -60,7 +60,11 @@ class EodRetryTests(unittest.TestCase):
return db.fetchall("SELECT * FROM job_runs WHERE job_id = ? ORDER BY id", (job_id,))
def _batches(self, db: HubDB, day: str) -> list[dict]:
return db.fetchall("SELECT * FROM batches WHERE trade_date = ?", (day,))
placeholders = ",".join("?" for _ in OFFICIAL)
return db.fetchall(
f"SELECT * FROM batches WHERE trade_date = ? AND dataset IN ({placeholders})",
(day, *sorted(OFFICIAL)),
)
@staticmethod
def _batch_ids(db: HubDB, day: str) -> set[str]:
@@ -71,7 +75,11 @@ class EodRetryTests(unittest.TestCase):
return [name for name in transport.calls if name in DelayedTransport.DATE_APIS]
def _published(self, db: HubDB, day: str) -> set[str]:
rows = db.fetchall("SELECT dataset FROM publications WHERE trade_date = ?", (day,))
placeholders = ",".join("?" for _ in OFFICIAL)
rows = db.fetchall(
f"SELECT dataset FROM publications WHERE trade_date = ? AND dataset IN ({placeholders})",
(day, *sorted(OFFICIAL)),
)
return {str(row["dataset"]) for row in rows}
def test_first_empty_then_retry_succeeds(self) -> None:
@@ -105,12 +113,12 @@ class EodRetryTests(unittest.TestCase):
# success stops all further same-day requests
batches_before = len(self._batches(db, day))
calls_before = len(transport.calls)
eod_calls_before = len(self._eod_calls(transport))
sched.tick(clock_at(day, 17, 0))
sched.tick(clock_at(day, 23, 0))
self.assertEqual(len(self._job_runs(db, "eod_retry")), 2)
self.assertEqual(len(self._batches(db, day)), batches_before)
self.assertEqual(len(transport.calls), calls_before)
self.assertEqual(len(self._eod_calls(transport)), eod_calls_before)
def test_never_ready_marks_cutoff_failed_and_stops(self) -> None:
day = "20240902"
@@ -144,9 +152,20 @@ class EodRetryTests(unittest.TestCase):
sched.tick(clock_at(day, 15, 5)) # eod_a publishes 4 datasets
sched.tick(clock_at(day, 15, 10)) # eod_b publishes index
self.assertEqual(self._published(db, day), OFFICIAL)
def official_batches() -> list[str]:
placeholders = ",".join("?" for _ in OFFICIAL)
return [
str(row["batch_id"])
for row in db.fetchall(
f"SELECT batch_id FROM batches WHERE trade_date = ? AND dataset IN ({placeholders})",
(day, *sorted(OFFICIAL)),
)
]
active = db.fetchall("SELECT dataset, active_batch FROM publications WHERE trade_date = ?", (day,))
active_map = {row["dataset"]: row["active_batch"] for row in active}
batches_before = self._batch_ids(db, day)
active_map = {row["dataset"]: row["active_batch"] for row in active if row["dataset"] in OFFICIAL}
batches_before = set(official_batches())
calls_before = self._eod_calls(transport)
# container restart: fresh scheduler, missed-time catch-up fires eod_a/eod_b
@@ -157,8 +176,11 @@ class EodRetryTests(unittest.TestCase):
self.assertNotIn("eod_retry", ran)
self.assertEqual(self._published(db, day), OFFICIAL)
after = db.fetchall("SELECT dataset, active_batch FROM publications WHERE trade_date = ?", (day,))
self.assertEqual({row["dataset"]: row["active_batch"] for row in after}, active_map)
self.assertEqual(self._batch_ids(db, day), batches_before) # no duplicate batches
self.assertEqual(
{row["dataset"]: row["active_batch"] for row in after if row["dataset"] in OFFICIAL},
active_map,
)
self.assertEqual(set(official_batches()), batches_before) # no duplicate batches
self.assertEqual(self._eod_calls(transport), calls_before) # no duplicate upstream EOD calls
self.assertEqual(sched2.eod_status(day, clock=clock_at(day, 21, 0))["state"], "done")