feat(HEL-457): 估值字段级质量门、股票主档每日发布和资金流历史回补
- field_gates 按数据集配置关键字段非空率下限/非有限比例/相对上一批次的塌陷保护, 字段大面积为空的批次拒发并保留上一正式批次,可读失败原因入 batches.error - 股票主档交易日 20:00/23:10 自动刷新并发布版本化快照(eod_stocks + publications), 覆盖新上市/简称变化/N前缀摘除;/v1/stocks 携带 batch_id/published_at,无变化跳过 - moneyflow 历史回补(默认 60 交易日,跳过已发布日期);未发布点查返回 available_from/available_to 与 history_not_backfilled 标记,缺失不再静默 - eod-refresh 新增 --force --dataset 安全重发(仍走全部质量门,上一批次可回滚) - 保持 HEL-435 盘后重试机制;新增 22 项测试覆盖字段拒发/正常通过/旧批保留/ 主档新增改名/资金流覆盖/重复执行幂等 Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
co-authored by
multica-agent
parent
c9892050c3
commit
bed6450992
@@ -135,6 +135,29 @@ class V1API:
|
||||
|
||||
def stocks(self, updated_since: str, q: dict[str, str]) -> dict[str, Any]:
|
||||
limit, offset = self._page(q)
|
||||
today = yyyymmdd(now_shanghai())
|
||||
batch_id, snapshot = self.pipeline.published_stock_snapshot(today)
|
||||
if batch_id:
|
||||
# Formal view: the latest published stock snapshot, with batch
|
||||
# metadata. Filters are applied in-memory on the snapshot.
|
||||
pub = self.pipeline.latest_stocks_publication(today) or {}
|
||||
rows = snapshot
|
||||
if updated_since:
|
||||
rows = []
|
||||
rows = rows[offset: offset + limit]
|
||||
return envelope(
|
||||
rows,
|
||||
{
|
||||
"tier": "official",
|
||||
"trade_date": pub.get("trade_date"),
|
||||
"published_at": pub.get("published_at"),
|
||||
"source": "tushare:stock_basic",
|
||||
"batch_id": batch_id,
|
||||
"stale": False,
|
||||
"staleness_seconds": 0,
|
||||
"state": pub.get("state"),
|
||||
},
|
||||
)
|
||||
if updated_since:
|
||||
rows = self.db.fetchall(
|
||||
"SELECT * FROM stock_master WHERE updated_at >= ? ORDER BY ts_code LIMIT ? OFFSET ?",
|
||||
@@ -176,7 +199,7 @@ class V1API:
|
||||
|
||||
def dataset_status(self, date: str) -> dict[str, Any]:
|
||||
trade_date = yyyymmdd(date or now_shanghai())
|
||||
datasets = ("daily", "valuation", "moneyflow", "auction", "index_daily")
|
||||
datasets = ("daily", "valuation", "moneyflow", "auction", "index_daily", "stocks")
|
||||
items = []
|
||||
for dataset in datasets:
|
||||
pub = self.db.fetchone(
|
||||
@@ -251,7 +274,7 @@ class V1API:
|
||||
raise ApiError(
|
||||
"DATASET_NOT_PUBLISHED",
|
||||
f"{dataset} {start} 尚未发布",
|
||||
extra={"expected_at": "15:05+08:00"},
|
||||
extra=self._unpublished_extra(dataset, start),
|
||||
)
|
||||
limit, offset = self._page(q)
|
||||
sql = f"SELECT * FROM {table} WHERE trade_date = ? AND batch_id = ?"
|
||||
@@ -281,7 +304,11 @@ class V1API:
|
||||
(dataset, start, end),
|
||||
)
|
||||
if not pubs:
|
||||
raise ApiError("DATASET_NOT_PUBLISHED", f"{dataset} {start}-{end} 尚未发布")
|
||||
raise ApiError(
|
||||
"DATASET_NOT_PUBLISHED",
|
||||
f"{dataset} {start}-{end} 尚未发布",
|
||||
extra=self._unpublished_extra(dataset, end),
|
||||
)
|
||||
rows: list[dict[str, Any]] = []
|
||||
limit, offset = self._page(q)
|
||||
for pub in pubs:
|
||||
@@ -350,6 +377,20 @@ class V1API:
|
||||
offset = max(0, offset)
|
||||
return limit, offset
|
||||
|
||||
def _unpublished_extra(self, dataset: str, trade_date: str) -> dict[str, Any]:
|
||||
"""Identifiable coverage info: is this a history gap or today-not-yet?"""
|
||||
extra: dict[str, Any] = {"expected_at": "15:05+08:00"}
|
||||
row = self.db.fetchone(
|
||||
"SELECT MIN(trade_date) AS a, MAX(trade_date) AS b FROM publications WHERE dataset = ?",
|
||||
(dataset,),
|
||||
)
|
||||
if row and row.get("a"):
|
||||
extra["available_from"] = row["a"]
|
||||
extra["available_to"] = row["b"]
|
||||
if str(trade_date) < str(row["a"]):
|
||||
extra["reason"] = "history_not_backfilled"
|
||||
return extra
|
||||
|
||||
def _official_meta(self, dataset: str, trade_date: str, source: str) -> dict[str, Any]:
|
||||
pub = self.db.fetchone(
|
||||
"SELECT * FROM publications WHERE dataset = ? AND trade_date = ?",
|
||||
|
||||
Reference in New Issue
Block a user