leefer/xiaobai-review
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(HEL-457): 估值字段级质量门、股票主档每日发布和资金流历史回补

Browse Source 
- field_gates 按数据集配置关键字段非空率下限/非有限比例/相对上一批次的塌陷保护，
  字段大面积为空的批次拒发并保留上一正式批次，可读失败原因入 batches.error
- 股票主档交易日 20:00/23:10 自动刷新并发布版本化快照（eod_stocks + publications），
  覆盖新上市/简称变化/N前缀摘除；/v1/stocks 携带 batch_id/published_at，无变化跳过
- moneyflow 历史回补（默认 60 交易日，跳过已发布日期）；未发布点查返回
  available_from/available_to 与 history_not_backfilled 标记，缺失不再静默
- eod-refresh 新增 --force --dataset 安全重发（仍走全部质量门，上一批次可回滚）
- 保持 HEL-435 盘后重试机制；新增 22 项测试覆盖字段拒发/正常通过/旧批保留/
  主档新增改名/资金流覆盖/重复执行幂等

Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
multica-agent
2026-09-04 21:36:20 +08:00
co-authored by multica-agent
parent c9892050c3
commit bed6450992
14 changed files with 1058 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
xiaobai-datahub/datahub/cli.py
+60 -2
View File
@@ -7,6 +7,7 @@ import json
import sys
from datahub.hub import build_hub
from datahub.pipeline import OFFICIAL_DATASETS
from datahub.settings import load_settings
from datahub.timeutil import yyyymmdd
@@ -20,6 +21,23 @@ def main(argv: list[str] | None = None) -> int:
history.add_argument("--force", action="store_true", help="覆盖已发布的指数日期")
refresh = sub.add_parser("eod-refresh", help="对指定交易日补跑盘后正式数据（跳过已发布数据集，仍走质量门禁）")
refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD，默认今天")
refresh.add_argument(
"--force", action="store_true",
help="对 --dataset 指定的数据集强制重取重发（生成新批次，保留上一批次可回滚）",
)
refresh.add_argument(
"--dataset", default=None,
help="配合 --force 使用：只强制重发该数据集（如 valuation）",
)
stocks_refresh = sub.add_parser("stocks-refresh", help="刷新股票主档并发布正式快照（幂等：无变化则跳过）")
stocks_refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD，默认今天")
stocks_refresh.add_argument("--force", action="store_true", help="即使快照无变化也重新发布")
moneyflow_backfill = sub.add_parser(
"moneyflow-backfill", help="回补资金流历史（默认覆盖网站所需的最近 N 个交易日，跳过已发布日期）",
)
moneyflow_backfill.add_argument("--end-date", default=None, help="截止交易日 YYYYMMDD，默认今天")
moneyflow_backfill.add_argument("--trading-days", type=int, default=None, help="回补交易日数量，默认配置 moneyflow_history_trading_days")
moneyflow_backfill.add_argument("--force", action="store_true", help="覆盖已发布的资金流日期")
args = parser.parse_args(argv)
settings = load_settings()
@@ -35,16 +53,56 @@ def main(argv: list[str] | None = None) -> int:
return 0 if result.get("ok") else 1
if args.command == "eod-refresh":
day = yyyymmdd(args.trade_date) if args.trade_date else yyyymmdd()
result = hub.pipeline.run_eod_missing(day)
if args.force:
datasets = tuple(sorted({args.dataset} & OFFICIAL_DATASETS)) if args.dataset else ()
if args.dataset and not datasets:
parser.error(f"unknown dataset: {args.dataset}")
if not datasets:
parser.error("--force requires --dataset (e.g. --dataset valuation)")
result = {}
for dataset in datasets:
result[dataset] = hub.pipeline.run_dataset(dataset, day)
else:
result = hub.pipeline.run_eod_missing(day)
hub.pipeline.audit("cli", "eod-refresh", f"eod:{day}", json.dumps(
{name: item.get("state") for name, item in result.items() if isinstance(item, dict)},
{"force": bool(args.force), "dataset": args.dataset,
**{name: item.get("state") for name, item in result.items() if isinstance(item, dict)}},
ensure_ascii=False,
))
if args.force:
payload = {"trade_date": day, "datasets": result}
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, default=str)
sys.stdout.write("\n")
return 0
missing = hub.pipeline.missing_official_datasets(day)
payload = {"trade_date": day, "datasets": result, "missing_after": missing}
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, default=str)
sys.stdout.write("\n")
return 0 if not missing else 1
if args.command == "stocks-refresh":
day = yyyymmdd(args.trade_date) if args.trade_date else yyyymmdd()
result = hub.pipeline.refresh_stocks(day, force=args.force)
hub.pipeline.audit("cli", "stocks-refresh", f"stocks:{day}", json.dumps(
{"force": bool(args.force), "state": result.get("state"), "batch_id": result.get("batch_id")},
ensure_ascii=False,
))
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, default=str)
sys.stdout.write("\n")
return 0 if result.get("state") != "failed" else 1
if args.command == "moneyflow-backfill":
result = hub.pipeline.backfill_moneyflow_history(
end_date=args.end_date,
trading_days=args.trading_days,
force=args.force,
)
hub.pipeline.audit("cli", "moneyflow-backfill", f"moneyflow:{result.get('end')}", json.dumps(
{"published": len(result.get("published") or []), "skipped": len(result.get("skipped") or []),
"failed": len(result.get("failed") or [])},
ensure_ascii=False,
))
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, default=str)
sys.stdout.write("\n")
return 0 if result.get("ok") else 1
parser.error(f"unknown command: {args.command}")
return 2