Compare commits

...
Author SHA1 Message Date
d175bb65d4 feat(HEL-478): 估值发布后晚间复核并原子追补上游修订
盘后成功发布后继续轻量比对 daily_basic 网站字段,发现修订才走质量门与整组原子切换,避免 17:10 快照落后于晚间上游改写。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-07 21:19:35 +08:00
16ba83ec01 fix(HEL-461): 切换事务失败写入 release-group 审计日志
整组切换中断时除回滚与废弃批次外,同步记录
action=release-group 的失败审计,便于后台追踪。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-05 16:02:11 +08:00
1c740a9d48 fix(HEL-461): 后台整组切换异常统一为 FAILED_PRECONDITION
管理后台补数在切换事务中断时不再抛出原始异常,
统一映射为 ApiError FAILED_PRECONDITION,并保留旧完整版本。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-05 15:56:28 +08:00
75c2e33b68 fix(HEL-461): CLI/后台强制重发改为整组边界切换
eod-refresh --force 与管理后台补数不再单数据集发布,
统一走 force_republish_boundary,避免绕过 A/B 完整边界。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-05 11:26:27 +08:00
32f565ecb9 fix(HEL-461): 整批发布按完整边界重暂存,主档与快照同事务
边界内任有缺失则整组重暂存后统一切换,避免旧新批次混发;
refresh_stocks 失败时主档保持旧值,并补齐回归测试。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-05 08:52:50 +08:00
14 changed files with 1347 additions and 78 deletions
+17 -3
View File
@@ -91,7 +91,7 @@ python -m datahub history-backfill
- 流程:组内全部成员先在暂存表完成拉取、字段质量门、覆盖检查和跨数据集交叉校验(`cross_gates` 配置 ts_code 覆盖重叠率下限),全部达标后才在**一个 SQLite 事务**里复制正式表并翻转全部 `publications` 指针。
- 任一成员失败(拉取失败、质量门拒绝、交叉校验不过、切换事务中断)→ 整批不切换,对外继续提供上一份完整正式版本,失败原因写入 `batches.error``audit_log``action=release-group`),等待晚间自动重试。
- 读取侧任何时刻只会看到"旧完整版本"或"新完整版本":发布指针在单事务内统一翻转,容器重启/事务中断自动回滚,不暴露字段残缺或跨数据集混合版本。
- 幂等:已发布数据集自动跳过;重复执行、并发重试不会生成重复批次或覆盖正常版本(调度器另有 EOD 互斥锁)。
- 幂等:仅当一致性边界内全部成员都已发布时才整组跳过;边界内任有缺失则整组重暂存后统一切换,避免旧批次与新批次混在同一次重发中。重复执行、并发重试不会在完整边界已就绪时生成重复批次(调度器另有 EOD 互斥锁)。
## 股票主档每日刷新与发布
@@ -115,11 +115,25 @@ python -m datahub moneyflow-backfill # --trading-days 60 --end-date --for
```bash
cd xiaobai-datahub
python -m datahub eod-refresh --trade-date 20260904 # 只补缺失数据集
python -m datahub eod-refresh --trade-date 20260904 # 补不完整的 A/B 边界
python -m datahub eod-refresh --trade-date 20260904 --force --dataset valuation
# 强制重取重发:仍走全部质量门,生成新批次,上一批次保留可回滚
# --force 按一致性边界整组重发:valuation/daily/moneyflow/auction/stocks → A 组;
# index_daily → B 组。不可再单独切换某一个正式数据集。
```
管理后台「补数」对盘后正式数据集同样走 `force_republish_boundary`,不会绕过 A/B 整批边界。
## 估值发布后复核与自动追补
Tushare `daily_basic` 会在盘后继续改当日字段。HEL-423 在 2026-09-07 观察到:中枢 17:10 发布 `003021.SZ turnover_rate=1.3565`21:05 上游/旧链路已是 `1.3572`;其余 7 类观察对象当日一致。日 K、资金流、竞价、指数没有同类晚间修订证据,股票主档已有 20:00/23:10 刷新,因此默认只复核估值,不盲目全量重拉。
窗口(可配):交易日 **20:0023:20**,每 30 分钟一次轻量比对(对齐网站 21:00 / 23:30 观察)。只拉取 `daily_basic`,按网站真实请求字段精确比较,无误差豁免。
- 无变化:不产生新批次,状态「已追平」。
- 发现修订:重新走字段质量门、覆盖检查和 A 组整批原子发布;读者全程只能看到上一完整版本或新完整版本。
- 上游空 / 接口失败 / 不完整 / 质量门拒绝:保留上一完整版本,状态「复核失败」。
- 23:20 截止后停止当晚复核;下一自然日盘前对上一交易日再做一次安全追赶。
-`eod_a` / `eod_retry` 共用互斥锁;容器重启会在窗口内立即补一次。
## 备份
+14 -1
View File
@@ -105,6 +105,7 @@ async function render() {
const data = await api("/admin/api/overview");
$("phase").textContent = data.session_phase;
const eod = data.eod_status || {};
const rev = data.revision_status || {};
const eodLabels = {
pending_first_attempt: "等待首次尝试",
waiting_upstream: "等待上游",
@@ -112,6 +113,14 @@ async function render() {
cutoff_failed: "已截止失败",
closed_day: "休市",
};
const revLabels = {
waiting_review: "等待复核",
review_failed: "复核失败",
aligned: "已追平",
cutoff: "已截止",
pending_publish: "待发布",
closed_day: "休市",
};
const eodExtra = [];
if (eod.state === "waiting_upstream") {
eodExtra.push(`已试 ${eod.attempts}`);
@@ -121,12 +130,16 @@ async function render() {
if (eod.state === "cutoff_failed" && eod.missing_datasets) {
eodExtra.push(`${esc(eod.missing_datasets.join(","))}`);
}
const revExtra = [];
if (rev.detail) revExtra.push(esc(String(rev.detail)));
if (rev.window) revExtra.push(esc(String(rev.window)));
page.innerHTML = `
<div class="cards">
<div class="card"><div class="muted">交易日</div><strong>${esc(data.trade_date)}</strong></div>
<div class="card"><div class="muted">阶段</div><strong>${esc(data.session_phase)}</strong></div>
<div class="card"><div class="muted">今日发布</div><strong>${data.publications.length}</strong></div>
<div class="card"><div class="muted">盘后补跑</div><strong>${esc(eodLabels[eod.state] || eod.state || "-")}</strong><div class="muted">${eodExtra.join(" · ")}</div></div>
<div class="card"><div class="muted">估值复核</div><strong>${esc(revLabels[rev.state] || rev.state || "-")}</strong><div class="muted">${revExtra.join(" · ")}</div></div>
<div class="card"><div class="muted">异常批次</div><strong class="${data.anomalies.length ? "fail" : "ok"}">${data.anomalies.length}</strong></div>
</div>
<h2>最近调用</h2>
@@ -269,7 +282,7 @@ function renderRelease(data) {
async function dangerous(kind, dataset) {
const date = ($("rel-date") && $("rel-date").value) || "";
const ds = dataset || prompt("数据集(daily / valuation / moneyflow / auction / index_daily / reference", "daily");
const ds = dataset || prompt("数据集(daily/valuation/moneyflow/auction/stocks→A组整批;index_daily→B组;或 reference", "daily");
if (!ds) return;
const password = prompt("二次确认:输入管理密码");
if (!password) return;
@@ -17,6 +17,10 @@
"eod_retry_start": "15:15",
"eod_retry_interval_minutes": 30,
"eod_retry_cutoff": "23:30",
"revision_review_datasets": ["valuation"],
"revision_review_start": "20:00",
"revision_review_interval_minutes": 30,
"revision_review_cutoff": "23:20",
"moneyflow_history_trading_days": 60,
"stocks_refresh_times": [
"20:00",
+18 -2
View File
@@ -6,7 +6,7 @@ from typing import Any
from datahub.adapters import RESERVED
from datahub.auth import AuthService
from datahub.db import HubDB
from datahub.pipeline import Pipeline
from datahub.pipeline import OFFICIAL_DATASETS, STOCKS_DATASET, Pipeline
from datahub.scheduler import Scheduler
from datahub.serving import ApiError
from datahub.timeutil import isoformat, now_shanghai, session_phase, yyyymmdd
@@ -39,6 +39,7 @@ class AdminAPI:
"session_phase": session_phase(now_shanghai(), is_open),
"is_open_day": is_open,
"eod_status": self.scheduler.eod_status(today),
"revision_status": self.scheduler.revision_status(today),
"publications": pubs,
"anomalies": failed,
"recent_calls": _public_calls(calls),
@@ -91,6 +92,7 @@ class AdminAPI:
{"id": "eod_a", "at": "15:05", "title": "盘后批 A daily/valuation/moneyflow/auction"},
{"id": "eod_b", "at": "15:10", "title": "盘后批 B index_daily"},
{"id": "eod_retry", "at": "15:15-23:30", "title": "盘后未出数自动重试(每 30 分钟,成功即停)"},
{"id": "eod_revise", "at": "20:00-23:20", "title": "估值发布后复核(轻量比对,有修订才整组原子追补)"},
{"id": "stocks_refresh", "at": stocks_times, "title": "股票主档刷新与正式发布(新上市/更名,无变化跳过)"},
{"id": "history_backfill", "at": "manual", "title": "回补历史日历与指数日 K"},
{"id": "cleanup", "at": "00:30", "title": "清理 staging / 日志"},
@@ -143,8 +145,22 @@ class AdminAPI:
self._dangerous(password, confirm, f"{dataset}:{day}")
if dataset == "reference":
result = self.pipeline.ingest_reference(day)
elif dataset in OFFICIAL_DATASETS or dataset == STOCKS_DATASET:
# Manual same-day republish must rebuild the full A/B boundary.
# Gate failures and mid-switch exceptions both surface as
# FAILED_PRECONDITION so the admin API never leaks raw
# transaction errors to the client.
try:
result = self.pipeline.force_republish_boundary(dataset, day)
failures = self.pipeline.eod_failures(result)
if failures:
raise ApiError("FAILED_PRECONDITION", "; ".join(failures))
except ApiError:
raise
except Exception as exc:
raise ApiError("FAILED_PRECONDITION", str(exc)) from exc
else:
result = self.pipeline.run_dataset(dataset, day)
raise ApiError("INVALID_ARGUMENT", f"unsupported backfill dataset: {dataset}")
self.pipeline.audit(actor, "backfill", f"{dataset}:{day}", json.dumps({"ok": True}))
return result
+15 -14
View File
@@ -7,7 +7,7 @@ import json
import sys
from datahub.hub import build_hub
from datahub.pipeline import OFFICIAL_DATASETS
from datahub.pipeline import EOD_A_DATASETS, OFFICIAL_DATASETS, STOCKS_DATASET
from datahub.settings import load_settings
from datahub.timeutil import yyyymmdd
@@ -19,15 +19,15 @@ def main(argv: list[str] | None = None) -> int:
history.add_argument("--calendar-start", default=None, help="日历起点,默认配置 calendar_start")
history.add_argument("--index-days", type=int, default=None, help="指数回补交易日数量,默认 260")
history.add_argument("--force", action="store_true", help="覆盖已发布的指数日期")
refresh = sub.add_parser("eod-refresh", help="对指定交易日补跑盘后正式数据(跳过已发布数据集,仍走质量门禁)")
refresh = sub.add_parser("eod-refresh", help="对指定交易日补跑盘后正式数据(跳过已完整发布的一致性边界,仍走质量门禁)")
refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD,默认今天")
refresh.add_argument(
"--force", action="store_true",
help=" --dataset 指定的数据集强制重取重发(生成新批次保留上一批次可回滚",
help="强制重发 --dataset 所属的完整一致性边界(A 组或 B 组),生成新批次保留上一批次可回滚",
)
refresh.add_argument(
"--dataset", default=None,
help="配合 --force 使用:只强制重发该数据集(如 valuation",
help="配合 --force:指定边界内任一成员(如 valuation→整组 Aindex_daily→整组 B",
)
stocks_refresh = sub.add_parser("stocks-refresh", help="刷新股票主档并发布正式快照(幂等:无变化则跳过)")
stocks_refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD,默认今天")
@@ -54,26 +54,27 @@ def main(argv: list[str] | None = None) -> int:
if args.command == "eod-refresh":
day = yyyymmdd(args.trade_date) if args.trade_date else yyyymmdd()
if args.force:
datasets = tuple(sorted({args.dataset} & OFFICIAL_DATASETS)) if args.dataset else ()
if args.dataset and not datasets:
parser.error(f"unknown dataset: {args.dataset}")
if not datasets:
allowed = set(OFFICIAL_DATASETS) | {STOCKS_DATASET}
if not args.dataset:
parser.error("--force requires --dataset (e.g. --dataset valuation)")
result = {}
for dataset in datasets:
result[dataset] = hub.pipeline.run_dataset(dataset, day)
if args.dataset not in allowed:
parser.error(f"unknown dataset: {args.dataset}")
result = hub.pipeline.force_republish_boundary(args.dataset, day)
boundary = "A" if args.dataset in EOD_A_DATASETS or args.dataset == STOCKS_DATASET else "B"
else:
result = hub.pipeline.run_eod_missing(day)
boundary = None
hub.pipeline.audit("cli", "eod-refresh", f"eod:{day}", json.dumps(
{"force": bool(args.force), "dataset": args.dataset,
{"force": bool(args.force), "dataset": args.dataset, "boundary": boundary,
**{name: item.get("state") for name, item in result.items() if isinstance(item, dict)}},
ensure_ascii=False,
))
if args.force:
payload = {"trade_date": day, "datasets": result}
failures = hub.pipeline.eod_failures(result)
payload = {"trade_date": day, "boundary": boundary, "datasets": result}
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, default=str)
sys.stdout.write("\n")
return 0
return 0 if not failures else 1
missing = hub.pipeline.missing_official_datasets(day)
payload = {"trade_date": day, "datasets": result, "missing_after": missing}
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, default=str)
+13
View File
@@ -238,6 +238,19 @@ CREATE TABLE IF NOT EXISTS eod_progress (
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS revision_progress (
trade_date TEXT PRIMARY KEY,
state TEXT NOT NULL,
attempts INTEGER NOT NULL DEFAULT 0,
last_attempt_at TEXT,
next_retry_at TEXT,
finished_at TEXT,
catchup_done INTEGER NOT NULL DEFAULT 0,
last_diff TEXT,
detail TEXT,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS audit_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
actor TEXT NOT NULL,
+364 -32
View File
@@ -15,6 +15,12 @@ from datahub.governance.ratelimit import TokenBucket
from datahub.governance.retry import RetryError, retry_call
from datahub.logutil import get_logger
from datahub.normalize import finite_number, normalize_daily
from datahub.revision import (
compare_fields,
diff_published_vs_upstream,
official_table,
revision_datasets,
)
from datahub.settings import Settings
from datahub.timeutil import add_days, isoformat, now_shanghai, yyyymmdd
@@ -348,11 +354,20 @@ class Pipeline:
is published; ``force`` re-publishes unconditionally. New listings,
renames (incl. N/C prefix removal) and status changes all flow into
the snapshot, which carries batch_id/published_at metadata.
The ``stock_master`` UPSERT happens inside the same publish
transaction as the snapshot switch — fetch / quality-gate / switch
failures leave the master on the previous complete values.
"""
day = yyyymmdd(trade_date or self.clock())
try:
rows = self._fetch_dataset(STOCKS_DATASET, day)
with self.db.write() as connection:
self._upsert_stock_master(connection, rows, isoformat(self.clock()))
except Exception as exc:
self.audit(
"pipeline", "stocks-refresh", f"{STOCKS_DATASET}:{day}",
json.dumps({"state": "failed", "error": str(exc)}, ensure_ascii=False),
)
raise
if not force:
active, snapshot = self.published_stock_snapshot(day)
if active is not None:
@@ -369,7 +384,14 @@ class Pipeline:
"batch_id": active,
"rows": len(snapshot),
}
try:
result = self.run_dataset(STOCKS_DATASET, day, prepared_rows=rows)
except Exception as exc:
self.audit(
"pipeline", "stocks-refresh", f"{STOCKS_DATASET}:{day}",
json.dumps({"state": "failed", "error": str(exc)}, ensure_ascii=False),
)
raise
self.audit(
"pipeline", "stocks-refresh", f"{STOCKS_DATASET}:{day}",
json.dumps({"batch_id": result["batch_id"], "rows": result["rows"]}, ensure_ascii=False),
@@ -637,53 +659,324 @@ class Pipeline:
return [dataset for dataset in sorted(OFFICIAL_DATASETS) if dataset not in published]
def run_eod_missing(self, trade_date: str) -> dict[str, Any]:
"""Fetch/publish every official dataset still missing for the date.
"""Republish every incomplete EOD consistency group for the date.
Idempotent: datasets with an existing publication are skipped, so
repeats never overwrite the current official batch. All missing
members stage first and switch in one atomic release group; a single
member failure keeps the previous complete official version serving.
A-group (daily/valuation/moneyflow/auction + stocks) and B-group
(index_daily) are separate boundaries. Within a group, either the
whole boundary is already published (idempotent skip) or every
member is re-staged and switched together — never fill only the
missing members on top of older batches from an earlier partial run.
"""
return self.run_release_group(tuple(sorted(OFFICIAL_DATASETS)), trade_date, include_stocks=True)
results: dict[str, Any] = {}
results.update(self.run_eod_batch_a(trade_date))
results.update(self.run_eod_batch_b(trade_date))
return results
def run_eod_batch_a(self, trade_date: str) -> dict[str, Any]:
return self.run_release_group(EOD_A_DATASETS, trade_date, include_stocks=True)
def run_eod_batch_a(self, trade_date: str, force: bool = False) -> dict[str, Any]:
return self.run_release_group(EOD_A_DATASETS, trade_date, include_stocks=True, force=force)
def run_eod_batch_b(self, trade_date: str) -> dict[str, Any]:
return self.run_release_group(EOD_B_DATASETS, trade_date)
def run_eod_batch_b(self, trade_date: str, force: bool = False) -> dict[str, Any]:
return self.run_release_group(EOD_B_DATASETS, trade_date, force=force)
def force_republish_boundary(self, dataset: str, trade_date: str) -> dict[str, Any]:
"""Force-republish the full A/B consistency boundary that owns ``dataset``.
CLI ``eod-refresh --force`` and admin manual backfill must not publish a
single official member alone — that would mix old and new batches inside
the same trade date. Naming any A-group member (or stocks) rebuilds the
whole A group; naming ``index_daily`` rebuilds B.
"""
name = str(dataset or "").strip()
if name in EOD_A_DATASETS or name == STOCKS_DATASET:
return self.run_eod_batch_a(trade_date, force=True)
if name in EOD_B_DATASETS:
return self.run_eod_batch_b(trade_date, force=True)
raise ValueError(f"dataset is not part of an EOD release boundary: {dataset}")
def published_official_rows(self, dataset: str, trade_date: str) -> list[dict[str, Any]]:
day = yyyymmdd(trade_date)
batch_id = self.active_batch(dataset, day)
if not batch_id:
return []
fields = compare_fields(dataset)
table = official_table(dataset)
if fields:
columns = ",".join(fields)
return self.db.fetchall(
f"SELECT {columns} FROM {table} WHERE batch_id = ?",
(batch_id,),
)
return self.db.fetchall(f"SELECT * FROM {table} WHERE batch_id = ?", (batch_id,))
def compare_revision(self, dataset: str, trade_date: str) -> dict[str, Any]:
"""Light fetch of one revision-risk dataset vs the published official rows."""
day = yyyymmdd(trade_date)
published = self.published_official_rows(dataset, day)
if not published:
return {
"dataset": dataset,
"trade_date": day,
"changed": False,
"state": "skipped",
"reason": "not_published",
}
try:
upstream = self._fetch_dataset(dataset, day)
except Exception as exc:
return {
"dataset": dataset,
"trade_date": day,
"changed": False,
"state": "failed",
"reason": "upstream_error",
"error": str(exc),
}
if not upstream:
return {
"dataset": dataset,
"trade_date": day,
"changed": False,
"state": "failed",
"reason": "upstream_empty",
"error": "revision review upstream empty",
"published_rows": len(published),
"upstream_rows": 0,
}
listed = self.db.fetchone(
"SELECT COUNT(*) AS n FROM stock_master WHERE list_status = 'L'",
)
listed_n = int((listed or {}).get("n") or 0)
floor = float(self.settings.quality.get("daily_row_ratio") or 0.98)
if listed_n and len(upstream) / listed_n < floor:
return {
"dataset": dataset,
"trade_date": day,
"changed": False,
"state": "failed",
"reason": "incomplete",
"error": (
f"revision review incomplete: upstream {len(upstream)} "
f"/ listed {listed_n} < {floor}"
),
"published_rows": len(published),
"upstream_rows": len(upstream),
}
if len(upstream) < len(published) * floor:
return {
"dataset": dataset,
"trade_date": day,
"changed": False,
"state": "failed",
"reason": "incomplete",
"error": (
f"revision review incomplete: upstream {len(upstream)} "
f"< published {len(published)} * {floor}"
),
"published_rows": len(published),
"upstream_rows": len(upstream),
}
compared = diff_published_vs_upstream(dataset, published, upstream)
compared["trade_date"] = day
compared["state"] = "changed" if compared["changed"] else "unchanged"
compared["reason"] = "revised" if compared["changed"] else "unchanged"
return compared
def review_published_revisions(self, trade_date: str) -> dict[str, Any]:
"""Evening/morning catch-up: compare website fields, republish only on change.
Unchanged → no new batch. Changed → full A/B boundary quality gate +
atomic switch (HEL-459/460/461). Empty/failed/incomplete upstream keeps
the previous complete official version.
"""
day = yyyymmdd(trade_date)
results: dict[str, Any] = {}
for dataset in revision_datasets(self.settings.quality):
compared = self.compare_revision(dataset, day)
if compared.get("state") == "skipped":
results[dataset] = compared
continue
if compared.get("state") == "failed":
results[dataset] = compared
LOGGER.warning(
"revision review kept previous official version",
extra={
"hub": {
"dataset": dataset,
"trade_date": day,
"reason": compared.get("reason"),
"event": "revision_review_failed",
}
},
)
self.audit(
"pipeline", "revision-review", f"{dataset}:{day}",
json.dumps(
{
"state": "failed",
"reason": compared.get("reason"),
"error": compared.get("error"),
},
ensure_ascii=False,
),
)
continue
if not compared.get("changed"):
results[dataset] = {
"dataset": dataset,
"trade_date": day,
"state": "aligned",
"reason": "unchanged",
"published_rows": compared.get("published_rows"),
"upstream_rows": compared.get("upstream_rows"),
}
self.audit(
"pipeline", "revision-review", f"{dataset}:{day}",
json.dumps({"state": "aligned", "reason": "unchanged"}, ensure_ascii=False),
)
continue
LOGGER.info(
"revision review detected upstream rewrite, republishing boundary",
extra={
"hub": {
"dataset": dataset,
"trade_date": day,
"diffs": compared.get("diffs"),
"event": "revision_review_changed",
}
},
)
try:
published = self.force_republish_boundary(dataset, day)
except Exception as exc:
results[dataset] = {
"dataset": dataset,
"trade_date": day,
"state": "failed",
"reason": "republish_error",
"error": str(exc),
"diffs": compared.get("diffs"),
}
LOGGER.warning(
"revision republish failed, previous official version keeps serving",
extra={
"hub": {
"dataset": dataset,
"trade_date": day,
"reason": str(exc),
"event": "revision_review_failed",
}
},
)
self.audit(
"pipeline", "revision-review", f"{dataset}:{day}",
json.dumps(
{"state": "failed", "reason": "republish_error", "error": str(exc)},
ensure_ascii=False,
),
)
continue
failures = self.eod_failures(published)
if failures:
results.update(published)
results[dataset] = {
**(published.get(dataset) or {}),
"dataset": dataset,
"trade_date": day,
"state": "failed",
"reason": "quality_gate",
"error": "; ".join(failures),
"diffs": compared.get("diffs"),
}
self.audit(
"pipeline", "revision-review", f"{dataset}:{day}",
json.dumps(
{
"state": "failed",
"reason": "quality_gate",
"error": "; ".join(failures),
"diffs": compared.get("diffs"),
},
ensure_ascii=False,
),
)
continue
results.update(published)
results["review"] = {
"dataset": dataset,
"trade_date": day,
"state": "aligned",
"reason": "revised",
"diffs": compared.get("diffs"),
"missing_codes": compared.get("missing_codes"),
"extra_codes": compared.get("extra_codes"),
}
self.audit(
"pipeline", "revision-review", f"{dataset}:{day}",
json.dumps(
{
"state": "aligned",
"reason": "revised",
"diffs": compared.get("diffs"),
"switched": sorted(
name for name, item in published.items()
if isinstance(item, dict) and item.get("state") == "published"
),
},
ensure_ascii=False,
),
)
return results
def run_release_group(
self,
datasets: tuple[str, ...],
trade_date: str,
include_stocks: bool = False,
force: bool = False,
) -> dict[str, Any]:
"""One post-market publish/republish becomes one atomic visibility flip.
Consistency boundary: every member (official datasets, plus the daily
stocks snapshot when ``include_stocks`` and not yet published) is
fetched, staged, field-gated and cross-validated BEFORE any reader can
see it. Only when the whole group passes does a single SQLite
transaction copy all staging batches to the official tables and flip
every ``publications`` row at once. Any member failure aborts the
group: the previous complete official version keeps serving and the
reason is recorded on the batches and in the audit log.
stocks snapshot when ``include_stocks``) is fetched, staged,
field-gated and cross-validated BEFORE any reader can see it. Only
when the whole group passes does a single SQLite transaction copy
all staging batches to the official tables and flip every
``publications`` row at once. Any member failure aborts the group:
the previous complete official version keeps serving and the reason
is recorded on the batches and in the audit log.
Skip is all-or-nothing for the boundary unless ``force``: if every
official member (and stocks when required) is already published, the
group is skipped. If any official member is still missing — or
``force`` is set — every official member is re-staged, so a retry or
manual republish never mixes old and new batches in one release.
"""
day = yyyymmdd(trade_date)
results: dict[str, Any] = {}
staged: dict[str, dict[str, Any]] = {}
failure: str | None = None
pending: list[str] = []
missing_official = [dataset for dataset in datasets if self.active_batch(dataset, day) is None]
stocks_missing = include_stocks and self.active_batch(STOCKS_DATASET, day) is None
if not force and not missing_official and not stocks_missing:
for dataset in datasets:
if self.active_batch(dataset, day) is not None:
results[dataset] = {
"dataset": dataset,
"trade_date": day,
"state": "skipped",
"reason": "already_published",
}
else:
pending.append(dataset)
if include_stocks:
results[STOCKS_DATASET] = {
"dataset": STOCKS_DATASET,
"trade_date": day,
"state": "skipped",
"reason": "already_published",
}
return results
# Incomplete or forced boundary → restage every official member together.
pending = list(datasets)
for dataset in pending:
if failure is not None:
@@ -705,9 +998,13 @@ class Pipeline:
"error": str(exc),
}
if include_stocks and failure is None and self.active_batch(STOCKS_DATASET, day) is None:
# Stocks join the same switch when the official boundary is being
# rebuilt (missing or forced), or when only the stocks snapshot is
# still missing.
rebuild_official = bool(force or missing_official)
if include_stocks and failure is None and (rebuild_official or stocks_missing):
try:
stocks_plan = self._stage_stocks_snapshot(day)
stocks_plan = self._stage_stocks_snapshot(day, force=rebuild_official)
except Exception as exc:
failure = f"{STOCKS_DATASET}: {exc}"
results[STOCKS_DATASET] = {
@@ -756,8 +1053,32 @@ class Pipeline:
try:
self._switch_release_group(day, staged)
except Exception as exc:
reason = f"release group switch failed: {exc}"
for item in staged.values():
self._abandon_batch(item["batch_id"], f"release group switch failed: {exc}")
self._abandon_batch(item["batch_id"], reason)
LOGGER.warning(
"release group switch failed, previous official version keeps serving",
extra={
"hub": {
"trade_date": day,
"datasets": sorted(staged),
"reason": reason,
"event": "release_group_switch_failed",
}
},
)
self.audit(
"pipeline", "release-group", f"eod:{day}",
json.dumps(
{
"state": "failed",
"reason": reason,
"switched": [],
"force": bool(force),
},
ensure_ascii=False,
),
)
raise
for dataset, item in staged.items():
results[dataset] = {
@@ -770,7 +1091,8 @@ class Pipeline:
self.audit(
"pipeline", "release-group", f"eod:{day}",
json.dumps(
{"state": "ok", "switched": sorted(staged)}, ensure_ascii=False
{"state": "ok", "switched": sorted(staged), "force": bool(force)},
ensure_ascii=False,
),
)
return results
@@ -820,18 +1142,18 @@ class Pipeline:
"state": "degraded" if report["soft_fail"] else "published",
}
def _stage_stocks_snapshot(self, trade_date: str) -> dict[str, Any] | None:
def _stage_stocks_snapshot(self, trade_date: str, force: bool = False) -> dict[str, Any] | None:
"""Stage the daily stocks snapshot for a release group switch.
Returns None when the published snapshot is already identical to
upstream (idempotent skip). The stock_master upsert is deferred into
the group switch transaction so the master never runs ahead of the
published snapshot.
upstream (idempotent skip) unless ``force`` is set. The stock_master
upsert is deferred into the group switch / publish transaction so the
master never runs ahead of the published snapshot.
"""
day = yyyymmdd(trade_date)
active, snapshot = self.published_stock_snapshot(day)
rows = self._fetch_dataset(STOCKS_DATASET, day)
if active is not None:
if active is not None and not force:
upstream = sorted(
tuple(str(row.get(field)) for field in STOCK_SNAPSHOT_FIELDS) for row in rows
)
@@ -1077,6 +1399,16 @@ class Pipeline:
with self.db.write() as connection:
_staging_count_or_raise(connection, dataset, trade_date, batch_id)
connection.execute(EOD_COPY[dataset], (batch_id,))
if dataset == STOCKS_DATASET:
staging = DATASET_TABLES[STOCKS_DATASET][1]
stock_rows = [
dict(row)
for row in connection.execute(
f"SELECT * FROM {staging} WHERE batch_id = ?",
(batch_id,),
).fetchall()
]
self._upsert_stock_master(connection, stock_rows, published_at)
if self.before_commit:
self.before_commit()
_upsert_publication(connection, dataset, trade_date, batch_id, state, published_at)
+111
View File
@@ -0,0 +1,111 @@
"""Post-publish revision review for datasets whose upstream may rewrite T-day fields.
HEL-423 field evidence, not a whitelist of tolerated diffs:
- 2026-09-07 valuation/daily_basic: hub published 003021.SZ turnover_rate=1.3565
at 17:10; website legacy and a direct Tushare read at 21:05 both showed 1.3572.
The other seven observed objects (daily, moneyflow, auction, stocks, status,
index_daily, calendar) matched. Hub had already stopped the day after the
first successful publish, so the revision never self-healed.
- 2026-09-02: same dataset, opposite direction (hub already held the later
value). Confirms daily_basic is rewritten after the first complete dump.
Daily bars, moneyflow, auction and index_daily have no same-evening field
revision evidence. Stocks already refreshes at 20:00/23:10. Review therefore
fetches only configured revision-risk datasets (default: valuation) and
compares the website-requested field set. No numeric tolerance.
"""
from __future__ import annotations
from typing import Any
from datahub.db import DATASET_TABLES
from datahub.normalize import VALUATION_FIELDS
from datahub.numbers import finite_number, round4
# Datasets with proven same-evening upstream rewrites. Config may replace this
# list; it must not silently expand to a full EOD re-pull.
DEFAULT_REVISION_DATASETS = ("valuation",)
# Website daily_basic request (HEL-423): ts_code/trade_date plus the eight
# value fields used by the old link and field_gates.
WEBSITE_COMPARE_FIELDS: dict[str, tuple[str, ...]] = {
"valuation": VALUATION_FIELDS,
}
REVISION_STATES = ("waiting_review", "review_failed", "aligned", "cutoff")
def revision_datasets(quality: dict[str, Any] | None) -> tuple[str, ...]:
raw = (quality or {}).get("revision_review_datasets")
if isinstance(raw, (list, tuple)) and raw:
names = tuple(str(item) for item in raw if str(item))
if names:
return names
return DEFAULT_REVISION_DATASETS
def compare_fields(dataset: str) -> tuple[str, ...]:
fields = WEBSITE_COMPARE_FIELDS.get(dataset)
if fields:
return fields
gate = {}
return tuple(str(item) for item in (gate.get("fields") or []) if str(item))
def _norm_value(field: str, value: Any) -> Any:
if field in {"ts_code", "trade_date"}:
return str(value or "")
number = round4(finite_number(value))
return number
def row_signature(row: dict[str, Any], fields: tuple[str, ...]) -> tuple[Any, ...]:
return tuple(_norm_value(field, row.get(field)) for field in fields)
def diff_published_vs_upstream(
dataset: str,
published: list[dict[str, Any]],
upstream: list[dict[str, Any]],
*,
max_diffs: int = 20,
) -> dict[str, Any]:
"""Exact compare on website-requested fields. No tolerance / exemption."""
fields = compare_fields(dataset)
if not fields:
fields = tuple(sorted({key for row in published + upstream for key in row if key != "batch_id"}))
pub_map = {str(row.get("ts_code") or "").upper(): row for row in published}
up_map = {str(row.get("ts_code") or "").upper(): row for row in upstream}
missing = sorted(code for code in pub_map if code not in up_map)
extra = sorted(code for code in up_map if code not in pub_map)
diffs: list[dict[str, Any]] = []
for code in sorted(set(pub_map) & set(up_map)):
left = row_signature(pub_map[code], fields)
right = row_signature(up_map[code], fields)
if left == right:
continue
for field, old, new in zip(fields, left, right):
if old == new:
continue
diffs.append({"ts_code": code, "field": field, "published": old, "upstream": new})
if len(diffs) >= max_diffs:
break
if len(diffs) >= max_diffs:
break
changed = bool(diffs or missing or extra)
return {
"changed": changed,
"dataset": dataset,
"fields": list(fields),
"published_rows": len(published),
"upstream_rows": len(upstream),
"missing_codes": missing[:max_diffs],
"extra_codes": extra[:max_diffs],
"diffs": diffs,
}
def official_table(dataset: str) -> str:
return DATASET_TABLES[dataset][0]
+218 -1
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import json
import threading
from collections.abc import Callable
from datetime import datetime, time, timedelta
@@ -8,13 +9,14 @@ from typing import Any
from datahub.db import HubDB
from datahub.logutil import get_logger
from datahub.pipeline import Pipeline
from datahub.revision import revision_datasets
from datahub.timeutil import isoformat, now_shanghai, yyyymmdd
LOGGER = get_logger()
JobFn = Callable[[str], Any]
EOD_JOB_IDS = {"eod_a", "eod_b", "eod_retry"}
EOD_JOB_IDS = {"eod_a", "eod_b", "eod_retry", "eod_revise"}
def is_open_day(db: HubDB, day: str) -> bool:
@@ -27,6 +29,20 @@ def is_open_day(db: HubDB, day: str) -> bool:
return int(row["is_open"]) == 1
def previous_open_day(db: HubDB, day: str) -> str | None:
row = db.fetchone(
"""
SELECT cal_date FROM trade_calendar
WHERE exchange = 'SSE' AND is_open = 1 AND cal_date < ?
ORDER BY cal_date DESC LIMIT 1
""",
(day,),
)
if row is None:
return None
return str(row["cal_date"])
def _hhmm(value: str) -> time:
return datetime.strptime(value, "%H:%M").time()
@@ -49,6 +65,7 @@ class Scheduler:
"eod_a": self._eod_a,
"eod_b": self._eod_b,
"eod_retry": self._eod_retry,
"eod_revise": self._eod_revise,
"stocks_refresh": self._stocks_refresh,
"cleanup": self._cleanup,
"backup": self._backup,
@@ -118,6 +135,8 @@ class Scheduler:
if job_id in {"eod_a", "eod_b"}:
self._settle_eod(day)
ran.extend(self._eod_retry_tick(now, day, open_day))
ran.extend(self._revision_review_tick(now, day, open_day))
ran.extend(self._revision_catchup_tick(now, day))
return ran
# ------------------------------------------------------------------
@@ -219,6 +238,201 @@ class Scheduler:
"detail": (row or {}).get("detail"),
}
def revision_progress(self, day: str) -> dict[str, Any] | None:
return self.db.fetchone("SELECT * FROM revision_progress WHERE trade_date = ?", (day,))
def revision_status(self, trade_date: str | None = None, clock: datetime | None = None) -> dict[str, Any]:
"""等待复核 / 复核失败 / 已追平 / 已截止."""
day = yyyymmdd(trade_date or now_shanghai(clock))
row = self.revision_progress(day)
open_day = is_open_day(self.db, day)
published = self._revision_ready(day)
if row and row["state"] in {"aligned", "review_failed", "cutoff", "waiting_review"}:
state = str(row["state"])
elif not open_day:
state = "closed_day"
elif not published:
state = "pending_publish"
else:
state = "waiting_review"
return {
"trade_date": day,
"is_open_day": open_day,
"state": state,
"datasets": list(revision_datasets(self.pipeline.settings.quality)),
"attempts": int((row or {}).get("attempts") or 0),
"last_attempt_at": (row or {}).get("last_attempt_at"),
"next_retry_at": (row or {}).get("next_retry_at") if state in {"waiting_review", "review_failed"} else None,
"finished_at": (row or {}).get("finished_at"),
"catchup_done": bool(int((row or {}).get("catchup_done") or 0)),
"detail": (row or {}).get("detail"),
"window": f"{self.pipeline.settings.revision_review_start}-{self.pipeline.settings.revision_review_cutoff}",
}
def _revision_ready(self, day: str) -> bool:
return all(
self.pipeline.active_batch(dataset, day)
for dataset in revision_datasets(self.pipeline.settings.quality)
)
def _revision_due(self, now: datetime, row: dict[str, Any] | None) -> bool:
if row is None or not row.get("last_attempt_at"):
return True
try:
last = datetime.fromisoformat(str(row["last_attempt_at"]))
except ValueError:
return True
interval = timedelta(minutes=self.pipeline.settings.revision_review_interval_minutes)
return now_shanghai(last).replace(tzinfo=None) + interval <= now.replace(tzinfo=None)
def _revision_review_tick(self, now: datetime, day: str, open_day: bool) -> list[str]:
if not open_day or not self._revision_ready(day):
return []
settings = self.pipeline.settings
current = now.time()
start = _hhmm(settings.revision_review_start)
cutoff = _hhmm(settings.revision_review_cutoff)
row = self.revision_progress(day)
if current < start:
if row is None:
self._save_revision_progress(day, state="waiting_review")
return []
if current >= cutoff:
if row is None or row["state"] not in {"aligned", "cutoff"}:
detail = "复核窗口已截止"
self._save_revision_progress(
day, state="cutoff", finished_at=isoformat(now), detail=detail,
)
with self.db.write() as connection:
connection.execute(
"INSERT INTO job_runs(job_id, state, started_at, finished_at, error, attempt, detail)"
" VALUES ('eod_revise','failed',?,?,?,?,?)",
(
isoformat(now), isoformat(now), detail,
int((row or {}).get("attempts") or 0), "revision cutoff reached",
),
)
elif row["state"] == "aligned" and not row.get("finished_at"):
self._save_revision_progress(day, finished_at=isoformat(now))
return []
if not self._revision_due(now, row):
return []
if "eod_revise" not in self.jobs:
return []
return self._run_revision_job(day, now, catchup=False)
def _revision_catchup_tick(self, now: datetime, day: str) -> list[str]:
prev = previous_open_day(self.db, day)
if prev is None or prev >= day:
return []
if not self._revision_ready(prev):
return []
row = self.revision_progress(prev)
if row and int(row.get("catchup_done") or 0):
return []
if not self._revision_due(now, row):
return []
if "eod_revise" not in self.jobs:
return []
return self._run_revision_job(prev, now, catchup=True)
def _run_revision_job(self, day: str, now: datetime, catchup: bool) -> list[str]:
attempts = int((self.revision_progress(day) or {}).get("attempts") or 0) + 1
interval = self.pipeline.settings.revision_review_interval_minutes
self._save_revision_progress(
day,
state="waiting_review",
attempts=attempts,
last_attempt_at=isoformat(now),
next_retry_at=isoformat(now + timedelta(minutes=interval)),
)
ran: list[str] = []
try:
out = self.run_job("eod_revise", day)
except Exception as exc:
LOGGER.warning("revision review failed for %s: %s", day, exc)
self._save_revision_progress(
day,
state="review_failed",
detail="复核失败,保留上一完整版本",
)
ran.append("eod_revise")
return ran
ran.append("eod_revise")
if out.get("state") == "skipped":
return ran
result = out.get("result") if isinstance(out.get("result"), dict) else {}
failed = [
name for name, item in result.items()
if isinstance(item, dict) and item.get("state") == "failed"
]
review = result.get("review") if isinstance(result.get("review"), dict) else None
watched = [
result[name]
for name in revision_datasets(self.pipeline.settings.quality)
if isinstance(result.get(name), dict)
]
diff_blob = None
if review and review.get("diffs"):
diff_blob = json.dumps(review.get("diffs"), ensure_ascii=False)
else:
for item in watched:
if item.get("diffs"):
diff_blob = json.dumps(item.get("diffs"), ensure_ascii=False)
break
revised = bool(review and review.get("reason") == "revised")
matched = any(item.get("reason") == "unchanged" or item.get("state") == "aligned" for item in watched)
if failed:
self._save_revision_progress(
day,
state="review_failed",
detail="复核失败,保留上一完整版本",
last_diff=diff_blob,
)
elif revised or matched:
fields: dict[str, Any] = {
"state": "aligned",
"finished_at": isoformat(now),
"detail": "已追平" if revised else "已追平(无变化)",
"last_diff": diff_blob,
}
if catchup:
fields["catchup_done"] = 1
self._save_revision_progress(day, **fields)
return ran
def _save_revision_progress(self, day: str, **fields: Any) -> None:
columns = [
"trade_date", "state", "attempts", "last_attempt_at",
"next_retry_at", "finished_at", "catchup_done", "last_diff", "detail", "updated_at",
]
with self.db.write() as connection:
existing = connection.execute(
"SELECT trade_date FROM revision_progress WHERE trade_date = ?",
(day,),
).fetchone()
if existing is None:
payload = {name: None for name in columns}
payload.update({
"trade_date": day,
"state": "waiting_review",
"attempts": 0,
"catchup_done": 0,
})
payload.update(fields)
payload["updated_at"] = isoformat()
placeholders = ",".join("?" for _ in columns)
connection.execute(
f"INSERT INTO revision_progress({','.join(columns)}) VALUES ({placeholders})",
tuple(payload[name] for name in columns),
)
else:
assignments = ", ".join(f"{name} = ?" for name in fields)
connection.execute(
f"UPDATE revision_progress SET {assignments}, updated_at = ? WHERE trade_date = ?",
(*fields.values(), isoformat(), day),
)
def _record_eod_attempt(self, day: str, now: datetime) -> None:
row = self.eod_progress(day)
attempts = int((row or {}).get("attempts") or 0) + 1
@@ -320,6 +534,9 @@ class Scheduler:
def _eod_retry(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.run_eod_missing(trade_date)
def _eod_revise(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.review_published_revisions(trade_date)
def _stocks_refresh(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.refresh_stocks(trade_date)
+14
View File
@@ -79,6 +79,20 @@ class Settings:
def eod_retry_cutoff(self) -> str:
return str(self.quality.get("eod_retry_cutoff") or "23:30")
@property
def revision_review_start(self) -> str:
# Before the 21:00 website shadow observation.
return str(self.quality.get("revision_review_start") or "20:00")
@property
def revision_review_interval_minutes(self) -> int:
return int(self.quality.get("revision_review_interval_minutes") or 30)
@property
def revision_review_cutoff(self) -> str:
# Last light review ~23:00; cutoff before the 23:30 observation.
return str(self.quality.get("revision_review_cutoff") or "23:20")
def load_settings(
env: dict[str, str] | None = None,
+181 -8
View File
@@ -21,14 +21,14 @@ class GroupTransport:
def __init__(self) -> None:
self.empty: set[str] = set()
self.keep_rows: dict[str, int] = {}
self.stocks: list[dict] = []
self.stocks: list[dict] | None = None
self.calls: list[str] = []
def __call__(self, api_name: str, params: dict, fields: str):
self.calls.append(api_name)
if api_name in self.empty:
return []
if api_name == "stock_basic" and self.stocks:
if api_name == "stock_basic" and self.stocks is not None:
return [dict(row) for row in self.stocks]
rows = fake_transport(api_name, params, fields)
keep = self.keep_rows.get(api_name)
@@ -113,19 +113,48 @@ class ReleaseGroupSwitchTests(unittest.TestCase):
first = self.pipe.run_dataset("daily", TRADE_DATE)
self.transport.empty = {"daily_basic"}
results = self.pipe.run_eod_missing(TRADE_DATE)
self.assertEqual(results["daily"]["state"], "skipped")
# incomplete A-group restages daily with the others; valuation fails → no A switch
self.assertEqual(results["daily"]["state"], "failed")
self.assertEqual(results["valuation"]["state"], "failed")
# the already-published complete batch is untouched and keeps serving
# the already-published daily batch is untouched and keeps serving
self.assertEqual(self.pipe.active_batch("daily", TRADE_DATE), first["batch_id"])
self.assertEqual(
publications_map(self.db, TRADE_DATE),
{"daily": first["batch_id"]},
)
pubs = publications_map(self.db, TRADE_DATE)
self.assertEqual(pubs["daily"], first["batch_id"])
self.assertNotIn("valuation", pubs)
self.assertNotIn("moneyflow", pubs)
self.assertNotIn("auction", pubs)
# B-group is an independent boundary and may still publish
self.assertEqual(results["index_daily"]["state"], "published")
payload = V1API(self.db, self.pipe, self.pipe.settings).handle(
"/v1/bars/daily", {"date": [TRADE_DATE], "code": ["600000.SH"]}
)
self.assertEqual(payload["meta"]["batch_id"], first["batch_id"])
def test_partial_group_retry_does_not_mix_batches(self) -> None:
"""Already-published A members must be restaged with missing ones."""
first_daily = self.pipe.run_dataset("daily", TRADE_DATE)
first_moneyflow = self.pipe.run_dataset("moneyflow", TRADE_DATE)
results = self.pipe.run_eod_missing(TRADE_DATE)
# A-group switched as one boundary; B-group (index) also published
for name in (*GROUP_A, "stocks"):
self.assertEqual(results[name]["state"], "published", name)
self.assertEqual(results["index_daily"]["state"], "published")
pubs = self.db.fetchall(
"SELECT dataset, active_batch, published_at FROM publications WHERE trade_date = ?",
(TRADE_DATE,),
)
by_ds = {str(row["dataset"]): row for row in pubs}
# old partial batches replaced — no cross-batch mix of the first wave
self.assertNotEqual(by_ds["daily"]["active_batch"], first_daily["batch_id"])
self.assertNotEqual(by_ds["moneyflow"]["active_batch"], first_moneyflow["batch_id"])
a_times = {by_ds[name]["published_at"] for name in (*GROUP_A, "stocks")}
self.assertEqual(len(a_times), 1)
# serving resolves the new complete A-group batches
api = V1API(self.db, self.pipe, self.pipe.settings)
daily = api.handle("/v1/bars/daily", {"date": [TRADE_DATE], "code": ["600000.SH"]})
self.assertEqual(daily["meta"]["batch_id"], results["daily"]["batch_id"])
self.assertEqual(daily["meta"]["batch_id"], by_ds["daily"]["active_batch"])
def test_reads_during_switch_see_old_state_until_commit(self) -> None:
snapshots: list[dict] = []
@@ -155,6 +184,13 @@ class ReleaseGroupSwitchTests(unittest.TestCase):
for table in ("eod_bars", "eod_valuation", "eod_moneyflow", "eod_auction", "eod_stocks"):
rows = self.db.fetchall(f"SELECT * FROM {table} WHERE trade_date = ?", (TRADE_DATE,))
self.assertEqual(rows, [], table)
audit = self.db.fetchone(
"SELECT * FROM audit_log WHERE action = 'release-group' ORDER BY id DESC"
)
self.assertIsNotNone(audit)
detail = str(audit["detail"])
self.assertIn("killed mid-switch", detail)
self.assertIn("failed", detail)
def test_duplicate_runs_are_idempotent(self) -> None:
self.pipe.run_eod_batch_a(TRADE_DATE)
@@ -231,5 +267,142 @@ class ReleaseGroupSwitchTests(unittest.TestCase):
self.assertIsNone(stocks_pub)
class StocksRefreshAtomicTests(unittest.TestCase):
def setUp(self) -> None:
self.transport = GroupTransport()
self.pipe, self.db = make_pipe(self.transport)
self.pipe.ingest_reference(TRADE_DATE)
self.transport.stocks = [
{"ts_code": "600000.SH", "symbol": "600000", "name": "浦发银行", "area": "上海",
"industry": "银行", "market": "主板", "list_status": "L", "list_date": "19991110"},
{"ts_code": "920071.BJ", "symbol": "920071", "name": "N金钛", "area": "辽宁",
"industry": "小金属", "market": "北交所", "list_status": "L", "list_date": "20240901"},
]
first = self.pipe.refresh_stocks(TRADE_DATE)
self.assertEqual(first["state"], "published")
self.first_batch = first["batch_id"]
def test_refresh_keeps_master_when_snapshot_publish_fails(self) -> None:
self.transport.stocks = [
{"ts_code": "600000.SH", "symbol": "600000", "name": "浦发银行", "area": "上海",
"industry": "银行", "market": "主板", "list_status": "L", "list_date": "19991110"},
{"ts_code": "920071.BJ", "symbol": "920071", "name": "金钛股份", "area": "辽宁",
"industry": "小金属", "market": "北交所", "list_status": "L", "list_date": "20240901"},
]
def explode() -> None:
raise RuntimeError("snapshot switch killed")
self.pipe.before_commit = explode
with self.assertRaises(RuntimeError):
self.pipe.refresh_stocks(TRADE_DATE)
master = self.db.fetchone("SELECT name FROM stock_master WHERE ts_code = '920071.BJ'")
self.assertEqual(master["name"], "N金钛") # rename not applied
self.assertEqual(self.pipe.active_batch("stocks", TRADE_DATE), self.first_batch)
audit = self.db.fetchone(
"SELECT * FROM audit_log WHERE action = 'stocks-refresh' ORDER BY id DESC"
)
self.assertIn("failed", str(audit["detail"]))
self.assertIn("snapshot switch killed", str(audit["detail"]))
def test_refresh_keeps_master_when_quality_gate_rejects(self) -> None:
self.transport.stocks = [] # empty → hard fail before publish
with self.assertRaises(Exception):
self.pipe.refresh_stocks(TRADE_DATE)
master = self.db.fetchone("SELECT name FROM stock_master WHERE ts_code = '920071.BJ'")
self.assertEqual(master["name"], "N金钛")
self.assertEqual(self.pipe.active_batch("stocks", TRADE_DATE), self.first_batch)
audit = self.db.fetchone(
"SELECT * FROM audit_log WHERE action = 'stocks-refresh' ORDER BY id DESC"
)
self.assertIn("failed", str(audit["detail"]))
class ForceBoundaryEntryTests(unittest.TestCase):
"""CLI force / admin backfill must rebuild the full A/B boundary."""
def setUp(self) -> None:
self.transport = GroupTransport()
self.pipe, self.db = make_pipe(self.transport)
self.pipe.ingest_reference(TRADE_DATE)
self.first = self.pipe.run_eod_batch_a(TRADE_DATE)
self.pipe.run_eod_batch_b(TRADE_DATE)
def test_force_republish_valuation_rebuilds_whole_a_group(self) -> None:
before = publications_map(self.db, TRADE_DATE)
results = self.pipe.force_republish_boundary("valuation", TRADE_DATE)
self.assertEqual({item["state"] for item in results.values()}, {"published"})
after = publications_map(self.db, TRADE_DATE)
for name in (*GROUP_A, "stocks"):
self.assertNotEqual(after[name], before[name], name)
self.assertEqual(after[name], results[name]["batch_id"], name)
# B-group left alone
self.assertEqual(after["index_daily"], before["index_daily"])
pubs = self.db.fetchall(
"SELECT dataset, published_at FROM publications WHERE trade_date = ?",
(TRADE_DATE,),
)
a_times = {row["published_at"] for row in pubs if row["dataset"] in {*GROUP_A, "stocks"}}
self.assertEqual(len(a_times), 1)
def test_force_republish_index_rebuilds_only_b_group(self) -> None:
before = publications_map(self.db, TRADE_DATE)
results = self.pipe.force_republish_boundary("index_daily", TRADE_DATE)
self.assertEqual(results["index_daily"]["state"], "published")
after = publications_map(self.db, TRADE_DATE)
self.assertNotEqual(after["index_daily"], before["index_daily"])
for name in GROUP_A:
self.assertEqual(after[name], before[name], name)
def test_admin_backfill_official_dataset_uses_boundary(self) -> None:
from datahub.admin_api import AdminAPI
from datahub.auth import AuthService
from datahub.crypto import SecretVault
from datahub.scheduler import Scheduler
from datahub.serving import ApiError
vault = SecretVault(self.pipe.settings.encryption_key)
auth = AuthService(self.db, vault, self.pipe.settings.api_token, "StartPass1")
admin = AdminAPI(self.db, self.pipe, Scheduler(self.db, self.pipe), auth)
before = publications_map(self.db, TRADE_DATE)
result = admin.backfill("moneyflow", TRADE_DATE, "StartPass1", f"moneyflow:{TRADE_DATE}", "tester")
self.assertEqual(result["moneyflow"]["state"], "published")
after = publications_map(self.db, TRADE_DATE)
for name in (*GROUP_A, "stocks"):
self.assertNotEqual(after[name], before[name], name)
# bad password / wrong confirm still rejected
with self.assertRaises(ApiError):
admin.backfill("daily", TRADE_DATE, "wrong", f"daily:{TRADE_DATE}", "tester")
def test_admin_backfill_switch_crash_is_failed_precondition(self) -> None:
from datahub.admin_api import AdminAPI
from datahub.auth import AuthService
from datahub.crypto import SecretVault
from datahub.scheduler import Scheduler
from datahub.serving import ApiError
vault = SecretVault(self.pipe.settings.encryption_key)
auth = AuthService(self.db, vault, self.pipe.settings.api_token, "StartPass1")
admin = AdminAPI(self.db, self.pipe, Scheduler(self.db, self.pipe), auth)
before = publications_map(self.db, TRADE_DATE)
def explode() -> None:
raise RuntimeError("killed mid-switch")
self.pipe.before_commit = explode
with self.assertRaises(ApiError) as ctx:
admin.backfill("valuation", TRADE_DATE, "StartPass1", f"valuation:{TRADE_DATE}", "tester")
self.assertEqual(ctx.exception.code, "FAILED_PRECONDITION")
self.assertIn("killed mid-switch", ctx.exception.message)
# previous complete A/B versions keep serving
self.assertEqual(publications_map(self.db, TRADE_DATE), before)
audit = self.db.fetchone(
"SELECT * FROM audit_log WHERE action = 'release-group' ORDER BY id DESC"
)
self.assertIsNotNone(audit)
self.assertIn("failed", str(audit["detail"]))
self.assertIn("killed mid-switch", str(audit["detail"]))
if __name__ == "__main__":
unittest.main()
+22 -3
View File
@@ -111,15 +111,24 @@ class EodRetryTests(unittest.TestCase):
self.assertEqual(progress["state"], "done")
self.assertEqual(progress["attempts"], 4) # eod_a + eod_b + 2 retries
# success stops all further same-day requests
# success stops further eod_retry; revision window has not started yet
batches_before = len(self._batches(db, day))
eod_calls_before = len(self._eod_calls(transport))
sched.tick(clock_at(day, 17, 0))
sched.tick(clock_at(day, 23, 0))
self.assertEqual(len(self._job_runs(db, "eod_retry")), 2)
self.assertEqual(len(self._job_runs(db, "eod_revise")), 0)
self.assertEqual(len(self._batches(db, day)), batches_before)
self.assertEqual(len(self._eod_calls(transport)), eod_calls_before)
# 23:00 is inside the valuation review window: light daily_basic only, no new batch
sched.tick(clock_at(day, 23, 0))
self.assertEqual(len(self._job_runs(db, "eod_retry")), 2)
self.assertEqual(len(self._job_runs(db, "eod_revise")), 1)
self.assertEqual(len(self._batches(db, day)), batches_before)
extra = [name for name in self._eod_calls(transport)[eod_calls_before:]]
self.assertTrue(extra)
self.assertTrue(all(name == "daily_basic" for name in extra))
def test_never_ready_marks_cutoff_failed_and_stops(self) -> None:
day = "20240902"
db, transport, pipe, sched = self._make(set())
@@ -174,6 +183,7 @@ class EodRetryTests(unittest.TestCase):
self.assertIn("eod_a", ran)
self.assertIn("eod_b", ran)
self.assertNotIn("eod_retry", ran)
self.assertIn("eod_revise", ran)
self.assertEqual(self._published(db, day), OFFICIAL)
after = db.fetchall("SELECT dataset, active_batch FROM publications WHERE trade_date = ?", (day,))
self.assertEqual(
@@ -181,7 +191,9 @@ class EodRetryTests(unittest.TestCase):
active_map,
)
self.assertEqual(set(official_batches()), batches_before) # no duplicate batches
self.assertEqual(self._eod_calls(transport), calls_before) # no duplicate upstream EOD calls
extra = self._eod_calls(transport)[len(calls_before):]
self.assertTrue(extra)
self.assertTrue(all(name == "daily_basic" for name in extra))
self.assertEqual(sched2.eod_status(day, clock=clock_at(day, 21, 0))["state"], "done")
def test_restart_with_partial_publish_only_fetches_missing(self) -> None:
@@ -206,6 +218,7 @@ class EodRetryTests(unittest.TestCase):
for hh, mm in ((15, 5), (15, 10), (15, 40), (16, 10), (20, 0), (23, 40)):
ran = sched.tick(clock_at(day, hh, mm))
self.assertNotIn("eod_retry", ran)
self.assertNotIn("eod_revise", ran)
eod_runs = db.fetchall("SELECT * FROM job_runs WHERE job_id LIKE 'eod%'")
self.assertEqual(eod_runs, [])
self.assertIsNone(db.fetchone("SELECT * FROM eod_progress WHERE trade_date = ?", (day,)))
@@ -226,12 +239,18 @@ class EodRetryTests(unittest.TestCase):
self.assertEqual(len(self._batches(db, day)), batches_before)
self.assertEqual(len(transport.calls), calls_before)
revised = sched.run_job("eod_revise", day)
self.assertEqual(revised["state"], "ok")
self.assertEqual(len(self._batches(db, day)), batches_before)
sched._eod_lock.acquire() # simulate an in-flight EOD job
try:
busy = sched.run_job("eod_retry", day)
self.assertEqual(busy["state"], "skipped")
busy_a = sched.run_job("eod_a", day)
self.assertEqual(busy_a["state"], "skipped")
busy_r = sched.run_job("eod_revise", day)
self.assertEqual(busy_r["state"], "skipped")
finally:
sched._eod_lock.release()
self.assertEqual(len(self._batches(db, day)), batches_before)
+20 -11
View File
@@ -228,25 +228,34 @@ class GateRetryInterplayTests(unittest.TestCase):
class ForceRepublishTests(unittest.TestCase):
def test_run_dataset_over_published_keeps_prev_for_rollback(self) -> None:
def test_force_boundary_republish_keeps_prev_for_rollback(self) -> None:
transport = ValuationTransport()
pipe, db = make_pipe(transport)
pipe.ingest_reference(TRADE_DATE)
first = pipe.run_dataset("valuation", TRADE_DATE)
first = pipe.run_eod_batch_a(TRADE_DATE)
first_val = first["valuation"]["batch_id"]
first_daily = first["daily"]["batch_id"]
transport.mode = "vr_all_null"
with self.assertRaises(QualityError):
pipe.run_dataset("valuation", TRADE_DATE) # gate holds: bad re-publish refused
blocked = pipe.force_republish_boundary("valuation", TRADE_DATE)
self.assertEqual(blocked["valuation"]["state"], "failed")
self.assertEqual(pipe.active_batch("valuation", TRADE_DATE), first_val)
self.assertEqual(pipe.active_batch("daily", TRADE_DATE), first_daily)
transport.mode = "ok"
second = pipe.run_dataset("valuation", TRADE_DATE) # CLI --force path
self.assertNotEqual(first["batch_id"], second["batch_id"])
pub = db.fetchone(
"SELECT * FROM publications WHERE dataset='valuation' AND trade_date=?",
second = pipe.force_republish_boundary("valuation", TRADE_DATE)
self.assertEqual(second["valuation"]["state"], "published")
self.assertNotEqual(second["valuation"]["batch_id"], first_val)
self.assertNotEqual(second["daily"]["batch_id"], first_daily)
pubs = db.fetchall(
"SELECT dataset, active_batch, prev_batch, published_at FROM publications WHERE trade_date=?",
(TRADE_DATE,),
)
self.assertEqual(pub["active_batch"], second["batch_id"])
self.assertEqual(pub["prev_batch"], first["batch_id"])
by_ds = {str(row["dataset"]): row for row in pubs}
a_times = {by_ds[name]["published_at"] for name in ("daily", "valuation", "moneyflow", "auction", "stocks")}
self.assertEqual(len(a_times), 1)
self.assertEqual(by_ds["valuation"]["active_batch"], second["valuation"]["batch_id"])
self.assertEqual(by_ds["valuation"]["prev_batch"], first_val)
rolled = pipe.rollback("valuation", TRADE_DATE, actor="cli")
self.assertEqual(rolled["active_batch"], first["batch_id"])
self.assertEqual(rolled["active_batch"], first_val)
if __name__ == "__main__":
@@ -0,0 +1,333 @@
from __future__ import annotations
import copy
import unittest
from pathlib import Path
import tempfile
from datahub.adapters.base import AdapterError
from datahub.adapters.tushare import TushareAdapter
from datahub.crypto import SecretVault
from datahub.db import HubDB
from datahub.pipeline import Pipeline
from datahub.scheduler import Scheduler
from datahub.serving import V1API
from datahub.settings import Settings
from datahub.timeutil import SHANGHAI
from tests.fixtures import RAW, TRADE_DATE, fake_transport
from tests.test_eod_retry import clock_at
from tests.test_quality_gates import FIELD_GATES
SAMPLE_DAY = "20260907"
NEXT_DAY = "20260908"
SAMPLE_CODE = "003021.SZ"
def _dated(row: dict, day: str) -> dict:
item = dict(row)
if "trade_date" in item:
item["trade_date"] = day
return item
class RevisingTransport:
"""Fixture transport that can rewrite daily_basic after the first publish."""
DATE_APIS = {"daily", "daily_basic", "adj_factor", "moneyflow", "stk_auction", "index_daily"}
def __init__(self, extra_calendar: list[dict] | None = None) -> None:
self.calls: list[str] = []
self.fail_daily_basic = False
self.empty_daily_basic = False
self.null_volume_ratio = False
self.turnover_by_code: dict[str, float] = {}
self.extra_calendar = extra_calendar or []
def __call__(self, api_name: str, params: dict, fields: str):
self.calls.append(api_name)
day = str(params.get("trade_date") or "")
if api_name == "trade_cal":
rows = fake_transport(api_name, params, fields)
extra = [
row for row in self.extra_calendar
if str(params.get("start_date") or "") <= row["cal_date"] <= str(params.get("end_date") or "99999999")
]
return rows + extra
if self.fail_daily_basic and api_name == "daily_basic":
raise AdapterError("tushare daily_basic unavailable")
if self.empty_daily_basic and api_name == "daily_basic":
return []
if api_name == "index_daily":
code = params.get("ts_code")
rows = [row for row in RAW["index_daily"] if row["ts_code"] == code]
if day:
rows = [_dated(row, day) for row in rows]
return rows
rows = fake_transport(api_name, params, fields)
if api_name == "stock_basic":
rows = list(rows)
rows.append({
"ts_code": SAMPLE_CODE, "symbol": "003021", "name": "兆威机电",
"area": "广东", "industry": "元器件", "market": "主板",
"list_status": "L", "list_date": "20201202",
})
return rows
if api_name in self.DATE_APIS:
template = RAW.get(api_name) or []
if not day:
return [_dated(row, TRADE_DATE) for row in template]
out = [_dated(row, day) for row in template]
extra = copy.deepcopy(template[0])
extra["ts_code"] = SAMPLE_CODE
extra["trade_date"] = day
if api_name == "daily_basic":
extra["turnover_rate"] = self.turnover_by_code.get(SAMPLE_CODE, extra.get("turnover_rate"))
if self.null_volume_ratio:
extra["volume_ratio"] = None
for row in out:
row["volume_ratio"] = None
out.append(extra)
if api_name == "daily_basic":
for row in out:
code = str(row.get("ts_code") or "")
if code in self.turnover_by_code:
row["turnover_rate"] = self.turnover_by_code[code]
return out
return rows
def make_revision_env(quality_extra: dict | None = None, extra_calendar: list[dict] | None = None):
tmp = tempfile.TemporaryDirectory()
db = HubDB(Path(tmp.name) / "hub.db")
transport = RevisingTransport(extra_calendar=extra_calendar)
adapter = TushareAdapter("x", transport=transport)
quality = {
"daily_row_ratio": 0.5,
"null_rate_max": 0.5,
"max_publish_attempts": 2,
"publication_generations": 3,
"field_gates": FIELD_GATES,
"revision_review_start": "20:00",
"revision_review_interval_minutes": 30,
"revision_review_cutoff": "23:20",
"revision_review_datasets": ["valuation"],
}
if quality_extra:
quality.update(quality_extra)
settings = Settings(
encryption_key=SecretVault.generate_key(),
api_token="t" * 32,
db_path=db.path,
backup_dir=Path(tmp.name) / "backups",
quality=quality,
scheduler_enabled=False,
)
pipe = Pipeline(db, adapter, settings)
sched = Scheduler(db, pipe)
return tmp, db, transport, pipe, sched
SAMPLE_CALENDAR = [
{"exchange": "SSE", "cal_date": SAMPLE_DAY, "is_open": 1, "pretrade_date": "20260906"},
{"exchange": "SSE", "cal_date": NEXT_DAY, "is_open": 1, "pretrade_date": SAMPLE_DAY},
]
class RevisionReviewTests(unittest.TestCase):
def _publish(self, pipe: Pipeline, day: str) -> None:
pipe.ingest_reference(day)
pipe.run_eod_batch_a(day)
pipe.run_eod_batch_b(day)
def _turnover(self, db: HubDB, day: str, code: str = SAMPLE_CODE) -> float | None:
pub = db.fetchone(
"SELECT active_batch FROM publications WHERE dataset='valuation' AND trade_date=?",
(day,),
)
row = db.fetchone(
"SELECT turnover_rate FROM eod_valuation WHERE batch_id=? AND ts_code=?",
(pub["active_batch"], code),
)
return None if row is None else row["turnover_rate"]
def _batch_ids(self, db: HubDB, day: str) -> set[str]:
return {str(row["batch_id"]) for row in db.fetchall("SELECT batch_id FROM batches WHERE trade_date=?", (day,))}
def test_no_change_does_not_create_a_new_batch(self) -> None:
tmp, db, transport, pipe, sched = make_revision_env()
self.addCleanup(tmp.cleanup)
self._publish(pipe, TRADE_DATE)
before = self._batch_ids(db, TRADE_DATE)
sched.tick(clock_at(TRADE_DATE, 20, 0))
self.assertEqual(self._batch_ids(db, TRADE_DATE), before)
progress = db.fetchone("SELECT * FROM revision_progress WHERE trade_date=?", (TRADE_DATE,))
self.assertEqual(progress["state"], "aligned")
self.assertIn("无变化", progress["detail"])
status = sched.revision_status(TRADE_DATE, clock=clock_at(TRADE_DATE, 20, 0))
self.assertEqual(status["state"], "aligned")
def test_hel423_20260907_single_field_revision_is_caught_up(self) -> None:
tmp, db, transport, pipe, sched = make_revision_env(extra_calendar=SAMPLE_CALENDAR)
self.addCleanup(tmp.cleanup)
transport.turnover_by_code[SAMPLE_CODE] = 1.3565
self._publish(pipe, SAMPLE_DAY)
self.assertEqual(self._turnover(db, SAMPLE_DAY), 1.3565)
first = db.fetchone(
"SELECT active_batch FROM publications WHERE dataset='valuation' AND trade_date=?",
(SAMPLE_DAY,),
)["active_batch"]
transport.turnover_by_code[SAMPLE_CODE] = 1.3572
seen: list[float | None] = []
def watch() -> None:
seen.append(self._turnover(db, SAMPLE_DAY))
pipe.before_commit = watch
ran = sched.tick(clock_at(SAMPLE_DAY, 20, 0))
self.assertIn("eod_revise", ran)
self.assertEqual(seen, [1.3565]) # readers still see the previous complete version mid-switch
self.assertEqual(self._turnover(db, SAMPLE_DAY), 1.3572)
second = db.fetchone(
"SELECT active_batch FROM publications WHERE dataset='valuation' AND trade_date=?",
(SAMPLE_DAY,),
)["active_batch"]
self.assertNotEqual(second, first)
api = V1API(db, pipe, pipe.settings)
payload = api.valuation({"date": SAMPLE_DAY, "code": SAMPLE_CODE})
row = next(item for item in payload["data"] if item["ts_code"] == SAMPLE_CODE)
self.assertEqual(row["turnover_rate"], 1.3572)
progress = db.fetchone("SELECT * FROM revision_progress WHERE trade_date=?", (SAMPLE_DAY,))
self.assertEqual(progress["state"], "aligned")
self.assertEqual(progress["detail"], "已追平")
audit = db.fetchone(
"SELECT * FROM audit_log WHERE action='revision-review' ORDER BY id DESC"
)
self.assertIn("1.3572", str(audit["detail"]))
self.assertIn(SAMPLE_CODE, str(audit["detail"]))
def test_empty_or_failed_upstream_keeps_previous_version(self) -> None:
tmp, db, transport, pipe, sched = make_revision_env()
self.addCleanup(tmp.cleanup)
self._publish(pipe, TRADE_DATE)
active = db.fetchone(
"SELECT active_batch FROM publications WHERE dataset='valuation' AND trade_date=?",
(TRADE_DATE,),
)["active_batch"]
batches = self._batch_ids(db, TRADE_DATE)
transport.empty_daily_basic = True
sched.tick(clock_at(TRADE_DATE, 20, 0))
self.assertEqual(
db.fetchone(
"SELECT active_batch FROM publications WHERE dataset='valuation' AND trade_date=?",
(TRADE_DATE,),
)["active_batch"],
active,
)
self.assertEqual(
db.fetchone("SELECT state FROM revision_progress WHERE trade_date=?", (TRADE_DATE,))["state"],
"review_failed",
)
transport.empty_daily_basic = False
transport.fail_daily_basic = True
sched.tick(clock_at(TRADE_DATE, 20, 30))
self.assertEqual(
db.fetchone(
"SELECT active_batch FROM publications WHERE dataset='valuation' AND trade_date=?",
(TRADE_DATE,),
)["active_batch"],
active,
)
self.assertEqual(self._batch_ids(db, TRADE_DATE), batches)
def test_quality_gate_rejects_catchup_and_keeps_previous(self) -> None:
tmp, db, transport, pipe, sched = make_revision_env()
self.addCleanup(tmp.cleanup)
self._publish(pipe, TRADE_DATE)
active = db.fetchone(
"SELECT active_batch FROM publications WHERE dataset='valuation' AND trade_date=?",
(TRADE_DATE,),
)["active_batch"]
transport.turnover_by_code[SAMPLE_CODE] = 9.9999
transport.null_volume_ratio = True
sched.tick(clock_at(TRADE_DATE, 20, 0))
self.assertEqual(
db.fetchone(
"SELECT active_batch FROM publications WHERE dataset='valuation' AND trade_date=?",
(TRADE_DATE,),
)["active_batch"],
active,
)
self.assertEqual(
db.fetchone("SELECT state FROM revision_progress WHERE trade_date=?", (TRADE_DATE,))["state"],
"review_failed",
)
def test_repeat_ticks_after_align_do_not_republish(self) -> None:
tmp, db, transport, pipe, sched = make_revision_env()
self.addCleanup(tmp.cleanup)
transport.turnover_by_code[SAMPLE_CODE] = 1.3565
self._publish(pipe, TRADE_DATE)
transport.turnover_by_code[SAMPLE_CODE] = 1.3572
sched.tick(clock_at(TRADE_DATE, 20, 0))
after_fix = self._batch_ids(db, TRADE_DATE)
sched.tick(clock_at(TRADE_DATE, 20, 10)) # inside interval
self.assertEqual(len(db.fetchall("SELECT * FROM job_runs WHERE job_id='eod_revise'")), 1)
sched.tick(clock_at(TRADE_DATE, 20, 30)) # next light compare, no change
self.assertEqual(self._batch_ids(db, TRADE_DATE), after_fix)
self.assertEqual(self._turnover(db, TRADE_DATE), 1.3572)
def test_restart_catches_up_inside_window(self) -> None:
tmp, db, transport, pipe, sched = make_revision_env()
self.addCleanup(tmp.cleanup)
transport.turnover_by_code[SAMPLE_CODE] = 1.3565
self._publish(pipe, TRADE_DATE)
transport.turnover_by_code[SAMPLE_CODE] = 1.3572
sched2 = Scheduler(db, pipe)
ran = sched2.tick(clock_at(TRADE_DATE, 21, 0))
self.assertIn("eod_revise", ran)
self.assertEqual(self._turnover(db, TRADE_DATE), 1.3572)
def test_cutoff_stops_evening_reviews_and_morning_catchup_runs(self) -> None:
tmp, db, transport, pipe, sched = make_revision_env(extra_calendar=SAMPLE_CALENDAR)
self.addCleanup(tmp.cleanup)
transport.turnover_by_code[SAMPLE_CODE] = 1.3565
self._publish(pipe, SAMPLE_DAY)
sched.tick(clock_at(SAMPLE_DAY, 23, 25)) # past 23:20 cutoff, no review yet
cutoff = db.fetchone("SELECT * FROM revision_progress WHERE trade_date=?", (SAMPLE_DAY,))
self.assertEqual(cutoff["state"], "cutoff")
self.assertEqual(self._turnover(db, SAMPLE_DAY), 1.3565)
transport.turnover_by_code[SAMPLE_CODE] = 1.3572
sched.tick(clock_at(SAMPLE_DAY, 23, 50)) # still same calendar day, no catch-up
self.assertEqual(self._turnover(db, SAMPLE_DAY), 1.3565)
ran = sched.tick(clock_at(NEXT_DAY, 8, 45))
self.assertIn("eod_revise", ran)
self.assertEqual(self._turnover(db, SAMPLE_DAY), 1.3572)
progress = db.fetchone("SELECT * FROM revision_progress WHERE trade_date=?", (SAMPLE_DAY,))
self.assertEqual(progress["state"], "aligned")
self.assertEqual(int(progress["catchup_done"]), 1)
batches = self._batch_ids(db, SAMPLE_DAY)
sched.tick(clock_at(NEXT_DAY, 8, 50))
self.assertEqual(self._batch_ids(db, SAMPLE_DAY), batches)
def test_only_valuation_is_light_fetched(self) -> None:
tmp, db, transport, pipe, sched = make_revision_env()
self.addCleanup(tmp.cleanup)
self._publish(pipe, TRADE_DATE)
before = [name for name in transport.calls]
sched.tick(clock_at(TRADE_DATE, 20, 0))
extra = transport.calls[len(before):]
self.assertIn("daily_basic", extra)
self.assertNotIn("daily", extra)
self.assertNotIn("moneyflow", extra)
self.assertNotIn("stk_auction", extra)
self.assertNotIn("index_daily", extra)
if __name__ == "__main__":
unittest.main()