feat(HEL-478): 估值发布后晚间复核并原子追补上游修订
盘后成功发布后继续轻量比对 daily_basic 网站字段，发现修订才走质量门与整组原子切换，避免 17:10 快照落后于晚间上游改写。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
co-authored by
Cursor
multica-agent
Cursor
multica-agent
parent
16ba83ec01
commit
d175bb65d4
@@ -15,6 +15,12 @@ from datahub.governance.ratelimit import TokenBucket
|
||||
from datahub.governance.retry import RetryError, retry_call
|
||||
from datahub.logutil import get_logger
|
||||
from datahub.normalize import finite_number, normalize_daily
|
||||
from datahub.revision import (
|
||||
compare_fields,
|
||||
diff_published_vs_upstream,
|
||||
official_table,
|
||||
revision_datasets,
|
||||
)
|
||||
from datahub.settings import Settings
|
||||
from datahub.timeutil import add_days, isoformat, now_shanghai, yyyymmdd
|
||||
|
||||
@@ -687,6 +693,240 @@ class Pipeline:
|
||||
return self.run_eod_batch_b(trade_date, force=True)
|
||||
raise ValueError(f"dataset is not part of an EOD release boundary: {dataset}")
|
||||
|
||||
def published_official_rows(self, dataset: str, trade_date: str) -> list[dict[str, Any]]:
|
||||
day = yyyymmdd(trade_date)
|
||||
batch_id = self.active_batch(dataset, day)
|
||||
if not batch_id:
|
||||
return []
|
||||
fields = compare_fields(dataset)
|
||||
table = official_table(dataset)
|
||||
if fields:
|
||||
columns = ",".join(fields)
|
||||
return self.db.fetchall(
|
||||
f"SELECT {columns} FROM {table} WHERE batch_id = ?",
|
||||
(batch_id,),
|
||||
)
|
||||
return self.db.fetchall(f"SELECT * FROM {table} WHERE batch_id = ?", (batch_id,))
|
||||
|
||||
def compare_revision(self, dataset: str, trade_date: str) -> dict[str, Any]:
|
||||
"""Light fetch of one revision-risk dataset vs the published official rows."""
|
||||
day = yyyymmdd(trade_date)
|
||||
published = self.published_official_rows(dataset, day)
|
||||
if not published:
|
||||
return {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"changed": False,
|
||||
"state": "skipped",
|
||||
"reason": "not_published",
|
||||
}
|
||||
try:
|
||||
upstream = self._fetch_dataset(dataset, day)
|
||||
except Exception as exc:
|
||||
return {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"changed": False,
|
||||
"state": "failed",
|
||||
"reason": "upstream_error",
|
||||
"error": str(exc),
|
||||
}
|
||||
if not upstream:
|
||||
return {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"changed": False,
|
||||
"state": "failed",
|
||||
"reason": "upstream_empty",
|
||||
"error": "revision review upstream empty",
|
||||
"published_rows": len(published),
|
||||
"upstream_rows": 0,
|
||||
}
|
||||
listed = self.db.fetchone(
|
||||
"SELECT COUNT(*) AS n FROM stock_master WHERE list_status = 'L'",
|
||||
)
|
||||
listed_n = int((listed or {}).get("n") or 0)
|
||||
floor = float(self.settings.quality.get("daily_row_ratio") or 0.98)
|
||||
if listed_n and len(upstream) / listed_n < floor:
|
||||
return {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"changed": False,
|
||||
"state": "failed",
|
||||
"reason": "incomplete",
|
||||
"error": (
|
||||
f"revision review incomplete: upstream {len(upstream)} "
|
||||
f"/ listed {listed_n} < {floor}"
|
||||
),
|
||||
"published_rows": len(published),
|
||||
"upstream_rows": len(upstream),
|
||||
}
|
||||
if len(upstream) < len(published) * floor:
|
||||
return {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"changed": False,
|
||||
"state": "failed",
|
||||
"reason": "incomplete",
|
||||
"error": (
|
||||
f"revision review incomplete: upstream {len(upstream)} "
|
||||
f"< published {len(published)} * {floor}"
|
||||
),
|
||||
"published_rows": len(published),
|
||||
"upstream_rows": len(upstream),
|
||||
}
|
||||
compared = diff_published_vs_upstream(dataset, published, upstream)
|
||||
compared["trade_date"] = day
|
||||
compared["state"] = "changed" if compared["changed"] else "unchanged"
|
||||
compared["reason"] = "revised" if compared["changed"] else "unchanged"
|
||||
return compared
|
||||
|
||||
def review_published_revisions(self, trade_date: str) -> dict[str, Any]:
|
||||
"""Evening/morning catch-up: compare website fields, republish only on change.
|
||||
|
||||
Unchanged → no new batch. Changed → full A/B boundary quality gate +
|
||||
atomic switch (HEL-459/460/461). Empty/failed/incomplete upstream keeps
|
||||
the previous complete official version.
|
||||
"""
|
||||
day = yyyymmdd(trade_date)
|
||||
results: dict[str, Any] = {}
|
||||
for dataset in revision_datasets(self.settings.quality):
|
||||
compared = self.compare_revision(dataset, day)
|
||||
if compared.get("state") == "skipped":
|
||||
results[dataset] = compared
|
||||
continue
|
||||
if compared.get("state") == "failed":
|
||||
results[dataset] = compared
|
||||
LOGGER.warning(
|
||||
"revision review kept previous official version",
|
||||
extra={
|
||||
"hub": {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"reason": compared.get("reason"),
|
||||
"event": "revision_review_failed",
|
||||
}
|
||||
},
|
||||
)
|
||||
self.audit(
|
||||
"pipeline", "revision-review", f"{dataset}:{day}",
|
||||
json.dumps(
|
||||
{
|
||||
"state": "failed",
|
||||
"reason": compared.get("reason"),
|
||||
"error": compared.get("error"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
)
|
||||
continue
|
||||
if not compared.get("changed"):
|
||||
results[dataset] = {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"state": "aligned",
|
||||
"reason": "unchanged",
|
||||
"published_rows": compared.get("published_rows"),
|
||||
"upstream_rows": compared.get("upstream_rows"),
|
||||
}
|
||||
self.audit(
|
||||
"pipeline", "revision-review", f"{dataset}:{day}",
|
||||
json.dumps({"state": "aligned", "reason": "unchanged"}, ensure_ascii=False),
|
||||
)
|
||||
continue
|
||||
LOGGER.info(
|
||||
"revision review detected upstream rewrite, republishing boundary",
|
||||
extra={
|
||||
"hub": {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"diffs": compared.get("diffs"),
|
||||
"event": "revision_review_changed",
|
||||
}
|
||||
},
|
||||
)
|
||||
try:
|
||||
published = self.force_republish_boundary(dataset, day)
|
||||
except Exception as exc:
|
||||
results[dataset] = {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"state": "failed",
|
||||
"reason": "republish_error",
|
||||
"error": str(exc),
|
||||
"diffs": compared.get("diffs"),
|
||||
}
|
||||
LOGGER.warning(
|
||||
"revision republish failed, previous official version keeps serving",
|
||||
extra={
|
||||
"hub": {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"reason": str(exc),
|
||||
"event": "revision_review_failed",
|
||||
}
|
||||
},
|
||||
)
|
||||
self.audit(
|
||||
"pipeline", "revision-review", f"{dataset}:{day}",
|
||||
json.dumps(
|
||||
{"state": "failed", "reason": "republish_error", "error": str(exc)},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
)
|
||||
continue
|
||||
failures = self.eod_failures(published)
|
||||
if failures:
|
||||
results.update(published)
|
||||
results[dataset] = {
|
||||
**(published.get(dataset) or {}),
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"state": "failed",
|
||||
"reason": "quality_gate",
|
||||
"error": "; ".join(failures),
|
||||
"diffs": compared.get("diffs"),
|
||||
}
|
||||
self.audit(
|
||||
"pipeline", "revision-review", f"{dataset}:{day}",
|
||||
json.dumps(
|
||||
{
|
||||
"state": "failed",
|
||||
"reason": "quality_gate",
|
||||
"error": "; ".join(failures),
|
||||
"diffs": compared.get("diffs"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
)
|
||||
continue
|
||||
results.update(published)
|
||||
results["review"] = {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"state": "aligned",
|
||||
"reason": "revised",
|
||||
"diffs": compared.get("diffs"),
|
||||
"missing_codes": compared.get("missing_codes"),
|
||||
"extra_codes": compared.get("extra_codes"),
|
||||
}
|
||||
self.audit(
|
||||
"pipeline", "revision-review", f"{dataset}:{day}",
|
||||
json.dumps(
|
||||
{
|
||||
"state": "aligned",
|
||||
"reason": "revised",
|
||||
"diffs": compared.get("diffs"),
|
||||
"switched": sorted(
|
||||
name for name, item in published.items()
|
||||
if isinstance(item, dict) and item.get("state") == "published"
|
||||
),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
)
|
||||
return results
|
||||
|
||||
def run_release_group(
|
||||
self,
|
||||
datasets: tuple[str, ...],
|
||||
|
||||
Reference in New Issue
Block a user