fix(HEL-396): 禁止空数据批次冒充正式发布
统一发布入口在有效行数为 0 时不再写成 published，也不推进正式批次指针。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
co-authored by
Cursor
multica-agent
Cursor
multica-agent
parent
031eefab4d
commit
f5dc0f8076
@@ -21,6 +21,8 @@ LOGGER = get_logger()
|
||||
|
||||
HARD_DATASETS = {"daily", "valuation", "index_daily"}
|
||||
SOFT_DATASETS = {"moneyflow", "auction"}
|
||||
OFFICIAL_DATASETS = HARD_DATASETS | SOFT_DATASETS
|
||||
EMPTY_BATCH_ERROR = "empty official batch: 0 valid rows"
|
||||
|
||||
STAGING_INSERT = {
|
||||
"daily": (
|
||||
@@ -97,6 +99,15 @@ EOD_COPY = {
|
||||
}
|
||||
|
||||
|
||||
def _staging_row_count(connection: Any, dataset: str, batch_id: str) -> int:
|
||||
table = DATASET_TABLES[dataset][1]
|
||||
row = connection.execute(
|
||||
f"SELECT COUNT(*) AS n FROM {table} WHERE batch_id = ?",
|
||||
(batch_id,),
|
||||
).fetchone()
|
||||
return int(row["n"] if row is not None else 0)
|
||||
|
||||
|
||||
class QualityError(RuntimeError):
|
||||
def __init__(self, message: str, report: dict[str, Any]) -> None:
|
||||
super().__init__(message)
|
||||
@@ -176,6 +187,7 @@ class Pipeline:
|
||||
batch_id = self.next_batch_id(dataset, trade_date)
|
||||
max_attempts = attempts or self.settings.max_publish_attempts
|
||||
self._set_batch(batch_id, dataset, trade_date, "scheduled", 0)
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
self._set_batch(batch_id, dataset, trade_date, "fetching", 1)
|
||||
rows = retry_call(
|
||||
@@ -189,11 +201,7 @@ class Pipeline:
|
||||
self._set_batch(batch_id, dataset, trade_date, "validating", 1)
|
||||
report = self.validate(dataset, batch_id, trade_date, rows)
|
||||
if report["hard_fail"]:
|
||||
self._set_batch(
|
||||
batch_id, dataset, trade_date, "staged", 1,
|
||||
rows_in=len(rows), rows_out=len(rows),
|
||||
quality=report, error="; ".join(report["errors"]),
|
||||
)
|
||||
self._reject_batch(batch_id, dataset, trade_date, rows, report)
|
||||
raise QualityError("integrity gate failed", report)
|
||||
self._set_batch(batch_id, dataset, trade_date, "deriving", 1, rows_in=len(rows), rows_out=len(rows), quality=report)
|
||||
self._set_batch(batch_id, dataset, trade_date, "publishing", 1, rows_in=len(rows), rows_out=len(rows), quality=report)
|
||||
@@ -207,7 +215,10 @@ class Pipeline:
|
||||
except RetryError as exc:
|
||||
self._set_batch(batch_id, dataset, trade_date, "failed", max_attempts, error=str(exc), finished=True)
|
||||
raise
|
||||
except QualityError:
|
||||
except QualityError as exc:
|
||||
current = self.db.fetchone("SELECT state FROM batches WHERE batch_id = ?", (batch_id,))
|
||||
if current and current["state"] not in {"staged", "failed"}:
|
||||
self._reject_batch(batch_id, dataset, trade_date, rows, exc.report)
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._set_batch(batch_id, dataset, trade_date, "failed", 1, error=str(exc), finished=True)
|
||||
@@ -247,11 +258,13 @@ class Pipeline:
|
||||
null_rate = nulls / row_n
|
||||
if null_rate >= float(quality.get("null_rate_max") or 0.01):
|
||||
errors.append(f"null rate {null_rate:.4f}")
|
||||
if dataset in SOFT_DATASETS and row_n == 0:
|
||||
warnings.append("empty soft dataset")
|
||||
hard_fail = bool(errors) and dataset in HARD_DATASETS.union({"daily", "valuation", "index_daily"})
|
||||
empty = row_n == 0
|
||||
if empty and dataset in OFFICIAL_DATASETS:
|
||||
errors.append(EMPTY_BATCH_ERROR)
|
||||
if dataset in SOFT_DATASETS:
|
||||
hard_fail = bool(dup or bad_date)
|
||||
hard_fail = bool(dup or bad_date or empty)
|
||||
else:
|
||||
hard_fail = bool(errors) and dataset in HARD_DATASETS
|
||||
return {
|
||||
"rows": row_n,
|
||||
"listed": listed_n,
|
||||
@@ -267,6 +280,31 @@ class Pipeline:
|
||||
copy_sql = EOD_COPY[dataset]
|
||||
published_at = isoformat(self.clock())
|
||||
with self.db.write() as connection:
|
||||
rows_out = _staging_row_count(connection, dataset, batch_id)
|
||||
if rows_out <= 0:
|
||||
report = {
|
||||
"rows": 0,
|
||||
"errors": [EMPTY_BATCH_ERROR],
|
||||
"warnings": [],
|
||||
"hard_fail": True,
|
||||
"soft_fail": False,
|
||||
"batch_id": batch_id,
|
||||
"dataset": dataset,
|
||||
"trade_date": trade_date,
|
||||
}
|
||||
LOGGER.warning(
|
||||
"skip official publish for empty batch",
|
||||
extra={
|
||||
"hub": {
|
||||
"dataset": dataset,
|
||||
"trade_date": trade_date,
|
||||
"batch_id": batch_id,
|
||||
"rows_out": rows_out,
|
||||
"reason": "upstream_empty",
|
||||
}
|
||||
},
|
||||
)
|
||||
raise QualityError("empty batch cannot be officially published", report)
|
||||
current = connection.execute(
|
||||
"SELECT active_batch FROM publications WHERE dataset = ? AND trade_date = ?",
|
||||
(dataset, trade_date),
|
||||
@@ -364,6 +402,34 @@ class Pipeline:
|
||||
(actor, action, target, detail, isoformat(self.clock())),
|
||||
)
|
||||
|
||||
def _reject_batch(
|
||||
self,
|
||||
batch_id: str,
|
||||
dataset: str,
|
||||
trade_date: str,
|
||||
rows: list[dict[str, Any]],
|
||||
report: dict[str, Any],
|
||||
) -> None:
|
||||
errors = report.get("errors") or []
|
||||
LOGGER.warning(
|
||||
"official batch rejected",
|
||||
extra={
|
||||
"hub": {
|
||||
"dataset": dataset,
|
||||
"trade_date": trade_date,
|
||||
"batch_id": batch_id,
|
||||
"rows_out": len(rows),
|
||||
"errors": errors,
|
||||
"reason": "upstream_empty" if EMPTY_BATCH_ERROR in errors else "integrity_gate",
|
||||
}
|
||||
},
|
||||
)
|
||||
self._set_batch(
|
||||
batch_id, dataset, trade_date, "staged", 1,
|
||||
rows_in=len(rows), rows_out=len(rows),
|
||||
quality=report, error="; ".join(str(item) for item in errors),
|
||||
)
|
||||
|
||||
def _fetch_dataset(self, dataset: str, trade_date: str) -> list[dict[str, Any]]:
|
||||
if dataset == "daily":
|
||||
raw = self._guarded_fetch("daily", {"trade_date": trade_date})
|
||||
|
||||
Reference in New Issue
Block a user