fix(HEL-459): 影子比较按请求字段投影，盘后整批原子发布
比较侧只对网站本次请求字段计业务差异，忽略数据中枢额外列； 盘后 A/B/重发改为先整批暂存与交叉校验，再单事务切换公开版本。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
co-authored by
Cursor
multica-agent
Cursor
multica-agent
parent
bed6450992
commit
16841e9ae3
@@ -133,6 +133,95 @@ def _staging_row_count(connection: Any, dataset: str, batch_id: str) -> int:
|
||||
return int(row["n"] if row is not None else 0)
|
||||
|
||||
|
||||
def _staging_count_or_raise(connection: Any, dataset: str, trade_date: str, batch_id: str) -> int:
|
||||
rows_out = _staging_row_count(connection, dataset, batch_id)
|
||||
if rows_out <= 0:
|
||||
report = {
|
||||
"rows": 0,
|
||||
"errors": [EMPTY_BATCH_ERROR],
|
||||
"warnings": [],
|
||||
"hard_fail": True,
|
||||
"soft_fail": False,
|
||||
"batch_id": batch_id,
|
||||
"dataset": dataset,
|
||||
"trade_date": trade_date,
|
||||
}
|
||||
LOGGER.warning(
|
||||
"skip official publish for empty batch",
|
||||
extra={
|
||||
"hub": {
|
||||
"dataset": dataset,
|
||||
"trade_date": trade_date,
|
||||
"batch_id": batch_id,
|
||||
"rows_out": rows_out,
|
||||
"reason": "upstream_empty",
|
||||
}
|
||||
},
|
||||
)
|
||||
raise QualityError("empty batch cannot be officially published", report)
|
||||
return rows_out
|
||||
|
||||
|
||||
def _upsert_publication(
|
||||
connection: Any,
|
||||
dataset: str,
|
||||
trade_date: str,
|
||||
batch_id: str,
|
||||
state: str,
|
||||
published_at: str,
|
||||
) -> None:
|
||||
current = connection.execute(
|
||||
"SELECT active_batch FROM publications WHERE dataset = ? AND trade_date = ?",
|
||||
(dataset, trade_date),
|
||||
).fetchone()
|
||||
prev = str(current["active_batch"]) if current else None
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO publications(dataset, trade_date, active_batch, prev_batch, state, published_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(dataset, trade_date) DO UPDATE SET
|
||||
prev_batch=excluded.prev_batch,
|
||||
active_batch=excluded.active_batch,
|
||||
state=excluded.state,
|
||||
published_at=excluded.published_at
|
||||
""",
|
||||
(dataset, trade_date, batch_id, prev, state, published_at),
|
||||
)
|
||||
|
||||
|
||||
def _record_publication_history(
|
||||
connection: Any,
|
||||
dataset: str,
|
||||
trade_date: str,
|
||||
batch_id: str,
|
||||
published_at: str,
|
||||
quality: dict[str, Any],
|
||||
) -> None:
|
||||
max_gen = connection.execute(
|
||||
"SELECT COALESCE(MAX(generation), 0) AS g FROM publication_history WHERE dataset = ? AND trade_date = ?",
|
||||
(dataset, trade_date),
|
||||
).fetchone()
|
||||
generation = int(max_gen["g"]) + 1
|
||||
connection.execute(
|
||||
"INSERT OR REPLACE INTO publication_history(dataset, trade_date, batch_id, published_at, generation) VALUES (?,?,?,?,?)",
|
||||
(dataset, trade_date, batch_id, published_at, generation),
|
||||
)
|
||||
keep = int(quality.get("publication_generations") or 3)
|
||||
stale = connection.execute(
|
||||
"""
|
||||
SELECT batch_id FROM publication_history
|
||||
WHERE dataset = ? AND trade_date = ?
|
||||
ORDER BY generation DESC
|
||||
""",
|
||||
(dataset, trade_date),
|
||||
).fetchall()
|
||||
for row in stale[keep:]:
|
||||
connection.execute(
|
||||
"DELETE FROM publication_history WHERE dataset = ? AND trade_date = ? AND batch_id = ?",
|
||||
(dataset, trade_date, row["batch_id"]),
|
||||
)
|
||||
|
||||
|
||||
class QualityError(RuntimeError):
|
||||
def __init__(self, message: str, report: dict[str, Any]) -> None:
|
||||
super().__init__(message)
|
||||
@@ -551,20 +640,40 @@ class Pipeline:
|
||||
"""Fetch/publish every official dataset still missing for the date.
|
||||
|
||||
Idempotent: datasets with an existing publication are skipped, so
|
||||
repeats never overwrite the current official batch. Per-dataset
|
||||
failures are collected instead of aborting the remaining datasets.
|
||||
repeats never overwrite the current official batch. All missing
|
||||
members stage first and switch in one atomic release group; a single
|
||||
member failure keeps the previous complete official version serving.
|
||||
"""
|
||||
return self._run_eod_datasets(tuple(sorted(OFFICIAL_DATASETS)), trade_date)
|
||||
return self.run_release_group(tuple(sorted(OFFICIAL_DATASETS)), trade_date, include_stocks=True)
|
||||
|
||||
def run_eod_batch_a(self, trade_date: str) -> dict[str, Any]:
|
||||
return self._run_eod_datasets(EOD_A_DATASETS, trade_date)
|
||||
return self.run_release_group(EOD_A_DATASETS, trade_date, include_stocks=True)
|
||||
|
||||
def run_eod_batch_b(self, trade_date: str) -> dict[str, Any]:
|
||||
return self._run_eod_datasets(EOD_B_DATASETS, trade_date)
|
||||
return self.run_release_group(EOD_B_DATASETS, trade_date)
|
||||
|
||||
def _run_eod_datasets(self, datasets: tuple[str, ...], trade_date: str) -> dict[str, Any]:
|
||||
def run_release_group(
|
||||
self,
|
||||
datasets: tuple[str, ...],
|
||||
trade_date: str,
|
||||
include_stocks: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""One post-market publish/republish becomes one atomic visibility flip.
|
||||
|
||||
Consistency boundary: every member (official datasets, plus the daily
|
||||
stocks snapshot when ``include_stocks`` and not yet published) is
|
||||
fetched, staged, field-gated and cross-validated BEFORE any reader can
|
||||
see it. Only when the whole group passes does a single SQLite
|
||||
transaction copy all staging batches to the official tables and flip
|
||||
every ``publications`` row at once. Any member failure aborts the
|
||||
group: the previous complete official version keeps serving and the
|
||||
reason is recorded on the batches and in the audit log.
|
||||
"""
|
||||
day = yyyymmdd(trade_date)
|
||||
results: dict[str, Any] = {}
|
||||
staged: dict[str, dict[str, Any]] = {}
|
||||
failure: str | None = None
|
||||
pending: list[str] = []
|
||||
for dataset in datasets:
|
||||
if self.active_batch(dataset, day) is not None:
|
||||
results[dataset] = {
|
||||
@@ -573,18 +682,247 @@ class Pipeline:
|
||||
"state": "skipped",
|
||||
"reason": "already_published",
|
||||
}
|
||||
else:
|
||||
pending.append(dataset)
|
||||
|
||||
for dataset in pending:
|
||||
if failure is not None:
|
||||
results[dataset] = {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"state": "aborted",
|
||||
"reason": f"release group aborted: {failure}",
|
||||
}
|
||||
continue
|
||||
try:
|
||||
results[dataset] = self.run_dataset(dataset, day)
|
||||
staged[dataset] = self._stage_and_validate(dataset, day)
|
||||
except Exception as exc:
|
||||
failure = f"{dataset}: {exc}"
|
||||
results[dataset] = {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"state": "failed",
|
||||
"error": str(exc),
|
||||
}
|
||||
|
||||
if include_stocks and failure is None and self.active_batch(STOCKS_DATASET, day) is None:
|
||||
try:
|
||||
stocks_plan = self._stage_stocks_snapshot(day)
|
||||
except Exception as exc:
|
||||
failure = f"{STOCKS_DATASET}: {exc}"
|
||||
results[STOCKS_DATASET] = {
|
||||
"dataset": STOCKS_DATASET,
|
||||
"trade_date": day,
|
||||
"state": "failed",
|
||||
"error": str(exc),
|
||||
}
|
||||
else:
|
||||
if stocks_plan is not None:
|
||||
staged[STOCKS_DATASET] = stocks_plan
|
||||
|
||||
if failure is None and staged:
|
||||
cross_errors = self._cross_gate_errors(staged)
|
||||
if cross_errors:
|
||||
failure = "; ".join(cross_errors)
|
||||
|
||||
if failure is not None:
|
||||
for dataset, item in staged.items():
|
||||
self._abandon_batch(item["batch_id"], f"release group not switched: {failure}")
|
||||
results[dataset] = {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"state": "failed",
|
||||
"error": f"release group not switched: {failure}",
|
||||
"batch_id": item["batch_id"],
|
||||
}
|
||||
LOGGER.warning(
|
||||
"release group blocked, previous official version keeps serving",
|
||||
extra={
|
||||
"hub": {
|
||||
"trade_date": day,
|
||||
"datasets": sorted(staged),
|
||||
"reason": failure,
|
||||
"event": "release_group_blocked",
|
||||
}
|
||||
},
|
||||
)
|
||||
self.audit(
|
||||
"pipeline", "release-group", f"eod:{day}",
|
||||
json.dumps({"state": "failed", "reason": failure}, ensure_ascii=False),
|
||||
)
|
||||
return results
|
||||
|
||||
if staged:
|
||||
try:
|
||||
self._switch_release_group(day, staged)
|
||||
except Exception as exc:
|
||||
for item in staged.values():
|
||||
self._abandon_batch(item["batch_id"], f"release group switch failed: {exc}")
|
||||
raise
|
||||
for dataset, item in staged.items():
|
||||
results[dataset] = {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"state": item["state"],
|
||||
"batch_id": item["batch_id"],
|
||||
"rows": item["rows"],
|
||||
}
|
||||
self.audit(
|
||||
"pipeline", "release-group", f"eod:{day}",
|
||||
json.dumps(
|
||||
{"state": "ok", "switched": sorted(staged)}, ensure_ascii=False
|
||||
),
|
||||
)
|
||||
return results
|
||||
|
||||
def _stage_and_validate(self, dataset: str, trade_date: str, attempts: int | None = None) -> dict[str, Any]:
|
||||
"""Fetch → stage → quality-gate one member without publishing it."""
|
||||
day = yyyymmdd(trade_date)
|
||||
batch_id = self.next_batch_id(dataset, day)
|
||||
max_attempts = attempts or self.settings.max_publish_attempts
|
||||
rows: list[dict[str, Any]] = []
|
||||
self._set_batch(batch_id, dataset, day, "scheduled", 0)
|
||||
try:
|
||||
self._set_batch(batch_id, dataset, day, "fetching", 1)
|
||||
rows = retry_call(
|
||||
lambda: self._fetch_dataset(dataset, day),
|
||||
attempts=max_attempts,
|
||||
base_delay=0.05,
|
||||
sleeper=lambda _d: time.sleep(_d),
|
||||
)
|
||||
self._stage(dataset, batch_id, rows)
|
||||
self._set_batch(batch_id, dataset, day, "staged", 1, rows_in=len(rows), rows_out=len(rows))
|
||||
self._set_batch(batch_id, dataset, day, "validating", 1, rows_in=len(rows), rows_out=len(rows))
|
||||
report = self.validate(dataset, batch_id, day, rows)
|
||||
if report["hard_fail"]:
|
||||
self._reject_batch(batch_id, dataset, day, rows, report)
|
||||
raise QualityError("integrity gate failed", report)
|
||||
except RetryError as exc:
|
||||
self._set_batch(batch_id, dataset, day, "failed", max_attempts, error=str(exc), finished=True)
|
||||
raise
|
||||
except QualityError as exc:
|
||||
current = self.db.fetchone("SELECT state FROM batches WHERE batch_id = ?", (batch_id,))
|
||||
if current and current["state"] not in {"staged", "failed"}:
|
||||
self._reject_batch(batch_id, dataset, day, rows, exc.report)
|
||||
raise
|
||||
except Exception as exc:
|
||||
self._set_batch(batch_id, dataset, day, "failed", 1, error=str(exc), finished=True)
|
||||
raise
|
||||
self._set_batch(
|
||||
batch_id, dataset, day, "ready", 1, rows_in=len(rows), rows_out=len(rows), quality=report
|
||||
)
|
||||
return {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"batch_id": batch_id,
|
||||
"rows": len(rows),
|
||||
"quality": report,
|
||||
"state": "degraded" if report["soft_fail"] else "published",
|
||||
}
|
||||
|
||||
def _stage_stocks_snapshot(self, trade_date: str) -> dict[str, Any] | None:
|
||||
"""Stage the daily stocks snapshot for a release group switch.
|
||||
|
||||
Returns None when the published snapshot is already identical to
|
||||
upstream (idempotent skip). The stock_master upsert is deferred into
|
||||
the group switch transaction so the master never runs ahead of the
|
||||
published snapshot.
|
||||
"""
|
||||
day = yyyymmdd(trade_date)
|
||||
active, snapshot = self.published_stock_snapshot(day)
|
||||
rows = self._fetch_dataset(STOCKS_DATASET, day)
|
||||
if active is not None:
|
||||
upstream = sorted(
|
||||
tuple(str(row.get(field)) for field in STOCK_SNAPSHOT_FIELDS) for row in rows
|
||||
)
|
||||
published = sorted(tuple(str(row.get(field)) for field in STOCK_SNAPSHOT_FIELDS) for row in snapshot)
|
||||
if upstream == published:
|
||||
return None
|
||||
batch_id = self.next_batch_id(STOCKS_DATASET, day)
|
||||
self._set_batch(batch_id, STOCKS_DATASET, day, "scheduled", 0)
|
||||
self._set_batch(batch_id, STOCKS_DATASET, day, "fetching", 1)
|
||||
self._stage(STOCKS_DATASET, batch_id, rows)
|
||||
self._set_batch(batch_id, STOCKS_DATASET, day, "staged", 1, rows_in=len(rows), rows_out=len(rows))
|
||||
self._set_batch(batch_id, STOCKS_DATASET, day, "validating", 1, rows_in=len(rows), rows_out=len(rows))
|
||||
report = self.validate(STOCKS_DATASET, batch_id, day, rows)
|
||||
if report["hard_fail"]:
|
||||
self._reject_batch(batch_id, STOCKS_DATASET, day, rows, report)
|
||||
raise QualityError("integrity gate failed", report)
|
||||
self._set_batch(
|
||||
batch_id, STOCKS_DATASET, day, "ready", 1, rows_in=len(rows), rows_out=len(rows), quality=report
|
||||
)
|
||||
return {
|
||||
"dataset": STOCKS_DATASET,
|
||||
"trade_date": day,
|
||||
"batch_id": batch_id,
|
||||
"rows": len(rows),
|
||||
"row_values": rows,
|
||||
"quality": report,
|
||||
"state": "degraded" if report["soft_fail"] else "published",
|
||||
}
|
||||
|
||||
def _cross_gate_errors(self, staged: dict[str, dict[str, Any]]) -> list[str]:
|
||||
"""Cross-dataset consistency checks on staged batches (交叉校验)."""
|
||||
errors: list[str] = []
|
||||
gates = self.settings.quality.get("cross_gates") or []
|
||||
for gate in gates if isinstance(gates, list) else []:
|
||||
if not isinstance(gate, dict):
|
||||
continue
|
||||
left = str(gate.get("left") or "")
|
||||
right = str(gate.get("right") or "")
|
||||
if not left or not right or left not in staged or right not in staged:
|
||||
continue
|
||||
floor = float(gate.get("min_key_overlap") or 0.98)
|
||||
left_keys = self._staging_keys(left, staged[left]["batch_id"])
|
||||
right_keys = self._staging_keys(right, staged[right]["batch_id"])
|
||||
denom = max(len(left_keys), len(right_keys))
|
||||
overlap = (len(left_keys & right_keys) / denom) if denom else 1.0
|
||||
if overlap < floor:
|
||||
errors.append(
|
||||
f"cross gate: {left} vs {right} key overlap {overlap:.4f} < {floor}"
|
||||
)
|
||||
return errors
|
||||
|
||||
def _staging_keys(self, dataset: str, batch_id: str) -> set[str]:
|
||||
table = DATASET_TABLES[dataset][1]
|
||||
rows = self.db.fetchall(
|
||||
f"SELECT DISTINCT ts_code FROM {table} WHERE batch_id = ?",
|
||||
(batch_id,),
|
||||
)
|
||||
return {str(row["ts_code"]) for row in rows}
|
||||
|
||||
def _switch_release_group(self, trade_date: str, members: dict[str, dict[str, Any]]) -> None:
|
||||
"""Single transaction: copy every member and flip every publication."""
|
||||
day = yyyymmdd(trade_date)
|
||||
published_at = isoformat(self.clock())
|
||||
with self.db.write() as connection:
|
||||
for dataset, item in members.items():
|
||||
_staging_count_or_raise(connection, dataset, day, item["batch_id"])
|
||||
for dataset, item in members.items():
|
||||
connection.execute(EOD_COPY[dataset], (item["batch_id"],))
|
||||
if dataset == STOCKS_DATASET:
|
||||
self._upsert_stock_master(connection, item["row_values"], published_at)
|
||||
if self.before_commit:
|
||||
self.before_commit()
|
||||
for dataset, item in members.items():
|
||||
_upsert_publication(connection, dataset, day, item["batch_id"], item["state"], published_at)
|
||||
_record_publication_history(
|
||||
connection, dataset, day, item["batch_id"], published_at, self.settings.quality
|
||||
)
|
||||
connection.execute(
|
||||
"UPDATE batches SET state='published', finished_at=? WHERE batch_id=?",
|
||||
(published_at, item["batch_id"]),
|
||||
)
|
||||
|
||||
def _abandon_batch(self, batch_id: str, reason: str) -> None:
|
||||
row = self.db.fetchone("SELECT dataset, trade_date FROM batches WHERE batch_id = ?", (batch_id,))
|
||||
if not row:
|
||||
return
|
||||
self._set_batch(
|
||||
batch_id, str(row["dataset"]), str(row["trade_date"]), "failed", 1,
|
||||
error=reason, finished=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def eod_failures(results: dict[str, Any]) -> list[str]:
|
||||
return [
|
||||
@@ -735,77 +1073,14 @@ class Pipeline:
|
||||
return batch_id, stats
|
||||
|
||||
def publish(self, dataset: str, trade_date: str, batch_id: str, state: str = "published") -> None:
|
||||
copy_sql = EOD_COPY[dataset]
|
||||
published_at = isoformat(self.clock())
|
||||
with self.db.write() as connection:
|
||||
rows_out = _staging_row_count(connection, dataset, batch_id)
|
||||
if rows_out <= 0:
|
||||
report = {
|
||||
"rows": 0,
|
||||
"errors": [EMPTY_BATCH_ERROR],
|
||||
"warnings": [],
|
||||
"hard_fail": True,
|
||||
"soft_fail": False,
|
||||
"batch_id": batch_id,
|
||||
"dataset": dataset,
|
||||
"trade_date": trade_date,
|
||||
}
|
||||
LOGGER.warning(
|
||||
"skip official publish for empty batch",
|
||||
extra={
|
||||
"hub": {
|
||||
"dataset": dataset,
|
||||
"trade_date": trade_date,
|
||||
"batch_id": batch_id,
|
||||
"rows_out": rows_out,
|
||||
"reason": "upstream_empty",
|
||||
}
|
||||
},
|
||||
)
|
||||
raise QualityError("empty batch cannot be officially published", report)
|
||||
current = connection.execute(
|
||||
"SELECT active_batch FROM publications WHERE dataset = ? AND trade_date = ?",
|
||||
(dataset, trade_date),
|
||||
).fetchone()
|
||||
prev = str(current["active_batch"]) if current else None
|
||||
connection.execute(copy_sql, (batch_id,))
|
||||
_staging_count_or_raise(connection, dataset, trade_date, batch_id)
|
||||
connection.execute(EOD_COPY[dataset], (batch_id,))
|
||||
if self.before_commit:
|
||||
self.before_commit()
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO publications(dataset, trade_date, active_batch, prev_batch, state, published_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(dataset, trade_date) DO UPDATE SET
|
||||
prev_batch=excluded.prev_batch,
|
||||
active_batch=excluded.active_batch,
|
||||
state=excluded.state,
|
||||
published_at=excluded.published_at
|
||||
""",
|
||||
(dataset, trade_date, batch_id, prev, state, published_at),
|
||||
)
|
||||
max_gen = connection.execute(
|
||||
"SELECT COALESCE(MAX(generation), 0) AS g FROM publication_history WHERE dataset = ? AND trade_date = ?",
|
||||
(dataset, trade_date),
|
||||
).fetchone()
|
||||
generation = int(max_gen["g"]) + 1
|
||||
connection.execute(
|
||||
"INSERT OR REPLACE INTO publication_history(dataset, trade_date, batch_id, published_at, generation) VALUES (?,?,?,?,?)",
|
||||
(dataset, trade_date, batch_id, published_at, generation),
|
||||
)
|
||||
keep = int(self.settings.quality.get("publication_generations") or 3)
|
||||
stale = connection.execute(
|
||||
"""
|
||||
SELECT batch_id FROM publication_history
|
||||
WHERE dataset = ? AND trade_date = ?
|
||||
ORDER BY generation DESC
|
||||
""",
|
||||
(dataset, trade_date),
|
||||
).fetchall()
|
||||
for row in stale[keep:]:
|
||||
connection.execute(
|
||||
"DELETE FROM publication_history WHERE dataset = ? AND trade_date = ? AND batch_id = ?",
|
||||
(dataset, trade_date, row["batch_id"]),
|
||||
)
|
||||
_upsert_publication(connection, dataset, trade_date, batch_id, state, published_at)
|
||||
_record_publication_history(connection, dataset, trade_date, batch_id, published_at, self.settings.quality)
|
||||
|
||||
def rollback(self, dataset: str, trade_date: str, actor: str = "admin") -> dict[str, Any]:
|
||||
trade_date = yyyymmdd(trade_date)
|
||||
|
||||
Reference in New Issue
Block a user