d175bb65d4
盘后成功发布后继续轻量比对 daily_basic 网站字段，发现修订才走质量门与整组原子切换，避免 17:10 快照落后于晚间上游改写。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
112 lines
4.1 KiB
Python
112 lines
4.1 KiB
Python
"""Post-publish revision review for datasets whose upstream may rewrite T-day fields.
|
|
|
|
HEL-423 field evidence, not a whitelist of tolerated diffs:
|
|
|
|
- 2026-09-07 valuation/daily_basic: hub published 003021.SZ turnover_rate=1.3565
|
|
at 17:10; website legacy and a direct Tushare read at 21:05 both showed 1.3572.
|
|
The other seven observed objects (daily, moneyflow, auction, stocks, status,
|
|
index_daily, calendar) matched. Hub had already stopped the day after the
|
|
first successful publish, so the revision never self-healed.
|
|
- 2026-09-02: same dataset, opposite direction (hub already held the later
|
|
value). Confirms daily_basic is rewritten after the first complete dump.
|
|
|
|
Daily bars, moneyflow, auction and index_daily have no same-evening field
|
|
revision evidence. Stocks already refreshes at 20:00/23:10. Review therefore
|
|
fetches only configured revision-risk datasets (default: valuation) and
|
|
compares the website-requested field set. No numeric tolerance.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from datahub.db import DATASET_TABLES
|
|
from datahub.normalize import VALUATION_FIELDS
|
|
from datahub.numbers import finite_number, round4
|
|
|
|
# Datasets with proven same-evening upstream rewrites. Config may replace this
|
|
# list; it must not silently expand to a full EOD re-pull.
|
|
DEFAULT_REVISION_DATASETS = ("valuation",)
|
|
|
|
# Website daily_basic request (HEL-423): ts_code/trade_date plus the eight
|
|
# value fields used by the old link and field_gates.
|
|
WEBSITE_COMPARE_FIELDS: dict[str, tuple[str, ...]] = {
|
|
"valuation": VALUATION_FIELDS,
|
|
}
|
|
|
|
REVISION_STATES = ("waiting_review", "review_failed", "aligned", "cutoff")
|
|
|
|
|
|
def revision_datasets(quality: dict[str, Any] | None) -> tuple[str, ...]:
|
|
raw = (quality or {}).get("revision_review_datasets")
|
|
if isinstance(raw, (list, tuple)) and raw:
|
|
names = tuple(str(item) for item in raw if str(item))
|
|
if names:
|
|
return names
|
|
return DEFAULT_REVISION_DATASETS
|
|
|
|
|
|
def compare_fields(dataset: str) -> tuple[str, ...]:
|
|
fields = WEBSITE_COMPARE_FIELDS.get(dataset)
|
|
if fields:
|
|
return fields
|
|
gate = {}
|
|
return tuple(str(item) for item in (gate.get("fields") or []) if str(item))
|
|
|
|
|
|
def _norm_value(field: str, value: Any) -> Any:
|
|
if field in {"ts_code", "trade_date"}:
|
|
return str(value or "")
|
|
number = round4(finite_number(value))
|
|
return number
|
|
|
|
|
|
def row_signature(row: dict[str, Any], fields: tuple[str, ...]) -> tuple[Any, ...]:
|
|
return tuple(_norm_value(field, row.get(field)) for field in fields)
|
|
|
|
|
|
def diff_published_vs_upstream(
|
|
dataset: str,
|
|
published: list[dict[str, Any]],
|
|
upstream: list[dict[str, Any]],
|
|
*,
|
|
max_diffs: int = 20,
|
|
) -> dict[str, Any]:
|
|
"""Exact compare on website-requested fields. No tolerance / exemption."""
|
|
fields = compare_fields(dataset)
|
|
if not fields:
|
|
fields = tuple(sorted({key for row in published + upstream for key in row if key != "batch_id"}))
|
|
pub_map = {str(row.get("ts_code") or "").upper(): row for row in published}
|
|
up_map = {str(row.get("ts_code") or "").upper(): row for row in upstream}
|
|
missing = sorted(code for code in pub_map if code not in up_map)
|
|
extra = sorted(code for code in up_map if code not in pub_map)
|
|
diffs: list[dict[str, Any]] = []
|
|
for code in sorted(set(pub_map) & set(up_map)):
|
|
left = row_signature(pub_map[code], fields)
|
|
right = row_signature(up_map[code], fields)
|
|
if left == right:
|
|
continue
|
|
for field, old, new in zip(fields, left, right):
|
|
if old == new:
|
|
continue
|
|
diffs.append({"ts_code": code, "field": field, "published": old, "upstream": new})
|
|
if len(diffs) >= max_diffs:
|
|
break
|
|
if len(diffs) >= max_diffs:
|
|
break
|
|
changed = bool(diffs or missing or extra)
|
|
return {
|
|
"changed": changed,
|
|
"dataset": dataset,
|
|
"fields": list(fields),
|
|
"published_rows": len(published),
|
|
"upstream_rows": len(upstream),
|
|
"missing_codes": missing[:max_diffs],
|
|
"extra_codes": extra[:max_diffs],
|
|
"diffs": diffs,
|
|
}
|
|
|
|
|
|
def official_table(dataset: str) -> str:
|
|
return DATASET_TABLES[dataset][0]
|