xiaobai-review/backend/data/datahub/compare.py

from __future__ import annotations

from typing import Any

from backend.data.datahub.native import SCALE_FIELDS, row_key, to_canonical_row, yyyymmdd

NUMERIC_TOLERANCE = 1e-4


def compare_rows(
    dataset: str,
    legacy_rows: list[dict[str, Any]],
    hub_rows: list[dict[str, Any]] | None,
    hub_meta: dict[str, Any] | None = None,
    hub_error: str | None = None,
) -> dict[str, Any]:
    hub = hub_rows or []
    legacy_map = {row_key(dataset, row): row for row in legacy_rows}
    hub_map = {row_key(dataset, _align_hub_row(row)): row for row in hub}
    missing_hub = sorted(key for key in legacy_map if key not in hub_map)
    missing_legacy = sorted(key for key in hub_map if key not in legacy_map)
    value_diffs: list[dict[str, Any]] = []
    unit_conversion: list[dict[str, Any]] = []
    matched = 0
    for key, legacy in legacy_map.items():
        hub_row = hub_map.get(key)
        if hub_row is None:
            continue
        field_report = _compare_fields(dataset, legacy, hub_row)
        if field_report["unit_conversion"]:
            unit_conversion.append({"key": list(key), "fields": field_report["unit_conversion"]})
        if field_report["value_diff"]:
            value_diffs.append({"key": list(key), "fields": field_report["value_diff"]})
        if not field_report["unit_conversion"] and not field_report["value_diff"]:
            matched += 1
    stale_seconds = int((hub_meta or {}).get("staleness_seconds") or 0)
    time_skew = bool((hub_meta or {}).get("stale")) or stale_seconds > 0
    return {
        "dataset": dataset,
        "legacy_rows": len(legacy_rows),
        "hub_rows": len(hub),
        "matched": matched,
        "missing_hub": [list(item) for item in missing_hub[:20]],
        "missing_legacy": [list(item) for item in missing_legacy[:20]],
        "missing_hub_count": len(missing_hub),
        "missing_legacy_count": len(missing_legacy),
        "value_diff_count": len(value_diffs),
        "unit_conversion_count": len(unit_conversion),
        "value_diffs": value_diffs[:20],
        "unit_conversion": unit_conversion[:20],
        "time_skew": time_skew,
        "staleness_seconds": stale_seconds,
        "published_at": (hub_meta or {}).get("published_at"),
        "trade_date": yyyymmdd((hub_meta or {}).get("trade_date")),
        "hub_error": hub_error,
        "equal": (
            not hub_error
            and not missing_hub
            and not missing_legacy
            and not value_diffs
            and not unit_conversion
            and not time_skew
        ),
    }


def _align_hub_row(row: dict[str, Any]) -> dict[str, Any]:
    aligned = dict(row)
    if "volume" in aligned and "vol" not in aligned:
        aligned["vol"] = aligned.get("volume")
    return aligned


def _compare_fields(dataset: str, legacy: dict[str, Any], hub: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
    canonical_legacy = to_canonical_row(dataset, legacy)
    hub_canonical = _hub_canonical(dataset, hub)
    native_hub = _align_hub_row(hub)
    value_diff: list[dict[str, Any]] = []
    unit_conversion: list[dict[str, Any]] = []
    keys = (set(canonical_legacy) | set(hub_canonical)) - {"batch_id", "updated_at", "volume"}
    scales = SCALE_FIELDS.get(dataset) or {}
    for field in sorted(keys):
        left = canonical_legacy.get(field)
        right = hub_canonical.get(field)
        if _same(left, right):
            continue
        native_left = legacy.get(field)
        hub_raw = native_hub.get(field)
        if field in scales and _near(_optional(native_left), _optional(hub_raw)):
            unit_conversion.append(
                {"field": field, "legacy": native_left, "hub": hub_raw, "reason": "unit_conversion"}
            )
            continue
        value_diff.append({"field": field, "legacy": left, "hub": right, "reason": "value_diff"})
    return {"value_diff": value_diff, "unit_conversion": unit_conversion}


def _hub_canonical(dataset: str, row: dict[str, Any]) -> dict[str, Any]:
    """Hub API rows are already canonical; only align field names."""
    aligned = dict(row)
    if "volume" in aligned and "vol" not in aligned:
        aligned["vol"] = aligned.get("volume")
    if dataset == "calendar":
        is_open = aligned.get("is_open")
        aligned["is_open"] = 1 if is_open in (True, 1, "1", "Y", "y") else 0
        aligned["cal_date"] = yyyymmdd(aligned.get("cal_date"))
        aligned["pretrade_date"] = yyyymmdd(aligned.get("pretrade_date")) or None
        aligned["exchange"] = str(aligned.get("exchange") or "SSE")
    return aligned


def _same(left: Any, right: Any) -> bool:
    if left in (None, "") and right in (None, ""):
        return True
    if isinstance(left, (int, float)) or isinstance(right, (int, float)):
        return _near(_optional(left), _optional(right))
    return str(left or "") == str(right or "")


def _near(left: float | None, right: float | None) -> bool:
    if left is None and right is None:
        return True
    if left is None or right is None:
        return False
    return abs(left - right) <= max(NUMERIC_TOLERANCE, abs(left) * 1e-9, abs(right) * 1e-9)


def _optional(value: Any) -> float | None:
    if value in (None, ""):
        return None
    try:
        return float(value)
    except (TypeError, ValueError):
        return None