from __future__ import annotations from typing import Any from backend.data.datahub.native import SCALE_FIELDS, row_key, to_canonical_row, yyyymmdd NUMERIC_TOLERANCE = 1e-4 def compare_rows( dataset: str, legacy_rows: list[dict[str, Any]], hub_rows: list[dict[str, Any]] | None, hub_meta: dict[str, Any] | None = None, hub_error: str | None = None, ) -> dict[str, Any]: hub = hub_rows or [] legacy_map = {row_key(dataset, row): row for row in legacy_rows} hub_map = {row_key(dataset, _align_hub_row(row)): row for row in hub} missing_hub = sorted(key for key in legacy_map if key not in hub_map) missing_legacy = sorted(key for key in hub_map if key not in legacy_map) value_diffs: list[dict[str, Any]] = [] unit_conversion: list[dict[str, Any]] = [] matched = 0 for key, legacy in legacy_map.items(): hub_row = hub_map.get(key) if hub_row is None: continue field_report = _compare_fields(dataset, legacy, hub_row) if field_report["unit_conversion"]: unit_conversion.append({"key": list(key), "fields": field_report["unit_conversion"]}) if field_report["value_diff"]: value_diffs.append({"key": list(key), "fields": field_report["value_diff"]}) if not field_report["unit_conversion"] and not field_report["value_diff"]: matched += 1 stale_seconds = int((hub_meta or {}).get("staleness_seconds") or 0) time_skew = bool((hub_meta or {}).get("stale")) or stale_seconds > 0 return { "dataset": dataset, "legacy_rows": len(legacy_rows), "hub_rows": len(hub), "matched": matched, "missing_hub": [list(item) for item in missing_hub[:20]], "missing_legacy": [list(item) for item in missing_legacy[:20]], "missing_hub_count": len(missing_hub), "missing_legacy_count": len(missing_legacy), "value_diff_count": len(value_diffs), "unit_conversion_count": len(unit_conversion), "value_diffs": value_diffs[:20], "unit_conversion": unit_conversion[:20], "time_skew": time_skew, "staleness_seconds": stale_seconds, "published_at": (hub_meta or {}).get("published_at"), "trade_date": yyyymmdd((hub_meta or {}).get("trade_date")), "hub_error": hub_error, "equal": ( not hub_error and not missing_hub and not missing_legacy and not value_diffs and not unit_conversion and not time_skew ), } def _align_hub_row(row: dict[str, Any]) -> dict[str, Any]: aligned = dict(row) if "volume" in aligned and "vol" not in aligned: aligned["vol"] = aligned.get("volume") return aligned def _compare_fields(dataset: str, legacy: dict[str, Any], hub: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: canonical_legacy = to_canonical_row(dataset, legacy) hub_canonical = _hub_canonical(dataset, hub) native_hub = _align_hub_row(hub) value_diff: list[dict[str, Any]] = [] unit_conversion: list[dict[str, Any]] = [] keys = (set(canonical_legacy) | set(hub_canonical)) - {"batch_id", "updated_at", "volume"} scales = SCALE_FIELDS.get(dataset) or {} for field in sorted(keys): left = canonical_legacy.get(field) right = hub_canonical.get(field) if _same(left, right): continue native_left = legacy.get(field) hub_raw = native_hub.get(field) if field in scales and _near(_optional(native_left), _optional(hub_raw)): unit_conversion.append( {"field": field, "legacy": native_left, "hub": hub_raw, "reason": "unit_conversion"} ) continue value_diff.append({"field": field, "legacy": left, "hub": right, "reason": "value_diff"}) return {"value_diff": value_diff, "unit_conversion": unit_conversion} def _hub_canonical(dataset: str, row: dict[str, Any]) -> dict[str, Any]: """Hub API rows are already canonical; only align field names.""" aligned = dict(row) if "volume" in aligned and "vol" not in aligned: aligned["vol"] = aligned.get("volume") if dataset == "calendar": is_open = aligned.get("is_open") aligned["is_open"] = 1 if is_open in (True, 1, "1", "Y", "y") else 0 aligned["cal_date"] = yyyymmdd(aligned.get("cal_date")) aligned["pretrade_date"] = yyyymmdd(aligned.get("pretrade_date")) or None aligned["exchange"] = str(aligned.get("exchange") or "SSE") return aligned def _same(left: Any, right: Any) -> bool: if left in (None, "") and right in (None, ""): return True if isinstance(left, (int, float)) or isinstance(right, (int, float)): return _near(_optional(left), _optional(right)) return str(left or "") == str(right or "") def _near(left: float | None, right: float | None) -> bool: if left is None and right is None: return True if left is None or right is None: return False return abs(left - right) <= max(NUMERIC_TOLERANCE, abs(left) * 1e-9, abs(right) * 1e-9) def _optional(value: Any) -> float | None: if value in (None, ""): return None try: return float(value) except (TypeError, ValueError): return None