migration: establish exact preserved app baseline

app/backend/data/__init__.py

@@ -0,0 +1,14 @@
+from .gateway import DataGateway, build_data_gateway
+from .policy import DataPolicyError, DataSourcePolicy
+from .quality import DataQualityError, DataQualityGate, QualityEvidence, QualityReport
+
+__all__ = [
+    "DataGateway",
+    "DataPolicyError",
+    "DataQualityError",
+    "DataQualityGate",
+    "DataSourcePolicy",
+    "QualityEvidence",
+    "QualityReport",
+    "build_data_gateway",
+]

app/backend/data/contracts.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+
+DataUsage = Literal["display", "calculation"]
+
+
+@dataclass(frozen=True)
+class ProviderContract:
+    id: str
+    provider_class: str
+    calculation_allowed: bool
+
+
+@dataclass(frozen=True)
+class DatasetContract:
+    id: str
+    entity: str
+    frequency: str
+    primary: str
+    fallbacks: tuple[str, ...]
+    usage: str
+    fields: tuple[str, ...]
+
+    @property
+    def providers(self) -> tuple[str, ...]:
+        return (self.primary, *self.fallbacks)

app/backend/data/gateway.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime
+
+from backend.data.contracts import DataUsage
+from backend.data.policy import DataSourcePolicy
+from backend.data.providers import IfindProvider, TushareProvider
+from backend.data.quality import DataQualityGate, QualityEvidence, QualityReport
+from chart_data_provider import EastmoneyChartClient, MarketChartClient
+from ifind_client import IfindHttpClient
+from realtime_aggregator import WebRealtimeAggregator
+from tushare_client import TushareClient
+
+
+@dataclass(frozen=True)
+class DataGateway:
+    policy: DataSourcePolicy
+    quality: DataQualityGate
+    tushare_provider: TushareProvider
+    ifind_provider: IfindProvider
+    chart_data: MarketChartClient
+    realtime_observer: WebRealtimeAggregator
+
+    @property
+    def ifind(self) -> IfindHttpClient:
+        return self.ifind_provider.client
+
+    def tushare(
+        self,
+        dataset_id: str = "",
+        usage: DataUsage = "calculation",
+    ) -> TushareClient:
+        if dataset_id:
+            self.policy.assert_allowed(dataset_id, "tushare", usage)
+        return self.tushare_provider.client()
+
+    def assert_source(self, dataset_id: str, provider_id: str, usage: DataUsage) -> None:
+        self.policy.assert_allowed(dataset_id, provider_id, usage)
+
+    def provider_chain(self, dataset_id: str, usage: DataUsage) -> tuple[str, ...]:
+        dataset = self.policy.dataset(dataset_id)
+        allowed = []
+        for provider_id in dataset.providers:
+            try:
+                self.policy.assert_allowed(dataset_id, provider_id, usage)
+            except Exception:
+                continue
+            allowed.append(provider_id)
+        if not allowed:
+            raise RuntimeError(f"No permitted provider for {dataset_id} ({usage})")
+        return tuple(allowed)
+
+    def require_quality(
+        self,
+        evidence: QualityEvidence,
+        usage: DataUsage,
+        as_of: str | datetime | None = None,
+    ) -> QualityReport:
+        return self.quality.require(evidence, usage, as_of)
+
+
+def build_data_gateway(
+    credentials: dict[str, object],
+    tushare_token_supplier: Callable[[], str] | None = None,
+) -> DataGateway:
+    ifind = IfindHttpClient(
+        str(credentials.get("ifind_refresh_token") or ""),
+        str(credentials.get("ifind_access_token") or ""),
+    )
+    token_supplier = tushare_token_supplier or (
+        lambda: str(credentials.get("tushare_token") or "")
+    )
+    policy = DataSourcePolicy.load()
+    return DataGateway(
+        policy=policy,
+        quality=DataQualityGate.load(policy),
+        tushare_provider=TushareProvider(token_supplier),
+        ifind_provider=IfindProvider(ifind),
+        chart_data=MarketChartClient(ifind, EastmoneyChartClient()),
+        realtime_observer=WebRealtimeAggregator(),
+    )

app/backend/data/policy.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from app_config import APP_DIR
+from backend.data.contracts import DataUsage, DatasetContract, ProviderContract
+
+
+class DataPolicyError(RuntimeError):
+    pass
+
+
+class DataSourcePolicy:
+    def __init__(
+        self,
+        providers: dict[str, ProviderContract],
+        datasets: dict[str, DatasetContract],
+    ) -> None:
+        self.providers = dict(providers)
+        self.datasets = dict(datasets)
+
+    @classmethod
+    def load(cls, path: Path | None = None) -> "DataSourcePolicy":
+        config_path = path or APP_DIR / "config" / "data-fields.config.json"
+        payload = json.loads(config_path.read_text(encoding="utf-8"))
+        providers = {
+            provider_id: ProviderContract(
+                id=provider_id,
+                provider_class=str(item["class"]),
+                calculation_allowed=bool(item["calculation_allowed"]),
+            )
+            for provider_id, item in payload["providers"].items()
+        }
+        datasets = {
+            item["id"]: DatasetContract(
+                id=str(item["id"]),
+                entity=str(item["entity"]),
+                frequency=str(item["frequency"]),
+                primary=str(item["primary"]),
+                fallbacks=tuple(str(value) for value in item.get("fallbacks", [])),
+                usage=str(item["usage"]),
+                fields=tuple(str(value) for value in item.get("fields", [])),
+            )
+            for item in payload["datasets"]
+        }
+        return cls(providers, datasets)
+
+    def dataset(self, dataset_id: str) -> DatasetContract:
+        try:
+            return self.datasets[dataset_id]
+        except KeyError as exc:
+            raise DataPolicyError(f"Unregistered dataset: {dataset_id}") from exc
+
+    def assert_allowed(
+        self,
+        dataset_id: str,
+        provider_id: str,
+        usage: DataUsage,
+    ) -> DatasetContract:
+        dataset = self.dataset(dataset_id)
+        if dataset.usage == "blocked":
+            raise DataPolicyError(f"Dataset is blocked: {dataset_id}")
+        if provider_id not in dataset.providers:
+            raise DataPolicyError(
+                f"Provider {provider_id} is not registered for dataset {dataset_id}"
+            )
+        try:
+            provider = self.providers[provider_id]
+        except KeyError as exc:
+            raise DataPolicyError(f"Unregistered provider: {provider_id}") from exc
+        if usage == "calculation":
+            if dataset.usage != "calculation" or not provider.calculation_allowed:
+                raise DataPolicyError(
+                    f"Provider {provider_id} cannot calculate dataset {dataset_id}"
+                )
+        return dataset

app/backend/data/providers/__init__.py

@@ -0,0 +1,4 @@
+from .ifind import IfindProvider
+from .tushare import TushareProvider
+
+__all__ = ["IfindProvider", "TushareProvider"]

app/backend/data/providers/ifind.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from ifind_client import IfindHttpClient
+
+
+class IfindProvider:
+    def __init__(self, client: IfindHttpClient) -> None:
+        self.client = client
+
+    def set_credentials(self, refresh_token: str, access_token: str = "") -> None:
+        self.client.set_credentials(refresh_token, access_token)

app/backend/data/providers/tushare.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from tushare_client import TushareClient
+
+
+class TushareProvider:
+    def __init__(
+        self,
+        token_supplier: Callable[[], str],
+        client_factory: Callable[[str], TushareClient] = TushareClient,
+    ) -> None:
+        self._token_supplier = token_supplier
+        self._client_factory = client_factory
+
+    def client(self) -> TushareClient:
+        return self._client_factory(str(self._token_supplier() or "").strip())

app/backend/data/quality.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import date, datetime, time, timedelta, timezone
+from pathlib import Path
+from typing import Any
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+from app_config import APP_DIR
+from backend.data.contracts import DataUsage
+from backend.data.policy import DataPolicyError, DataSourcePolicy
+
+
+class DataQualityError(RuntimeError):
+    pass
+
+
+def market_timezone(name: str = "Asia/Shanghai"):
+    try:
+        return ZoneInfo(name)
+    except ZoneInfoNotFoundError:
+        if name != "Asia/Shanghai":
+            raise
+        return timezone(timedelta(hours=8), name)
+
+
+@dataclass(frozen=True)
+class QualityEvidence:
+    dataset_id: str
+    provider_id: str
+    data_time: str | datetime
+    observed_at: str | datetime
+    actual_count: int | None = None
+    expected_count: int | None = None
+    units: dict[str, str] | None = None
+    adjustment: str = ""
+    available_at: str | datetime | None = None
+
+
+@dataclass(frozen=True)
+class QualityReport:
+    accepted: bool
+    dataset_id: str
+    provider_id: str
+    usage: DataUsage
+    coverage_ratio: float | None
+    age_seconds: float
+    issues: tuple[str, ...]
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "accepted": self.accepted,
+            "dataset_id": self.dataset_id,
+            "provider_id": self.provider_id,
+            "usage": self.usage,
+            "coverage_ratio": self.coverage_ratio,
+            "age_seconds": round(self.age_seconds, 3),
+            "issues": list(self.issues),
+        }
+
+
+class DataQualityGate:
+    def __init__(
+        self,
+        source_policy: DataSourcePolicy,
+        payload: dict[str, Any],
+    ) -> None:
+        self.source_policy = source_policy
+        self.timezone = market_timezone(
+            str(payload.get("timezone") or "Asia/Shanghai")
+        )
+        self.defaults = dict(payload.get("defaults") or {})
+        self.unit_profiles = dict(payload.get("unit_profiles") or {})
+        self.rules = dict(payload.get("datasets") or {})
+
+    @classmethod
+    def load(
+        cls,
+        source_policy: DataSourcePolicy,
+        path: Path | None = None,
+    ) -> "DataQualityGate":
+        config_path = path or APP_DIR / "config" / "data-quality.config.json"
+        payload = json.loads(config_path.read_text(encoding="utf-8"))
+        return cls(source_policy, payload)
+
+    def evaluate(
+        self,
+        evidence: QualityEvidence,
+        usage: DataUsage,
+        as_of: str | datetime | None = None,
+    ) -> QualityReport:
+        issues: list[str] = []
+        try:
+            self.source_policy.assert_allowed(
+                evidence.dataset_id, evidence.provider_id, usage
+            )
+        except DataPolicyError as exc:
+            issues.append(str(exc))
+
+        rule = self.rules.get(evidence.dataset_id)
+        if rule is None:
+            issues.append(f"Missing quality rule: {evidence.dataset_id}")
+            rule = {}
+        if rule.get("blocked"):
+            issues.append(f"Dataset quality is blocked: {evidence.dataset_id}")
+
+        reference = self._datetime(as_of or datetime.now(self.timezone))
+        data_time = self._datetime(evidence.data_time)
+        observed_at = self._datetime(evidence.observed_at)
+        tolerance = float(
+            (self.defaults.get(usage) or {}).get("future_tolerance_seconds") or 0
+        )
+        if data_time > reference + timedelta(seconds=tolerance):
+            issues.append("Data time is later than the evaluation time")
+        if observed_at > reference + timedelta(seconds=tolerance):
+            issues.append("Observation time is later than the evaluation time")
+        if observed_at < data_time:
+            issues.append("Observation time precedes data time")
+
+        age_seconds = max(0.0, (reference - data_time).total_seconds())
+        freshness = rule.get("freshness_seconds")
+        if freshness is not None and age_seconds > float(freshness):
+            issues.append(
+                f"Data is stale: {age_seconds:.1f}s exceeds {float(freshness):.1f}s"
+            )
+
+        coverage_ratio: float | None = None
+        if evidence.expected_count is not None:
+            if evidence.expected_count <= 0:
+                issues.append("Expected count must be positive")
+            elif evidence.actual_count is None or evidence.actual_count < 0:
+                issues.append("Actual count is missing or invalid")
+            else:
+                coverage_ratio = min(1.0, evidence.actual_count / evidence.expected_count)
+                minimum = float(rule.get("min_coverage_ratio") or 0)
+                if coverage_ratio < minimum:
+                    issues.append(
+                        f"Coverage {coverage_ratio:.3f} is below {minimum:.3f}"
+                    )
+
+        required_adjustment = str(rule.get("adjustment") or "")
+        if required_adjustment and evidence.adjustment != required_adjustment:
+            issues.append(
+                f"Adjustment {evidence.adjustment or 'missing'} does not match {required_adjustment}"
+            )
+
+        profile_id = str(rule.get("unit_profile") or "none")
+        required_units = dict(self.unit_profiles.get(profile_id) or {})
+        supplied_units = evidence.units or {}
+        for field, expected_unit in required_units.items():
+            actual_unit = supplied_units.get(field)
+            if actual_unit != expected_unit:
+                issues.append(
+                    f"Unit for {field} is {actual_unit or 'missing'}, expected {expected_unit}"
+                )
+
+        if rule.get("point_in_time") == "announcement_date" and usage == "calculation":
+            if evidence.available_at is None:
+                issues.append("Point-in-time availability is missing")
+            elif self._datetime(evidence.available_at) > reference:
+                issues.append("Point-in-time data was not available at evaluation time")
+
+        return QualityReport(
+            accepted=not issues,
+            dataset_id=evidence.dataset_id,
+            provider_id=evidence.provider_id,
+            usage=usage,
+            coverage_ratio=coverage_ratio,
+            age_seconds=age_seconds,
+            issues=tuple(issues),
+        )
+
+    def require(
+        self,
+        evidence: QualityEvidence,
+        usage: DataUsage,
+        as_of: str | datetime | None = None,
+    ) -> QualityReport:
+        report = self.evaluate(evidence, usage, as_of)
+        if not report.accepted:
+            raise DataQualityError("; ".join(report.issues))
+        return report
+
+    def _datetime(self, value: str | datetime) -> datetime:
+        if isinstance(value, datetime):
+            parsed = value
+        else:
+            text = str(value or "").strip()
+            if not text:
+                raise DataQualityError("Quality evidence timestamp is missing")
+            try:
+                parsed = datetime.fromisoformat(text)
+            except ValueError:
+                try:
+                    day = date.fromisoformat(text)
+                except ValueError as exc:
+                    raise DataQualityError(f"Invalid quality timestamp: {text}") from exc
+                parsed = datetime.combine(day, time.min)
+        if parsed.tzinfo is None:
+            return parsed.replace(tzinfo=self.timezone)
+        return parsed.astimezone(self.timezone)