migration: establish exact preserved app baseline

This commit is contained in:
leefer
2026-07-30 23:51:48 +08:00
commit e4a9b2e647
389 changed files with 126625 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
"""Application packages introduced by architecture governance."""
+9
View File
@@ -0,0 +1,9 @@
from .container import ApplicationContainer, build_application_container
from .settings import RuntimeSettings, load_runtime_settings
__all__ = [
"ApplicationContainer",
"RuntimeSettings",
"build_application_container",
"load_runtime_settings",
]
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from collections.abc import Callable
from backend.data import DataGateway, build_data_gateway
from backend.database.repositories import RepositoryBundle, build_repository_bundle
from backend.features.alerts import AlertService
from backend.features.review import TradeJournalService
from backend.features.screener import StrategyTrackingService
from backend.jobs import InProcessJobRunner, JobRegistry, SQLiteJobRunRepository
from chart_data_provider import MarketChartClient
from database import ReviewDatabase
from ifind_client import IfindHttpClient
from mentor_agent import MentorSkillRegistry
from realtime_aggregator import WebRealtimeAggregator
from screener import ScreenerEngine
@dataclass(frozen=True)
class ApplicationContainer:
database: ReviewDatabase
repositories: RepositoryBundle
data_gateway: DataGateway
ifind: IfindHttpClient
screener: ScreenerEngine
strategy_tracking: StrategyTrackingService
alert_service: AlertService
trade_journal: TradeJournalService
mentor_skills: MentorSkillRegistry
realtime_aggregator: WebRealtimeAggregator
chart_data: MarketChartClient
jobs: InProcessJobRunner
def build_application_container(
database: ReviewDatabase,
credentials: dict[str, object],
mentor_skills_dir: Path,
private_mentor_skills_dir: Path,
tushare_token_supplier: Callable[[], str] | None = None,
) -> ApplicationContainer:
data_gateway = build_data_gateway(credentials, tushare_token_supplier)
repositories = build_repository_bundle(database)
jobs = InProcessJobRunner(JobRegistry.load(), SQLiteJobRunRepository(database))
return ApplicationContainer(
database=database,
repositories=repositories,
data_gateway=data_gateway,
ifind=data_gateway.ifind,
screener=ScreenerEngine(database),
strategy_tracking=StrategyTrackingService(repositories.strategy_tracking),
alert_service=AlertService(repositories.alerts),
trade_journal=TradeJournalService(repositories.trades),
mentor_skills=MentorSkillRegistry(mentor_skills_dir, private_mentor_skills_dir),
realtime_aggregator=data_gateway.realtime_observer,
chart_data=data_gateway.chart_data,
jobs=jobs,
)
+55
View File
@@ -0,0 +1,55 @@
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Mapping
from app_config import load_local_env, save_local_env
from security import SecretVault
def environment_credentials(environment: Mapping[str, str]) -> dict[str, str]:
return {
"tushare_token": str(environment.get("TUSHARE_TOKEN") or "").strip(),
"ifind_refresh_token": str(environment.get("IFIND_REFRESH_TOKEN") or "").strip(),
"ifind_access_token": str(environment.get("IFIND_ACCESS_TOKEN") or "").strip(),
"platform_llm_primary_api_key": str(
environment.get("LLM_PRIMARY_API_KEY") or environment.get("LLM_API_KEY") or ""
).strip(),
"platform_llm_primary_base_url": str(
environment.get("LLM_PRIMARY_BASE_URL")
or environment.get("LLM_BASE_URL")
or "https://api.openai.com/v1"
).strip(),
"platform_llm_primary_model": str(
environment.get("LLM_PRIMARY_MODEL") or environment.get("LLM_MODEL") or ""
).strip(),
"platform_llm_fallback_api_key": str(
environment.get("LLM_FALLBACK_API_KEY") or ""
).strip(),
"platform_llm_fallback_base_url": str(
environment.get("LLM_FALLBACK_BASE_URL") or ""
).strip(),
"platform_llm_fallback_model": str(
environment.get("LLM_FALLBACK_MODEL") or ""
).strip(),
}
@dataclass(frozen=True)
class RuntimeSettings:
encryption_key: str
initial_credentials: dict[str, str]
def load_runtime_settings() -> RuntimeSettings:
load_local_env()
encryption_key = os.environ.get("APP_ENCRYPTION_KEY", "").strip()
if not encryption_key:
encryption_key = SecretVault.generate_key()
save_local_env({"APP_ENCRYPTION_KEY": encryption_key})
os.environ["APP_ENCRYPTION_KEY"] = encryption_key
return RuntimeSettings(
encryption_key=encryption_key,
initial_credentials=environment_credentials(os.environ),
)
+14
View File
@@ -0,0 +1,14 @@
from .gateway import DataGateway, build_data_gateway
from .policy import DataPolicyError, DataSourcePolicy
from .quality import DataQualityError, DataQualityGate, QualityEvidence, QualityReport
__all__ = [
"DataGateway",
"DataPolicyError",
"DataQualityError",
"DataQualityGate",
"DataSourcePolicy",
"QualityEvidence",
"QualityReport",
"build_data_gateway",
]
+29
View File
@@ -0,0 +1,29 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Literal
DataUsage = Literal["display", "calculation"]
@dataclass(frozen=True)
class ProviderContract:
id: str
provider_class: str
calculation_allowed: bool
@dataclass(frozen=True)
class DatasetContract:
id: str
entity: str
frequency: str
primary: str
fallbacks: tuple[str, ...]
usage: str
fields: tuple[str, ...]
@property
def providers(self) -> tuple[str, ...]:
return (self.primary, *self.fallbacks)
+83
View File
@@ -0,0 +1,83 @@
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime
from backend.data.contracts import DataUsage
from backend.data.policy import DataSourcePolicy
from backend.data.providers import IfindProvider, TushareProvider
from backend.data.quality import DataQualityGate, QualityEvidence, QualityReport
from chart_data_provider import EastmoneyChartClient, MarketChartClient
from ifind_client import IfindHttpClient
from realtime_aggregator import WebRealtimeAggregator
from tushare_client import TushareClient
@dataclass(frozen=True)
class DataGateway:
policy: DataSourcePolicy
quality: DataQualityGate
tushare_provider: TushareProvider
ifind_provider: IfindProvider
chart_data: MarketChartClient
realtime_observer: WebRealtimeAggregator
@property
def ifind(self) -> IfindHttpClient:
return self.ifind_provider.client
def tushare(
self,
dataset_id: str = "",
usage: DataUsage = "calculation",
) -> TushareClient:
if dataset_id:
self.policy.assert_allowed(dataset_id, "tushare", usage)
return self.tushare_provider.client()
def assert_source(self, dataset_id: str, provider_id: str, usage: DataUsage) -> None:
self.policy.assert_allowed(dataset_id, provider_id, usage)
def provider_chain(self, dataset_id: str, usage: DataUsage) -> tuple[str, ...]:
dataset = self.policy.dataset(dataset_id)
allowed = []
for provider_id in dataset.providers:
try:
self.policy.assert_allowed(dataset_id, provider_id, usage)
except Exception:
continue
allowed.append(provider_id)
if not allowed:
raise RuntimeError(f"No permitted provider for {dataset_id} ({usage})")
return tuple(allowed)
def require_quality(
self,
evidence: QualityEvidence,
usage: DataUsage,
as_of: str | datetime | None = None,
) -> QualityReport:
return self.quality.require(evidence, usage, as_of)
def build_data_gateway(
credentials: dict[str, object],
tushare_token_supplier: Callable[[], str] | None = None,
) -> DataGateway:
ifind = IfindHttpClient(
str(credentials.get("ifind_refresh_token") or ""),
str(credentials.get("ifind_access_token") or ""),
)
token_supplier = tushare_token_supplier or (
lambda: str(credentials.get("tushare_token") or "")
)
policy = DataSourcePolicy.load()
return DataGateway(
policy=policy,
quality=DataQualityGate.load(policy),
tushare_provider=TushareProvider(token_supplier),
ifind_provider=IfindProvider(ifind),
chart_data=MarketChartClient(ifind, EastmoneyChartClient()),
realtime_observer=WebRealtimeAggregator(),
)
+77
View File
@@ -0,0 +1,77 @@
from __future__ import annotations
import json
from pathlib import Path
from app_config import APP_DIR
from backend.data.contracts import DataUsage, DatasetContract, ProviderContract
class DataPolicyError(RuntimeError):
pass
class DataSourcePolicy:
def __init__(
self,
providers: dict[str, ProviderContract],
datasets: dict[str, DatasetContract],
) -> None:
self.providers = dict(providers)
self.datasets = dict(datasets)
@classmethod
def load(cls, path: Path | None = None) -> "DataSourcePolicy":
config_path = path or APP_DIR / "config" / "data-fields.config.json"
payload = json.loads(config_path.read_text(encoding="utf-8"))
providers = {
provider_id: ProviderContract(
id=provider_id,
provider_class=str(item["class"]),
calculation_allowed=bool(item["calculation_allowed"]),
)
for provider_id, item in payload["providers"].items()
}
datasets = {
item["id"]: DatasetContract(
id=str(item["id"]),
entity=str(item["entity"]),
frequency=str(item["frequency"]),
primary=str(item["primary"]),
fallbacks=tuple(str(value) for value in item.get("fallbacks", [])),
usage=str(item["usage"]),
fields=tuple(str(value) for value in item.get("fields", [])),
)
for item in payload["datasets"]
}
return cls(providers, datasets)
def dataset(self, dataset_id: str) -> DatasetContract:
try:
return self.datasets[dataset_id]
except KeyError as exc:
raise DataPolicyError(f"Unregistered dataset: {dataset_id}") from exc
def assert_allowed(
self,
dataset_id: str,
provider_id: str,
usage: DataUsage,
) -> DatasetContract:
dataset = self.dataset(dataset_id)
if dataset.usage == "blocked":
raise DataPolicyError(f"Dataset is blocked: {dataset_id}")
if provider_id not in dataset.providers:
raise DataPolicyError(
f"Provider {provider_id} is not registered for dataset {dataset_id}"
)
try:
provider = self.providers[provider_id]
except KeyError as exc:
raise DataPolicyError(f"Unregistered provider: {provider_id}") from exc
if usage == "calculation":
if dataset.usage != "calculation" or not provider.calculation_allowed:
raise DataPolicyError(
f"Provider {provider_id} cannot calculate dataset {dataset_id}"
)
return dataset
+4
View File
@@ -0,0 +1,4 @@
from .ifind import IfindProvider
from .tushare import TushareProvider
__all__ = ["IfindProvider", "TushareProvider"]
+11
View File
@@ -0,0 +1,11 @@
from __future__ import annotations
from ifind_client import IfindHttpClient
class IfindProvider:
def __init__(self, client: IfindHttpClient) -> None:
self.client = client
def set_credentials(self, refresh_token: str, access_token: str = "") -> None:
self.client.set_credentials(refresh_token, access_token)
+18
View File
@@ -0,0 +1,18 @@
from __future__ import annotations
from collections.abc import Callable
from tushare_client import TushareClient
class TushareProvider:
def __init__(
self,
token_supplier: Callable[[], str],
client_factory: Callable[[str], TushareClient] = TushareClient,
) -> None:
self._token_supplier = token_supplier
self._client_factory = client_factory
def client(self) -> TushareClient:
return self._client_factory(str(self._token_supplier() or "").strip())
+202
View File
@@ -0,0 +1,202 @@
from __future__ import annotations
import json
from dataclasses import dataclass
from datetime import date, datetime, time, timedelta, timezone
from pathlib import Path
from typing import Any
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from app_config import APP_DIR
from backend.data.contracts import DataUsage
from backend.data.policy import DataPolicyError, DataSourcePolicy
class DataQualityError(RuntimeError):
pass
def market_timezone(name: str = "Asia/Shanghai"):
try:
return ZoneInfo(name)
except ZoneInfoNotFoundError:
if name != "Asia/Shanghai":
raise
return timezone(timedelta(hours=8), name)
@dataclass(frozen=True)
class QualityEvidence:
dataset_id: str
provider_id: str
data_time: str | datetime
observed_at: str | datetime
actual_count: int | None = None
expected_count: int | None = None
units: dict[str, str] | None = None
adjustment: str = ""
available_at: str | datetime | None = None
@dataclass(frozen=True)
class QualityReport:
accepted: bool
dataset_id: str
provider_id: str
usage: DataUsage
coverage_ratio: float | None
age_seconds: float
issues: tuple[str, ...]
def as_dict(self) -> dict[str, Any]:
return {
"accepted": self.accepted,
"dataset_id": self.dataset_id,
"provider_id": self.provider_id,
"usage": self.usage,
"coverage_ratio": self.coverage_ratio,
"age_seconds": round(self.age_seconds, 3),
"issues": list(self.issues),
}
class DataQualityGate:
def __init__(
self,
source_policy: DataSourcePolicy,
payload: dict[str, Any],
) -> None:
self.source_policy = source_policy
self.timezone = market_timezone(
str(payload.get("timezone") or "Asia/Shanghai")
)
self.defaults = dict(payload.get("defaults") or {})
self.unit_profiles = dict(payload.get("unit_profiles") or {})
self.rules = dict(payload.get("datasets") or {})
@classmethod
def load(
cls,
source_policy: DataSourcePolicy,
path: Path | None = None,
) -> "DataQualityGate":
config_path = path or APP_DIR / "config" / "data-quality.config.json"
payload = json.loads(config_path.read_text(encoding="utf-8"))
return cls(source_policy, payload)
def evaluate(
self,
evidence: QualityEvidence,
usage: DataUsage,
as_of: str | datetime | None = None,
) -> QualityReport:
issues: list[str] = []
try:
self.source_policy.assert_allowed(
evidence.dataset_id, evidence.provider_id, usage
)
except DataPolicyError as exc:
issues.append(str(exc))
rule = self.rules.get(evidence.dataset_id)
if rule is None:
issues.append(f"Missing quality rule: {evidence.dataset_id}")
rule = {}
if rule.get("blocked"):
issues.append(f"Dataset quality is blocked: {evidence.dataset_id}")
reference = self._datetime(as_of or datetime.now(self.timezone))
data_time = self._datetime(evidence.data_time)
observed_at = self._datetime(evidence.observed_at)
tolerance = float(
(self.defaults.get(usage) or {}).get("future_tolerance_seconds") or 0
)
if data_time > reference + timedelta(seconds=tolerance):
issues.append("Data time is later than the evaluation time")
if observed_at > reference + timedelta(seconds=tolerance):
issues.append("Observation time is later than the evaluation time")
if observed_at < data_time:
issues.append("Observation time precedes data time")
age_seconds = max(0.0, (reference - data_time).total_seconds())
freshness = rule.get("freshness_seconds")
if freshness is not None and age_seconds > float(freshness):
issues.append(
f"Data is stale: {age_seconds:.1f}s exceeds {float(freshness):.1f}s"
)
coverage_ratio: float | None = None
if evidence.expected_count is not None:
if evidence.expected_count <= 0:
issues.append("Expected count must be positive")
elif evidence.actual_count is None or evidence.actual_count < 0:
issues.append("Actual count is missing or invalid")
else:
coverage_ratio = min(1.0, evidence.actual_count / evidence.expected_count)
minimum = float(rule.get("min_coverage_ratio") or 0)
if coverage_ratio < minimum:
issues.append(
f"Coverage {coverage_ratio:.3f} is below {minimum:.3f}"
)
required_adjustment = str(rule.get("adjustment") or "")
if required_adjustment and evidence.adjustment != required_adjustment:
issues.append(
f"Adjustment {evidence.adjustment or 'missing'} does not match {required_adjustment}"
)
profile_id = str(rule.get("unit_profile") or "none")
required_units = dict(self.unit_profiles.get(profile_id) or {})
supplied_units = evidence.units or {}
for field, expected_unit in required_units.items():
actual_unit = supplied_units.get(field)
if actual_unit != expected_unit:
issues.append(
f"Unit for {field} is {actual_unit or 'missing'}, expected {expected_unit}"
)
if rule.get("point_in_time") == "announcement_date" and usage == "calculation":
if evidence.available_at is None:
issues.append("Point-in-time availability is missing")
elif self._datetime(evidence.available_at) > reference:
issues.append("Point-in-time data was not available at evaluation time")
return QualityReport(
accepted=not issues,
dataset_id=evidence.dataset_id,
provider_id=evidence.provider_id,
usage=usage,
coverage_ratio=coverage_ratio,
age_seconds=age_seconds,
issues=tuple(issues),
)
def require(
self,
evidence: QualityEvidence,
usage: DataUsage,
as_of: str | datetime | None = None,
) -> QualityReport:
report = self.evaluate(evidence, usage, as_of)
if not report.accepted:
raise DataQualityError("; ".join(report.issues))
return report
def _datetime(self, value: str | datetime) -> datetime:
if isinstance(value, datetime):
parsed = value
else:
text = str(value or "").strip()
if not text:
raise DataQualityError("Quality evidence timestamp is missing")
try:
parsed = datetime.fromisoformat(text)
except ValueError:
try:
day = date.fromisoformat(text)
except ValueError as exc:
raise DataQualityError(f"Invalid quality timestamp: {text}") from exc
parsed = datetime.combine(day, time.min)
if parsed.tzinfo is None:
return parsed.replace(tzinfo=self.timezone)
return parsed.astimezone(self.timezone)
+11
View File
@@ -0,0 +1,11 @@
from .connection import ManagedConnection, SQLiteConnectionFactory
from .migrations import MIGRATIONS, Migration, MigrationError, MigrationRunner
__all__ = [
"MIGRATIONS",
"ManagedConnection",
"Migration",
"MigrationError",
"MigrationRunner",
"SQLiteConnectionFactory",
]
+33
View File
@@ -0,0 +1,33 @@
from __future__ import annotations
import sqlite3
from dataclasses import dataclass
from pathlib import Path
class ManagedConnection(sqlite3.Connection):
"""Commit or roll back, then release the SQLite handle on context exit."""
def __exit__(self, exc_type, exc_value, traceback):
try:
return super().__exit__(exc_type, exc_value, traceback)
finally:
self.close()
@dataclass(frozen=True)
class SQLiteConnectionFactory:
path: Path
timeout_seconds: float = 20
def connect(self) -> sqlite3.Connection:
connection = sqlite3.connect(
self.path,
timeout=self.timeout_seconds,
factory=ManagedConnection,
)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA journal_mode=WAL")
connection.execute("PRAGMA foreign_keys=ON")
connection.execute("PRAGMA busy_timeout=20000")
return connection
+8
View File
@@ -0,0 +1,8 @@
from .m0001_adopt_legacy import MIGRATION as M0001_ADOPT_LEGACY
from .m0002_job_runs import MIGRATION as M0002_JOB_RUNS
from .m0003_llm_audit import MIGRATION as M0003_LLM_AUDIT
from .runner import Migration, MigrationError, MigrationRunner
MIGRATIONS = (M0001_ADOPT_LEGACY, M0002_JOB_RUNS, M0003_LLM_AUDIT)
__all__ = ["MIGRATIONS", "Migration", "MigrationError", "MigrationRunner"]
@@ -0,0 +1,42 @@
from __future__ import annotations
import sqlite3
from backend.database.migrations.runner import Migration, MigrationError
REQUIRED_TABLES = frozenset(
{
"users",
"user_sessions",
"dashboard_snapshots",
"watchlist",
"review_notes",
"stock_master",
"daily_bars",
"screener_runs",
"mentor_messages",
"trade_entries",
"heaven_readings",
}
)
def adopt_legacy_schema(connection: sqlite3.Connection) -> None:
tables = {
str(row["name"])
for row in connection.execute(
"SELECT name FROM sqlite_master WHERE type = 'table'"
)
}
missing = sorted(REQUIRED_TABLES - tables)
if missing:
raise MigrationError(f"Legacy schema is incomplete: {', '.join(missing)}")
MIGRATION = Migration(
version="0001",
name="adopt_legacy_schema",
action=adopt_legacy_schema,
signature="required-tables:v1:" + ",".join(sorted(REQUIRED_TABLES)),
)
@@ -0,0 +1,47 @@
from __future__ import annotations
import sqlite3
from backend.database.migrations.runner import Migration
def create_job_runs(connection: sqlite3.Connection) -> None:
connection.execute(
"""
CREATE TABLE IF NOT EXISTS job_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
job_id TEXT NOT NULL,
idempotency_key TEXT NOT NULL,
status TEXT NOT NULL,
attempt INTEGER NOT NULL DEFAULT 1,
started_at TEXT NOT NULL,
finished_at TEXT,
elapsed_ms INTEGER NOT NULL DEFAULT 0,
error_code TEXT NOT NULL DEFAULT '',
message TEXT NOT NULL DEFAULT '',
output_version TEXT NOT NULL DEFAULT '',
metadata TEXT NOT NULL DEFAULT '{}',
UNIQUE(job_id, idempotency_key, attempt)
)
"""
)
connection.execute(
"""
CREATE INDEX IF NOT EXISTS idx_job_runs_job_started
ON job_runs(job_id, started_at DESC, id DESC)
"""
)
connection.execute(
"""
CREATE INDEX IF NOT EXISTS idx_job_runs_status
ON job_runs(status, started_at DESC, id DESC)
"""
)
MIGRATION = Migration(
version="0002",
name="create_job_runs",
action=create_job_runs,
signature="job-runs:v1:id,job,key,status,attempt,times,elapsed,error,output,metadata",
)
@@ -0,0 +1,32 @@
from __future__ import annotations
import sqlite3
from backend.database.migrations.runner import Migration
def extend_llm_audit(connection: sqlite3.Connection) -> None:
columns = {
str(row["name"])
for row in connection.execute("PRAGMA table_info(llm_usage)")
}
additions = (
("role", "TEXT NOT NULL DEFAULT ''"),
("prompt_version", "TEXT NOT NULL DEFAULT ''"),
("error_code", "TEXT NOT NULL DEFAULT ''"),
("input_tokens", "INTEGER NOT NULL DEFAULT 0"),
("output_tokens", "INTEGER NOT NULL DEFAULT 0"),
)
for name, declaration in additions:
if name not in columns:
connection.execute(
f"ALTER TABLE llm_usage ADD COLUMN {name} {declaration}"
)
MIGRATION = Migration(
version="0003",
name="extend_llm_audit",
action=extend_llm_audit,
signature="llm-audit:v1:role,prompt-version,error-code,input-tokens,output-tokens",
)
+98
View File
@@ -0,0 +1,98 @@
from __future__ import annotations
import hashlib
import sqlite3
from collections.abc import Callable, Iterable
from dataclasses import dataclass
from datetime import datetime, timezone
MigrationAction = Callable[[sqlite3.Connection], None]
class MigrationError(RuntimeError):
pass
@dataclass(frozen=True)
class Migration:
version: str
name: str
action: MigrationAction
signature: str
@property
def checksum(self) -> str:
return hashlib.sha256(self.signature.encode("utf-8")).hexdigest()
class MigrationRunner:
def apply(
self,
connection: sqlite3.Connection,
migrations: Iterable[Migration],
) -> tuple[str, ...]:
ordered = sorted(migrations, key=lambda item: item.version)
versions = [item.version for item in ordered]
if versions != sorted(set(versions)):
raise MigrationError("Migration versions must be unique and ordered")
self._ensure_ledger(connection)
applied = {
str(row["version"]): str(row["checksum"])
for row in connection.execute(
"SELECT version, checksum FROM schema_migrations ORDER BY version"
)
}
known = set(versions)
unknown = sorted(set(applied) - known)
if unknown:
raise MigrationError(f"Database contains unknown migrations: {', '.join(unknown)}")
completed: list[str] = []
for migration in ordered:
existing = applied.get(migration.version)
if existing:
if existing != migration.checksum:
raise MigrationError(
f"Migration checksum changed: {migration.version} {migration.name}"
)
continue
savepoint = f"migration_{migration.version.replace('-', '_')}"
connection.execute(f"SAVEPOINT {savepoint}")
try:
migration.action(connection)
connection.execute(
"""
INSERT INTO schema_migrations
(version, name, checksum, applied_at)
VALUES (?, ?, ?, ?)
""",
(
migration.version,
migration.name,
migration.checksum,
datetime.now(timezone.utc).isoformat(),
),
)
connection.execute(f"RELEASE SAVEPOINT {savepoint}")
except Exception as exc:
connection.execute(f"ROLLBACK TO SAVEPOINT {savepoint}")
connection.execute(f"RELEASE SAVEPOINT {savepoint}")
raise MigrationError(
f"Migration failed: {migration.version} {migration.name}"
) from exc
completed.append(migration.version)
return tuple(completed)
@staticmethod
def _ensure_ledger(connection: sqlite3.Connection) -> None:
connection.execute(
"""
CREATE TABLE IF NOT EXISTS schema_migrations (
version TEXT PRIMARY KEY,
name TEXT NOT NULL,
checksum TEXT NOT NULL,
applied_at TEXT NOT NULL
)
"""
)
+21
View File
@@ -0,0 +1,21 @@
from .ports import AlertRepository, StrategyTrackingRepository, TradeJournalRepository
from .sqlite import (
RepositoryBundle,
SQLiteAlertRepository,
SQLiteStrategyTrackingRepository,
SQLiteTradeJournalRepository,
build_repository_bundle,
require_user_id,
)
__all__ = [
"AlertRepository",
"RepositoryBundle",
"SQLiteAlertRepository",
"SQLiteStrategyTrackingRepository",
"SQLiteTradeJournalRepository",
"StrategyTrackingRepository",
"TradeJournalRepository",
"build_repository_bundle",
"require_user_id",
]
+52
View File
@@ -0,0 +1,52 @@
from __future__ import annotations
from typing import Any, Protocol
class AlertRepository(Protocol):
def save_alert(
self, user_id: int, kind: str, title: str, content: str,
available_date: str, code: str, dedupe_key: str,
) -> int: ...
def list_alerts(
self, user_id: int, as_of: str, unread_only: bool = False, limit: int = 100,
) -> list[dict[str, Any]]: ...
def count_unread_alerts(self, user_id: int, as_of: str) -> int: ...
def mark_alert_read(self, user_id: int, alert_id: int) -> bool: ...
def mark_all_alerts_read(self, user_id: int, as_of: str) -> int: ...
def delete_alert(self, user_id: int, alert_id: int) -> bool: ...
class TradeJournalRepository(Protocol):
def save_trade_entry(self, *args: Any, **kwargs: Any) -> int: ...
def list_trade_entries(
self, user_id: int, start_date: str = "", end_date: str = "",
code: str = "", limit: int = 300,
) -> list[dict[str, Any]]: ...
def delete_trade_entry(self, user_id: int, trade_id: int) -> bool: ...
class StrategyTrackingRepository(Protocol):
def save_strategy_tracks(
self, user_id: int, run_id: int, selection_date: str,
strategy_name: str, candidates: list[dict[str, Any]],
) -> int: ...
def get_screener_run(self, user_id: int, run_id: int) -> dict[str, Any] | None: ...
def delete_strategy_track(self, user_id: int, track_id: int) -> bool: ...
def list_strategy_tracks(
self, user_id: int, limit_batches: int = 12,
) -> list[dict[str, Any]]: ...
def load_tracking_bars(
self, targets: list[tuple[str, str]], limit: int = 5,
) -> dict[tuple[str, str], list[dict[str, Any]]]: ...
+108
View File
@@ -0,0 +1,108 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
from database import ReviewDatabase
def require_user_id(value: int) -> int:
user_id = int(value)
if user_id <= 0:
raise ValueError("A positive account owner is required")
return user_id
@dataclass(frozen=True)
class SQLiteAlertRepository:
database: ReviewDatabase
def save_alert(self, user_id: int, *args: Any, **kwargs: Any) -> int:
return self.database.save_alert(require_user_id(user_id), *args, **kwargs)
def list_alerts(
self, user_id: int, as_of: str, unread_only: bool = False, limit: int = 100,
) -> list[dict[str, Any]]:
return self.database.list_alerts(
require_user_id(user_id), as_of, unread_only, limit
)
def count_unread_alerts(self, user_id: int, as_of: str) -> int:
return self.database.count_unread_alerts(require_user_id(user_id), as_of)
def mark_alert_read(self, user_id: int, alert_id: int) -> bool:
return self.database.mark_alert_read(require_user_id(user_id), alert_id)
def mark_all_alerts_read(self, user_id: int, as_of: str) -> int:
return self.database.mark_all_alerts_read(require_user_id(user_id), as_of)
def delete_alert(self, user_id: int, alert_id: int) -> bool:
return self.database.delete_alert(require_user_id(user_id), alert_id)
@dataclass(frozen=True)
class SQLiteTradeJournalRepository:
database: ReviewDatabase
def save_trade_entry(self, user_id: int, *args: Any, **kwargs: Any) -> int:
return self.database.save_trade_entry(require_user_id(user_id), *args, **kwargs)
def list_trade_entries(
self, user_id: int, start_date: str = "", end_date: str = "",
code: str = "", limit: int = 300,
) -> list[dict[str, Any]]:
return self.database.list_trade_entries(
require_user_id(user_id), start_date, end_date, code, limit
)
def delete_trade_entry(self, user_id: int, trade_id: int) -> bool:
return self.database.delete_trade_entry(require_user_id(user_id), trade_id)
@dataclass(frozen=True)
class SQLiteStrategyTrackingRepository:
database: ReviewDatabase
def save_strategy_tracks(
self, user_id: int, run_id: int, selection_date: str,
strategy_name: str, candidates: list[dict[str, Any]],
) -> int:
return self.database.save_strategy_tracks(
require_user_id(user_id), run_id, selection_date, strategy_name, candidates
)
def get_screener_run(self, user_id: int, run_id: int) -> dict[str, Any] | None:
owner_id = int(user_id)
if owner_id < 0:
raise ValueError("Account owner cannot be negative")
return self.database.get_screener_run(owner_id, run_id)
def delete_strategy_track(self, user_id: int, track_id: int) -> bool:
return self.database.delete_strategy_track(require_user_id(user_id), track_id)
def list_strategy_tracks(
self, user_id: int, limit_batches: int = 12,
) -> list[dict[str, Any]]:
return self.database.list_strategy_tracks(
require_user_id(user_id), limit_batches
)
def load_tracking_bars(
self, targets: list[tuple[str, str]], limit: int = 5,
) -> dict[tuple[str, str], list[dict[str, Any]]]:
return self.database.load_tracking_bars(targets, limit)
@dataclass(frozen=True)
class RepositoryBundle:
alerts: SQLiteAlertRepository
trades: SQLiteTradeJournalRepository
strategy_tracking: SQLiteStrategyTrackingRepository
def build_repository_bundle(database: ReviewDatabase) -> RepositoryBundle:
return RepositoryBundle(
alerts=SQLiteAlertRepository(database),
trades=SQLiteTradeJournalRepository(database),
strategy_tracking=SQLiteStrategyTrackingRepository(database),
)
+1
View File
@@ -0,0 +1 @@
"""Feature-owned application services."""
+3
View File
@@ -0,0 +1,3 @@
from .service import AlertService
__all__ = ["AlertService"]
+95
View File
@@ -0,0 +1,95 @@
from __future__ import annotations
import secrets
from datetime import date, datetime
from typing import Any
from app_config import validate_text
from backend.database.repositories import AlertRepository
class AlertService:
def __init__(self, repository: AlertRepository) -> None:
self.repository = repository
def create_manual(self, user_id: int, payload: dict[str, Any]) -> int:
title = validate_text(payload.get("title"), "提醒标题", 80, required=True)
content = validate_text(payload.get("content"), "提醒内容", 500)
code = validate_text(payload.get("code"), "股票代码", 12)
available_date = self.calendar_date(
str(payload.get("remind_date") or date.today().isoformat())
)
return self.repository.save_alert(
user_id=user_id,
kind="manual",
title=title,
content=content,
available_date=available_date,
code=code,
dedupe_key=f"manual:{secrets.token_hex(12)}",
)
def sync_strategy_tracking(self, user_id: int, tracking: dict[str, Any]) -> int:
synced = 0
today = date.today().strftime("%Y%m%d")
for batch in tracking.get("batches") or []:
items = batch.get("items") or []
summary = batch.get("summary") or {}
if not items:
continue
run_id = int(batch.get("run_id") or 0)
strategy_name = str(batch.get("strategy_name") or "选股策略")
observed = int(summary.get("observed") or 0)
completed = int(summary.get("completed") or 0)
if observed:
win_rate = summary.get("t1_win_rate")
suffix = f",当前红盘率 {win_rate:.1f}%" if win_rate is not None else ""
self.repository.save_alert(
user_id, "strategy_t1", f"{strategy_name} 已有 T+1 反馈",
f"{observed}/{len(items)} 只标的已有首日表现{suffix}",
today, "", f"strategy:{run_id}:t1",
)
synced += 1
if completed == len(items):
average = summary.get("average_t5")
suffix = f",平均收益 {average:+.2f}%" if average is not None else ""
self.repository.save_alert(
user_id, "strategy_t5", f"{strategy_name} 五日跟踪完成",
f"本批 {len(items)} 只标的已完成 T+5 跟踪{suffix}",
today, "", f"strategy:{run_id}:t5",
)
synced += 1
return synced
def list_alerts(
self, user_id: int, status: str = "all", as_of: str = ""
) -> dict[str, Any]:
if status not in {"all", "unread"}:
raise ValueError("提醒筛选不支持。")
compact_date = self.calendar_date(as_of or date.today().isoformat())
items = self.repository.list_alerts(user_id, compact_date, status == "unread")
for item in items:
item["due"] = str(item.get("available_date") or "") <= compact_date
return {
"items": items,
"unread_count": self.repository.count_unread_alerts(user_id, compact_date),
"as_of": compact_date,
}
def mark_read(self, user_id: int, alert_id: int) -> bool:
return self.repository.mark_alert_read(user_id, alert_id)
def mark_all_read(self, user_id: int, as_of: str) -> int:
return self.repository.mark_all_alerts_read(user_id, as_of)
def delete(self, user_id: int, alert_id: int) -> bool:
return self.repository.delete_alert(user_id, alert_id)
@staticmethod
def calendar_date(value: str) -> str:
compact = value.replace("-", "").strip()
try:
parsed = datetime.strptime(compact, "%Y%m%d")
except ValueError as exc:
raise ValueError("提醒日期格式应为 YYYY-MM-DD。") from exc
return parsed.strftime("%Y%m%d")
+3
View File
@@ -0,0 +1,3 @@
from .trade_journal import EMOTIONS, TRADE_ACTIONS, TradeJournalService
__all__ = ["EMOTIONS", "TRADE_ACTIONS", "TradeJournalService"]
+100
View File
@@ -0,0 +1,100 @@
from __future__ import annotations
import json
from datetime import date
from typing import Any
from app_config import normalize_date, validate_stock_code, validate_text
from backend.database.repositories import TradeJournalRepository
TRADE_ACTIONS = {"buy": "买入", "sell": "卖出", "trim": "减仓", "add": "加仓", "watch": "观察"}
EMOTIONS = {"calm": "平静", "confident": "笃定", "hesitant": "犹豫", "anxious": "焦虑", "impulsive": "冲动"}
class TradeJournalService:
def __init__(self, repository: TradeJournalRepository) -> None:
self.repository = repository
def save(self, user_id: int, payload: dict[str, Any]) -> int:
trade_id = int(payload.get("id") or 0)
trade_date = normalize_date(str(payload.get("trade_date") or date.today().isoformat()))
code = validate_stock_code(str(payload.get("code") or ""))
name = validate_text(payload.get("name"), "股票名称", 40, required=True)
action = str(payload.get("action") or "")
if action not in TRADE_ACTIONS:
raise ValueError("交易动作不支持。")
emotion = str(payload.get("emotion") or "calm")
if emotion not in EMOTIONS:
raise ValueError("交易情绪不支持。")
price = self._number(payload.get("price"), "成交价格", 0, 1000000, required=True)
quantity = int(self._number(payload.get("quantity"), "成交数量", 0, 100000000))
position_pct = self._number(payload.get("position_pct"), "仓位", 0, 100)
pnl_amount = self._optional_number(payload.get("pnl_amount"), "盈亏金额", -1e12, 1e12)
pnl_pct = self._optional_number(payload.get("pnl_pct"), "盈亏比例", -1000, 10000)
thesis = validate_text(payload.get("thesis"), "交易逻辑", 2000)
execution = validate_text(payload.get("execution"), "执行复核", 2000)
raw_tags = payload.get("tags") or []
if isinstance(raw_tags, str):
raw_tags = [item.strip() for item in raw_tags.replace("", ",").split(",")]
if not isinstance(raw_tags, list):
raise ValueError("交易标签格式不正确。")
tags = [validate_text(item, "交易标签", 20) for item in raw_tags if str(item).strip()][:8]
return self.repository.save_trade_entry(
user_id, trade_date, code, name, action, price, quantity, position_pct,
pnl_amount, pnl_pct, thesis, execution, emotion, tags, trade_id or None,
)
def list_entries(
self, user_id: int, start_date: str = "", end_date: str = "", code: str = ""
) -> dict[str, Any]:
start = normalize_date(start_date) if start_date else ""
end = normalize_date(end_date) if end_date else date.today().strftime("%Y%m%d")
if start and start > end:
raise ValueError("开始日期不能晚于结束日期。")
code = validate_stock_code(code) if code else ""
items = self.repository.list_trade_entries(user_id, start, end, code)
for item in items:
item["tags"] = json.loads(item.get("tags") or "[]")
item["action_label"] = TRADE_ACTIONS.get(item["action"], item["action"])
item["emotion_label"] = EMOTIONS.get(item["emotion"], item["emotion"])
realized = [item for item in items if item.get("pnl_pct") is not None]
return {"items": items, "summary": self._summary(items, realized)}
def delete(self, user_id: int, trade_id: int) -> bool:
return self.repository.delete_trade_entry(user_id, trade_id)
@staticmethod
def _summary(items: list[dict[str, Any]], realized: list[dict[str, Any]]) -> dict[str, Any]:
pnl_amounts = [float(item["pnl_amount"]) for item in realized if item.get("pnl_amount") is not None]
positions = [float(item["position_pct"]) for item in items if float(item.get("position_pct") or 0) > 0]
wins = sum(float(item.get("pnl_pct") or 0) > 0 for item in realized)
return {
"total": len(items),
"realized": len(realized),
"win_rate": round(wins / len(realized) * 100, 1) if realized else None,
"pnl_amount": round(sum(pnl_amounts), 2) if pnl_amounts else None,
"average_position": round(sum(positions) / len(positions), 1) if positions else None,
}
@staticmethod
def _number(value: Any, label: str, minimum: float, maximum: float, required: bool = False) -> float:
if value in (None, ""):
if required:
raise ValueError(f"{label}不能为空。")
return 0.0
try:
parsed = float(value)
except (TypeError, ValueError) as exc:
raise ValueError(f"{label}格式不正确。") from exc
if parsed < minimum or parsed > maximum:
raise ValueError(f"{label}超出允许范围。")
return parsed
@classmethod
def _optional_number(
cls, value: Any, label: str, minimum: float, maximum: float
) -> float | None:
if value in (None, ""):
return None
return cls._number(value, label, minimum, maximum, required=True)
+3
View File
@@ -0,0 +1,3 @@
from .tracking import StrategyTrackingService
__all__ = ["StrategyTrackingService"]
+134
View File
@@ -0,0 +1,134 @@
from __future__ import annotations
from typing import Any
from backend.database.repositories import StrategyTrackingRepository
class StrategyTrackingService:
def __init__(self, repository: StrategyTrackingRepository) -> None:
self.repository = repository
def record_run(
self,
user_id: int,
run_id: int,
selection_date: str,
strategy_name: str,
candidates: list[dict[str, Any]],
) -> int:
return self.repository.save_strategy_tracks(
user_id, run_id, selection_date, strategy_name, candidates
)
def add_candidate(self, user_id: int, run_id: int, code: str) -> dict[str, Any]:
run = self.repository.get_screener_run(user_id, run_id)
if not run:
run = self.repository.get_screener_run(0, run_id)
if not run:
raise ValueError("选股结果不存在或不属于当前账号。")
normalized_code = str(code or "").strip().split(".")[0]
candidate = next(
(
item for item in run.get("candidates", [])
if str(item.get("code") or item.get("ts_code") or "").split(".")[0]
== normalized_code
),
None,
)
if not candidate:
raise ValueError("该股票不在本次选股结果中。")
added = self.record_run(
user_id,
run_id,
str(run.get("meta", {}).get("trade_date") or ""),
str(run.get("strategy_name") or "未命名策略"),
[candidate],
)
return {"added": added, "tracking": self.list_tracking(user_id)}
def remove_candidate(self, user_id: int, track_id: int) -> dict[str, Any]:
deleted = self.repository.delete_strategy_track(user_id, track_id)
return {"deleted": deleted, "tracking": self.list_tracking(user_id)}
def list_tracking(self, user_id: int, limit_batches: int = 12) -> dict[str, Any]:
tracks = self.repository.list_strategy_tracks(user_id, limit_batches)
if not tracks:
return {"batches": [], "summary": self._summary([])}
bars = self.repository.load_tracking_bars(
[(item["ts_code"], item["selection_date"]) for item in tracks], 5
)
batches: dict[int, dict[str, Any]] = {}
all_items: list[dict[str, Any]] = []
for track in tracks:
key = (track["ts_code"], track["selection_date"])
metrics = self.calculate_metrics(float(track["entry_price"]), bars.get(key, []))
item = {
"id": track["id"],
"code": track["code"],
"name": track["name"],
"sector": track["sector"],
"entry_price": round(float(track["entry_price"]), 2),
**metrics,
}
all_items.append(item)
batch = batches.setdefault(
int(track["run_id"]),
{
"run_id": int(track["run_id"]),
"selection_date": track["selection_date"],
"strategy_name": track["strategy_name"],
"items": [],
},
)
batch["items"].append(item)
ordered = list(batches.values())
for batch in ordered:
batch["summary"] = self._summary(batch["items"])
return {"batches": ordered, "summary": self._summary(all_items)}
@staticmethod
def calculate_metrics(entry_price: float, bars: list[dict[str, Any]]) -> dict[str, Any]:
valid = [row for row in bars[:5] if float(row.get("close") or 0) > 0]
if entry_price <= 0 or not valid:
return {
"observed_days": 0,
"status": "等待 T+1",
"t1_open": None,
"t1_close": None,
"t3_close": None,
"t5_close": None,
"max_gain": None,
"max_drawdown": None,
}
def change(price: Any) -> float:
return round((float(price or 0) / entry_price - 1) * 100, 2)
observed = len(valid)
return {
"observed_days": observed,
"status": "已完成" if observed >= 5 else f"跟踪中 {observed}/5",
"t1_open": change(valid[0]["open"]),
"t1_close": change(valid[0]["close"]),
"t3_close": change(valid[2]["close"]) if observed >= 3 else None,
"t5_close": change(valid[4]["close"]) if observed >= 5 else None,
"max_gain": max(change(row["high"]) for row in valid),
"max_drawdown": min(change(row["low"]) for row in valid),
}
@staticmethod
def _summary(items: list[dict[str, Any]]) -> dict[str, Any]:
completed = [item for item in items if item.get("t5_close") is not None]
t1 = [float(item["t1_close"]) for item in items if item.get("t1_close") is not None]
t5 = [float(item["t5_close"]) for item in completed]
return {
"total": len(items),
"observed": len(t1),
"completed": len(completed),
"t1_win_rate": round(sum(value > 0 for value in t1) / len(t1) * 100, 1) if t1 else None,
"t5_win_rate": round(sum(value > 0 for value in t5) / len(t5) * 100, 1) if t5 else None,
"average_t5": round(sum(t5) / len(t5), 2) if t5 else None,
}
+8
View File
@@ -0,0 +1,8 @@
from .context import correlation_id
from .errors import normalize_error_payload
from .router import AccessRole, ApiRoute, ApiRouteRegistry, RouteRegistryError
__all__ = [
"AccessRole", "ApiRoute", "ApiRouteRegistry", "RouteRegistryError",
"correlation_id", "normalize_error_payload",
]
+12
View File
@@ -0,0 +1,12 @@
from __future__ import annotations
import re
import uuid
REQUEST_ID_PATTERN = re.compile(r"[A-Za-z0-9._-]{8,80}")
def correlation_id(supplied: str = "") -> str:
value = str(supplied or "").strip()
return value if REQUEST_ID_PATTERN.fullmatch(value) else uuid.uuid4().hex
+31
View File
@@ -0,0 +1,31 @@
from __future__ import annotations
from http import HTTPStatus
from typing import Any
STATUS_CODES = {
HTTPStatus.BAD_REQUEST: "bad_request",
HTTPStatus.UNAUTHORIZED: "authentication_required",
HTTPStatus.FORBIDDEN: "access_denied",
HTTPStatus.NOT_FOUND: "not_found",
HTTPStatus.CONFLICT: "conflict",
HTTPStatus.INTERNAL_SERVER_ERROR: "internal_error",
HTTPStatus.SERVICE_UNAVAILABLE: "service_unavailable",
}
def normalize_error_payload(
payload: dict[str, Any], status: int | HTTPStatus, request_id: str,
) -> dict[str, Any]:
if "error" not in payload:
return payload
status_value = HTTPStatus(int(status))
message = str(payload.get("message") or payload.get("error") or status_value.phrase)
return {
**payload,
"error": message,
"code": str(payload.get("code") or STATUS_CODES.get(status_value) or "request_failed"),
"message": message,
"request_id": request_id,
}
+78
View File
@@ -0,0 +1,78 @@
from __future__ import annotations
import json
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Literal, cast
from app_config import APP_DIR
AccessRole = Literal["public", "authenticated", "member", "admin"]
MatchType = Literal["exact", "regex"]
class RouteRegistryError(RuntimeError):
pass
@dataclass(frozen=True)
class ApiRoute:
method: str
path: str
match: MatchType
feature: str
access: AccessRole
def matches(self, method: str, path: str) -> bool:
if self.method != method.upper():
return False
return self.path == path if self.match == "exact" else re.fullmatch(self.path, path) is not None
class ApiRouteRegistry:
def __init__(self, routes: tuple[ApiRoute, ...]) -> None:
self.routes = routes
self._exact: dict[tuple[str, str], ApiRoute] = {}
regex_routes: list[ApiRoute] = []
seen: set[tuple[str, str]] = set()
for route in routes:
key = (route.method, route.path)
if key in seen:
raise RouteRegistryError(f"Duplicate API route: {route.method} {route.path}")
seen.add(key)
if route.match == "exact":
self._exact[key] = route
else:
try:
re.compile(route.path)
except re.error as exc:
raise RouteRegistryError(f"Invalid API route regex: {route.path}") from exc
regex_routes.append(route)
self._regex = tuple(regex_routes)
@classmethod
def load(cls, path: Path | None = None) -> "ApiRouteRegistry":
config_path = path or APP_DIR / "config" / "api.config.json"
payload = json.loads(config_path.read_text(encoding="utf-8"))
routes = tuple(
ApiRoute(
method=str(item["method"]).upper(),
path=str(item["path"]),
match=cast(MatchType, str(item["match"])),
feature=str(item["feature"]),
access=cast(AccessRole, str(item["access"])),
)
for item in payload.get("routes") or []
)
if not routes:
raise RouteRegistryError("API route registry is empty")
return cls(routes)
def resolve(self, method: str, path: str) -> ApiRoute | None:
normalized = method.upper()
exact = self._exact.get((normalized, path))
if exact:
return exact
return next((route for route in self._regex if route.matches(normalized, path)), None)
+5
View File
@@ -0,0 +1,5 @@
from .registry import JobDefinition, JobRegistry
from .repository import SQLiteJobRunRepository
from .runner import InProcessJobRunner
__all__ = ["InProcessJobRunner", "JobDefinition", "JobRegistry", "SQLiteJobRunRepository"]
+54
View File
@@ -0,0 +1,54 @@
from __future__ import annotations
import json
from dataclasses import dataclass
from pathlib import Path
from app_config import APP_DIR
@dataclass(frozen=True)
class JobDefinition:
job_id: str
schedule: str
input_date_policy: str
dependencies: tuple[str, ...]
lock_key: str
timeout_seconds: int
max_attempts: int
output_version: str
class JobRegistry:
def __init__(self, definitions: tuple[JobDefinition, ...]) -> None:
self.definitions = definitions
self._by_id = {item.job_id: item for item in definitions}
if len(self._by_id) != len(definitions):
raise ValueError("Background job IDs must be unique")
@classmethod
def load(cls, path: Path | None = None) -> "JobRegistry":
config_path = path or APP_DIR / "config" / "jobs.config.json"
payload = json.loads(config_path.read_text(encoding="utf-8"))
definitions = tuple(
JobDefinition(
job_id=str(item["id"]),
schedule=str(item["schedule"]),
input_date_policy=str(item["input_date_policy"]),
dependencies=tuple(str(value) for value in item.get("dependencies") or []),
lock_key=str(item["lock_key"]),
timeout_seconds=max(1, int(item["timeout_seconds"])),
max_attempts=max(1, int(item["max_attempts"])),
output_version=str(item["output_version"]),
)
for item in payload.get("jobs") or []
)
if not definitions:
raise ValueError("Background job registry is empty")
return cls(definitions)
def get(self, job_id: str) -> JobDefinition:
try:
return self._by_id[job_id]
except KeyError as exc:
raise ValueError(f"Background job is not registered: {job_id}") from exc
+88
View File
@@ -0,0 +1,88 @@
from __future__ import annotations
import json
from dataclasses import dataclass
from datetime import datetime
from typing import Any
from database import ReviewDatabase
@dataclass(frozen=True)
class SQLiteJobRunRepository:
database: ReviewDatabase
def completed(self, job_id: str, idempotency_key: str) -> bool:
with self.database.connect() as connection:
row = connection.execute(
"""
SELECT 1 FROM job_runs
WHERE job_id = ? AND idempotency_key = ? AND status = 'success'
LIMIT 1
""",
(job_id, idempotency_key),
).fetchone()
return row is not None
def start(
self, job_id: str, idempotency_key: str, output_version: str,
metadata: dict[str, Any] | None = None,
) -> int:
now = datetime.now().astimezone().isoformat(timespec="seconds")
with self.database.connect() as connection:
row = connection.execute(
"""
SELECT COALESCE(MAX(attempt), 0) + 1 AS attempt FROM job_runs
WHERE job_id = ? AND idempotency_key = ?
""",
(job_id, idempotency_key),
).fetchone()
cursor = connection.execute(
"""
INSERT INTO job_runs
(job_id, idempotency_key, status, attempt, started_at,
output_version, metadata)
VALUES (?, ?, 'running', ?, ?, ?, ?)
""",
(
job_id, idempotency_key, int(row["attempt"]), now,
output_version,
json.dumps(metadata or {}, ensure_ascii=False, separators=(",", ":")),
),
)
connection.execute(
"""
DELETE FROM job_runs
WHERE id < (SELECT COALESCE(MAX(id), 0) - 20000 FROM job_runs)
AND status != 'running'
"""
)
return int(cursor.lastrowid)
def finish(
self, run_id: int, status: str, elapsed_ms: int,
error_code: str = "", message: str = "",
) -> None:
now = datetime.now().astimezone().isoformat(timespec="seconds")
with self.database.connect() as connection:
connection.execute(
"""
UPDATE job_runs
SET status = ?, finished_at = ?, elapsed_ms = ?,
error_code = ?, message = ?
WHERE id = ?
""",
(status, now, elapsed_ms, error_code, message[:1000], int(run_id)),
)
def recent(self, limit: int = 20) -> list[dict[str, Any]]:
with self.database.connect() as connection:
rows = connection.execute(
"""
SELECT id, job_id, idempotency_key, status, attempt, started_at,
finished_at, elapsed_ms, error_code, message, output_version
FROM job_runs ORDER BY id DESC LIMIT ?
""",
(max(1, min(100, int(limit))),),
).fetchall()
return [dict(row) for row in rows]
+118
View File
@@ -0,0 +1,118 @@
from __future__ import annotations
import threading
import time
from collections.abc import Callable
from typing import Any
from backend.jobs.registry import JobRegistry
from backend.jobs.repository import SQLiteJobRunRepository
JobAction = Callable[[], Any]
class InProcessJobRunner:
def __init__(self, registry: JobRegistry, repository: SQLiteJobRunRepository) -> None:
self.registry = registry
self.repository = repository
self._locks: dict[str, threading.Lock] = {}
self._locks_guard = threading.Lock()
def submit(
self, job_id: str, idempotency_key: str, action: JobAction,
metadata: dict[str, Any] | None = None,
) -> bool:
definition = self.registry.get(job_id)
if self.repository.completed(job_id, idempotency_key):
return False
lock = self._lock(definition.lock_key)
if not lock.acquire(blocking=False):
return False
thread = threading.Thread(
target=self._execute_locked,
args=(job_id, idempotency_key, action, metadata, lock),
name=f"job-{job_id}-{idempotency_key}"[:80],
daemon=True,
)
thread.start()
return True
def run_inline(
self, job_id: str, idempotency_key: str, action: JobAction,
metadata: dict[str, Any] | None = None,
) -> bool:
definition = self.registry.get(job_id)
if self.repository.completed(job_id, idempotency_key):
return False
lock = self._lock(definition.lock_key)
if not lock.acquire(blocking=False):
return False
self._execute_locked(job_id, idempotency_key, action, metadata, lock)
return True
def start_scheduler(
self, callback: Callable[[], None], stop_event: threading.Event,
interval_seconds: float, initial_delay_seconds: float = 0,
) -> threading.Thread:
def schedule_loop() -> None:
if stop_event.wait(initial_delay_seconds):
return
while not stop_event.is_set():
try:
callback()
except Exception:
# Submitted jobs persist their own failures; the scheduler must stay alive.
pass
stop_event.wait(interval_seconds)
thread = threading.Thread(
target=schedule_loop,
name="background-job-scheduler",
daemon=True,
)
thread.start()
return thread
def wait_for_idle(self, timeout_seconds: float = 5) -> bool:
deadline = time.monotonic() + max(0, timeout_seconds)
while time.monotonic() <= deadline:
with self._locks_guard:
busy = any(lock.locked() for lock in self._locks.values())
if not busy:
return True
time.sleep(0.01)
return False
def _execute_locked(
self, job_id: str, idempotency_key: str, action: JobAction,
metadata: dict[str, Any] | None, lock: threading.Lock,
) -> None:
definition = self.registry.get(job_id)
try:
for attempt in range(1, definition.max_attempts + 1):
run_id = self.repository.start(
job_id, idempotency_key, definition.output_version, metadata
)
started = time.perf_counter()
try:
result = action()
if isinstance(result, dict) and result.get("status") == "failed":
raise RuntimeError(str(result.get("error") or "Job reported failure"))
elapsed_ms = round((time.perf_counter() - started) * 1000)
self.repository.finish(run_id, "success", elapsed_ms)
return
except Exception as exc:
elapsed_ms = round((time.perf_counter() - started) * 1000)
self.repository.finish(
run_id, "failed", elapsed_ms,
type(exc).__name__, str(exc),
)
if attempt >= definition.max_attempts:
return
finally:
lock.release()
def _lock(self, lock_key: str) -> threading.Lock:
with self._locks_guard:
return self._locks.setdefault(lock_key, threading.Lock())
+15
View File
@@ -0,0 +1,15 @@
from .gateway import (
LLMGateway,
LLMGatewayError,
LLMResult,
LLMStreamEvent,
ModelProfile,
)
__all__ = [
"LLMGateway",
"LLMGatewayError",
"LLMResult",
"LLMStreamEvent",
"ModelProfile",
]
+254
View File
@@ -0,0 +1,254 @@
from __future__ import annotations
import time
from collections.abc import Callable, Iterator
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Any, Generic, TypeVar
T = TypeVar("T")
class LLMGatewayError(ValueError):
"""Stable application error that does not expose provider details."""
def __init__(self, message: str, code: str = "unavailable") -> None:
super().__init__(message)
self.code = code
@dataclass(frozen=True)
class ModelProfile:
role: str
api_key: str
base_url: str
model: str
@property
def configured(self) -> bool:
return bool(self.api_key and self.base_url and self.model)
@dataclass(frozen=True)
class LLMResult(Generic[T]):
value: T
source: str
role: str
model: str
latency_ms: int
@dataclass(frozen=True)
class LLMStreamEvent(Generic[T]):
kind: str
value: T | None = None
source: str = ""
role: str = ""
model: str = ""
latency_ms: int = 0
class LLMGateway:
"""Single policy boundary for access, model fallback, and call auditing."""
def __init__(
self,
*,
database: Any,
user_id_supplier: Callable[[], int],
membership_supplier: Callable[[], dict[str, Any]],
settings_supplier: Callable[[], dict[str, Any]],
profile_supplier: Callable[[], dict[str, Any]],
) -> None:
self.database = database
self.user_id_supplier = user_id_supplier
self.membership_supplier = membership_supplier
self.settings_supplier = settings_supplier
self.profile_supplier = profile_supplier
def ensure_access(self, feature: str) -> tuple[str, tuple[ModelProfile, ...]]:
del feature # Reserved for future feature-specific policy.
membership = self.membership_supplier()
profile = self.profile_supplier()
source = str(profile.get("source") or "none")
profiles = self._model_profiles(profile)
if source == "none" or not profiles:
raise LLMGatewayError("智能功能尚未配置,请联系管理员。", "not_configured")
if source == "platform":
settings = self.settings_supplier()
limit = max(1, int(settings.get("member_daily_limit") or 50))
if not membership.get("active"):
raise LLMGatewayError("开通会员后可使用智能功能。", "membership_required")
if self._usage_today(source) >= limit:
raise LLMGatewayError(
f"今日会员模型额度已用完({limit} 次)。", "quota_exhausted"
)
return source, profiles
def call(
self,
feature: str,
prompt_version: str,
invoke: Callable[[ModelProfile], T],
error_types: tuple[type[BaseException], ...],
) -> LLMResult[T]:
source, profiles = self.ensure_access(feature)
started = time.perf_counter()
last_error: BaseException | None = None
for profile in profiles:
try:
value = invoke(profile)
except error_types as exc:
last_error = exc
continue
latency_ms = round((time.perf_counter() - started) * 1000)
self.audit(
feature, source, profile, "success", latency_ms, prompt_version
)
return LLMResult(
value=value,
source=source,
role=profile.role,
model=profile.model,
latency_ms=latency_ms,
)
failed = profiles[-1]
latency_ms = round((time.perf_counter() - started) * 1000)
self.audit(
feature,
source,
failed,
"failed",
latency_ms,
prompt_version,
self._error_code(last_error),
)
raise LLMGatewayError("智能解读服务暂不可用,请稍后重试。") from last_error
@staticmethod
def probe(profile: dict[str, Any], invoke: Callable[[ModelProfile], T]) -> T:
"""Route an explicit administrator connection test through the gateway boundary."""
model = ModelProfile(
role="probe",
api_key=str(profile.get("api_key") or ""),
base_url=str(profile.get("base_url") or ""),
model=str(profile.get("model") or ""),
)
return invoke(model)
def stream(
self,
feature: str,
prompt_version: str,
invoke: Callable[[ModelProfile], Iterator[T]],
error_types: tuple[type[BaseException], ...],
) -> Iterator[LLMStreamEvent[T]]:
source, profiles = self.ensure_access(feature)
started = time.perf_counter()
last_error: BaseException | None = None
for profile in profiles:
try:
upstream = iter(invoke(profile))
first = next(upstream)
except (*error_types, StopIteration) as exc:
last_error = exc
continue
yield LLMStreamEvent(kind="delta", value=first)
try:
for chunk in upstream:
yield LLMStreamEvent(kind="delta", value=chunk)
except error_types as exc:
latency_ms = round((time.perf_counter() - started) * 1000)
self.audit(
feature,
source,
profile,
"failed",
latency_ms,
prompt_version,
self._error_code(exc),
)
raise LLMGatewayError(
"智能解读连接中断,请稍后重试。"
) from exc
latency_ms = round((time.perf_counter() - started) * 1000)
self.audit(
feature, source, profile, "success", latency_ms, prompt_version
)
yield LLMStreamEvent(
kind="complete",
source=source,
role=profile.role,
model=profile.model,
latency_ms=latency_ms,
)
return
failed = profiles[-1]
latency_ms = round((time.perf_counter() - started) * 1000)
self.audit(
feature,
source,
failed,
"failed",
latency_ms,
prompt_version,
self._error_code(last_error),
)
raise LLMGatewayError("智能解读服务暂不可用,请稍后重试。") from last_error
def audit(
self,
feature: str,
source: str,
profile: ModelProfile,
status: str,
latency_ms: int,
prompt_version: str,
error_code: str = "",
input_tokens: int = 0,
output_tokens: int = 0,
) -> None:
self.database.record_llm_usage(
self.user_id_supplier(),
feature,
source,
profile.model,
status,
latency_ms,
role=profile.role,
prompt_version=prompt_version,
error_code=error_code,
input_tokens=input_tokens,
output_tokens=output_tokens,
)
def _usage_today(self, source: str) -> int:
now = datetime.now().astimezone()
start = now.replace(
hour=0, minute=0, second=0, microsecond=0
).astimezone(timezone.utc)
return self.database.count_llm_usage_since(
self.user_id_supplier(), source, start.isoformat(timespec="seconds")
)
@staticmethod
def _model_profiles(profile: dict[str, Any]) -> tuple[ModelProfile, ...]:
result = []
for role in ("primary", "fallback"):
item = profile.get(role) or {}
candidate = ModelProfile(
role=role,
api_key=str(item.get("api_key") or ""),
base_url=str(item.get("base_url") or ""),
model=str(item.get("model") or ""),
)
if candidate.configured:
result.append(candidate)
return tuple(result)
@staticmethod
def _error_code(error: BaseException | None) -> str:
if error is None:
return "empty_response"
return type(error).__name__[:80]