multica-agentandmultica-agent
bed6450992
- field_gates 按数据集配置关键字段非空率下限/非有限比例/相对上一批次的塌陷保护， 字段大面积为空的批次拒发并保留上一正式批次，可读失败原因入 batches.error - 股票主档交易日 20:00/23:10 自动刷新并发布版本化快照（eod_stocks + publications）， 覆盖新上市/简称变化/N前缀摘除；/v1/stocks 携带 batch_id/published_at，无变化跳过 - moneyflow 历史回补（默认 60 交易日，跳过已发布日期）；未发布点查返回 available_from/available_to 与 history_not_backfilled 标记，缺失不再静默 - eod-refresh 新增 --force --dataset 安全重发（仍走全部质量门，上一批次可回滚） - 保持 HEL-435 盘后重试机制；新增 22 项测试覆盖字段拒发/正常通过/旧批保留/ 主档新增改名/资金流覆盖/重复执行幂等 Co-authored-by: multica-agent <github@multica.ai>
104 lines
3.5 KiB
Python
104 lines
3.5 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_DB_PATH = Path(os.environ.get("DATAHUB_DB_PATH") or (ROOT / "data" / "datahub.db"))
|
|
DEFAULT_BACKUP_DIR = Path(os.environ.get("DATAHUB_BACKUP_DIR") or (ROOT / "data" / "backups"))
|
|
DEFAULT_CONFIG_PATH = ROOT / "config" / "hub-quality.config.json"
|
|
|
|
|
|
def _load_quality(path: Path) -> dict[str, Any]:
|
|
if not path.is_file():
|
|
return {}
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
@dataclass
|
|
class Settings:
|
|
host: str = "127.0.0.1"
|
|
port: int = 8766
|
|
encryption_key: str = ""
|
|
api_token: str = ""
|
|
admin_password: str = ""
|
|
tushare_token: str = ""
|
|
db_path: Path = DEFAULT_DB_PATH
|
|
backup_dir: Path = DEFAULT_BACKUP_DIR
|
|
quality: dict[str, Any] = field(default_factory=dict)
|
|
log_level: str = "INFO"
|
|
scheduler_enabled: bool = True
|
|
|
|
@property
|
|
def tushare_rate_per_minute(self) -> int:
|
|
return int(self.quality.get("tushare_rate_per_minute") or 300)
|
|
|
|
@property
|
|
def max_publish_attempts(self) -> int:
|
|
return int(self.quality.get("max_publish_attempts") or 5)
|
|
|
|
@property
|
|
def list_limit_default(self) -> int:
|
|
return int(self.quality.get("list_limit_default") or 5000)
|
|
|
|
@property
|
|
def list_limit_max(self) -> int:
|
|
return int(self.quality.get("list_limit_max") or 5000)
|
|
|
|
@property
|
|
def calendar_start(self) -> str:
|
|
return str(self.quality.get("calendar_start") or "20160101")
|
|
|
|
@property
|
|
def index_history_trading_days(self) -> int:
|
|
return int(self.quality.get("index_history_trading_days") or 260)
|
|
|
|
@property
|
|
def moneyflow_history_trading_days(self) -> int:
|
|
return int(self.quality.get("moneyflow_history_trading_days") or 60)
|
|
|
|
@property
|
|
def stocks_refresh_times(self) -> tuple[str, ...]:
|
|
raw = self.quality.get("stocks_refresh_times") or ["20:00", "23:10"]
|
|
if isinstance(raw, str):
|
|
raw = [raw]
|
|
return tuple(str(item) for item in raw)
|
|
|
|
@property
|
|
def eod_retry_start(self) -> str:
|
|
return str(self.quality.get("eod_retry_start") or "15:15")
|
|
|
|
@property
|
|
def eod_retry_interval_minutes(self) -> int:
|
|
return int(self.quality.get("eod_retry_interval_minutes") or 30)
|
|
|
|
@property
|
|
def eod_retry_cutoff(self) -> str:
|
|
return str(self.quality.get("eod_retry_cutoff") or "23:30")
|
|
|
|
|
|
def load_settings(
|
|
env: dict[str, str] | None = None,
|
|
config_path: Path | None = None,
|
|
) -> Settings:
|
|
environ = env if env is not None else dict(os.environ)
|
|
quality_path = config_path or DEFAULT_CONFIG_PATH
|
|
db_path = Path(environ.get("DATAHUB_DB_PATH") or DEFAULT_DB_PATH)
|
|
backup_dir = Path(environ.get("DATAHUB_BACKUP_DIR") or DEFAULT_BACKUP_DIR)
|
|
return Settings(
|
|
host=environ.get("DATAHUB_HOST") or "127.0.0.1",
|
|
port=int(environ.get("DATAHUB_PORT") or 8766),
|
|
encryption_key=str(environ.get("DATAHUB_ENCRYPTION_KEY") or "").strip(),
|
|
api_token=str(environ.get("DATAHUB_TOKEN") or "").strip(),
|
|
admin_password=str(environ.get("DATAHUB_ADMIN_PASSWORD") or "").strip(),
|
|
tushare_token=str(environ.get("TUSHARE_TOKEN") or "").strip(),
|
|
db_path=db_path,
|
|
backup_dir=backup_dir,
|
|
quality=_load_quality(quality_path),
|
|
log_level=environ.get("DATAHUB_LOG_LEVEL") or "INFO",
|
|
scheduler_enabled=str(environ.get("DATAHUB_SCHEDULER") or "1") not in {"0", "false", "False"},
|
|
)
|