Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fbcab3064c | ||
|
|
b402b3b2a8 | ||
|
|
3eaa36a8d5 | ||
|
|
3203574b6a | ||
|
|
35ee43ea02 | ||
|
|
78931f9c42 | ||
|
|
6f99ee9d9e | ||
|
|
9b2f0993d3 | ||
|
|
3bafd30aad | ||
|
|
d9358ab377 | ||
|
|
31da37b49e | ||
|
|
2e9442ac5a | ||
|
|
ad70d8fccc |
@@ -12,6 +12,10 @@ TUSHARE_TOKEN=your_tushare_token_here
|
||||
DATAHUB_BASE_URL=http://127.0.0.1:8766
|
||||
DATAHUB_TOKEN=
|
||||
|
||||
# 数据中枢控制台(8766)用它调用 /api/hub-admin/* 校验主站会话、读写模型池与
|
||||
# 会员/邀请码。两个服务必须填同一个值;32+ 随机字节,未设置则桥接直接拒绝。
|
||||
HUB_ADMIN_TOKEN=
|
||||
|
||||
# iFinD credentials live on xiaobai-datahub, not the website process.
|
||||
# IFIND_REFRESH_TOKEN=your_ifind_refresh_token_here
|
||||
# IFIND_ACCESS_TOKEN=
|
||||
|
||||
@@ -274,3 +274,18 @@ docker compose restart xiaobai-review
|
||||
当前部署使用局域网 HTTP,账号密码和会话只适合可信内网使用。不要直接将
|
||||
`8765` 暴露到互联网。以后需要公网访问时,应在容器前增加 Caddy 或 Nginx,
|
||||
启用 HTTPS,并限制可信来源。
|
||||
|
||||
### 数据中枢控制台(8766)
|
||||
|
||||
数据中枢是管理员的统一配置入口(数据源凭证、模型池、会员与邀请码),叠加
|
||||
`compose.datahub.yaml` 部署,容器名 `xiaobai-datahub`:
|
||||
|
||||
- 它没有独立账号,用主站管理员账号进入;未登录会跳主站登录页,非管理员一律 403。
|
||||
每个页面与接口都在服务端校验,前端隐藏不作为权限依据。
|
||||
- 两侧 `.env` 的 `HUB_ADMIN_TOKEN` 必须填成同一个随机值(服务间桥接令牌)。缺失或
|
||||
不一致时控制台无法校验会话,页面会停在门禁面板。
|
||||
- `REVIEW_BASE_URL` 是中枢访问主站的地址(同一 compose 网络内用服务名
|
||||
`http://xiaobai-review:8765`);`REVIEW_PUBLIC_URL` 是浏览器可达的主站地址,
|
||||
留空则按当前主机名推导。
|
||||
- 会话 cookie 靠"同主机不同端口"共享,因此主站与中枢必须对浏览器暴露在同一主机名下;
|
||||
8766 与 8765 同样只在可信内网开放。
|
||||
|
||||
@@ -51,6 +51,8 @@ python server.py
|
||||
|
||||
默认监听 `127.0.0.1:8765`(仅本机可访问)。浏览器打开该地址,首次使用先注册账号;第一个账号自动成为管理员,之后注册的默认为普通用户。
|
||||
|
||||
第一个账号之外的注册都必须填写一次性邀请码:管理员在数据中枢「会员管理」里生成,一个码只能成功注册一次,已使用或已作废的码不再可用。
|
||||
|
||||
主行情不再回退演示数据:盘前、非交易日或临时取数失败时沿用最近真实收盘快照;没有任何真实快照时,页面会提示等待管理员完成首次同步。
|
||||
|
||||
可选参数:
|
||||
@@ -115,9 +117,11 @@ compose.yaml
|
||||
.env.example 环境变量模板(复制为 .env 后填写)
|
||||
```
|
||||
|
||||
管理员通过页面右上角「系统管理」保存公共 Tushare Token、平台主/辅助模型、会员每日额度和后台刷新开关。所有用户读取同一份 SQLite 行情快照。`.env` 中的 Tushare 和平台 LLM 配置只用于初始化系统配置。
|
||||
管理员的配置入口在数据中枢(页面右上角「数据中枢」按钮,指向 8766):数据源凭证、模型池、会员与邀请码都在那里维护,数据仍存在主站同一份 SQLite 里。主站自身只保留「行情管理」面板(Tushare Token、后台刷新开关、手动刷新与补数)。所有用户读取同一份 SQLite 行情快照。`.env` 中的 Tushare 和平台 LLM 配置只用于初始化系统配置。
|
||||
|
||||
普通用户在「账号设置」中维护个人资料、查看会员状态和修改密码,不配置个人 LLM。有效会员使用平台模型;管理员可开通、续期、停用会员。平台模型受每日调用次数限制,管理员账号始终可用。
|
||||
数据中枢用主站的管理员账号进入,没有独立账号;桥接令牌 `HUB_ADMIN_TOKEN` 需在主站与中枢两侧 `.env` 填成同一个值。详见 [xiaobai-datahub/README.md](xiaobai-datahub/README.md)。
|
||||
|
||||
普通用户在「账号设置」中维护个人资料、查看会员状态和修改密码,不配置个人 LLM。有效会员使用平台模型;管理员在数据中枢开通、续期、停用会员。平台模型受每日调用次数限制,管理员账号始终可用。
|
||||
|
||||
相关文档:
|
||||
|
||||
|
||||
@@ -50,9 +50,11 @@ from backend.features.themes.routes import ThemeRoutesMixin
|
||||
from backend.http import HttpTransportMixin
|
||||
from backend.http.dispatch import (
|
||||
AUTHENTICATED_POST_HANDLERS,
|
||||
HUB_SERVICE_HANDLERS,
|
||||
PUBLIC_POST_HANDLERS,
|
||||
ApplicationHttpDispatchMixin,
|
||||
)
|
||||
from backend.http.hubadmin import HubAdminHttpMixin
|
||||
from backend.jobs.service import JobServiceMixin
|
||||
from backend.llm import LLMGateway
|
||||
from backend.llm.http import LLMHttpMixin
|
||||
@@ -152,6 +154,7 @@ class RequestHandler(
|
||||
AlertHttpMixin,
|
||||
ReviewHttpMixin,
|
||||
LLMHttpMixin,
|
||||
HubAdminHttpMixin,
|
||||
ApplicationHttpDispatchMixin,
|
||||
HttpTransportMixin,
|
||||
BaseHTTPRequestHandler,
|
||||
|
||||
@@ -3,6 +3,7 @@ from .m0002_job_runs import MIGRATION as M0002_JOB_RUNS
|
||||
from .m0003_llm_audit import MIGRATION as M0003_LLM_AUDIT
|
||||
from .m0004_mentor_notes import MIGRATION as M0004_MENTOR_NOTES
|
||||
from .m0005_account_switch_grants import MIGRATION as M0005_ACCOUNT_SWITCH_GRANTS
|
||||
from .m0006_invite_codes import MIGRATION as M0006_INVITE_CODES
|
||||
from .runner import Migration, MigrationError, MigrationRunner
|
||||
|
||||
MIGRATIONS = (
|
||||
@@ -11,6 +12,7 @@ MIGRATIONS = (
|
||||
M0003_LLM_AUDIT,
|
||||
M0004_MENTOR_NOTES,
|
||||
M0005_ACCOUNT_SWITCH_GRANTS,
|
||||
M0006_INVITE_CODES,
|
||||
)
|
||||
|
||||
__all__ = ["MIGRATIONS", "Migration", "MigrationError", "MigrationRunner"]
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
|
||||
from backend.database.migrations.runner import Migration
|
||||
|
||||
|
||||
def create_invite_codes(connection: sqlite3.Connection) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS invite_codes (
|
||||
code TEXT PRIMARY KEY,
|
||||
status TEXT NOT NULL DEFAULT 'unused',
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
created_by INTEGER,
|
||||
created_at TEXT NOT NULL,
|
||||
used_by INTEGER,
|
||||
used_at TEXT NOT NULL DEFAULT '',
|
||||
revoked_at TEXT NOT NULL DEFAULT '',
|
||||
FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL,
|
||||
FOREIGN KEY (used_by) REFERENCES users(id) ON DELETE SET NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_invite_codes_status
|
||||
ON invite_codes(status, created_at DESC)
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
MIGRATION = Migration(
|
||||
version="0006",
|
||||
name="create_invite_codes",
|
||||
action=create_invite_codes,
|
||||
signature="invite-codes:v1:code,status,note,created,used,revoked",
|
||||
)
|
||||
@@ -28,8 +28,14 @@ class AccountApplicationMixin:
|
||||
def update_membership(self, payload: dict[str, Any]) -> None:
|
||||
self.accounts.update_membership(payload)
|
||||
|
||||
def register_account(self, username: str, password: str, device_hash: str = "") -> dict[str, Any]:
|
||||
return self.accounts.register(username, password, device_hash)
|
||||
def register_account(
|
||||
self,
|
||||
username: str,
|
||||
password: str,
|
||||
device_hash: str = "",
|
||||
invite_code: str = "",
|
||||
) -> dict[str, Any]:
|
||||
return self.accounts.register(username, password, device_hash, invite_code)
|
||||
|
||||
def login_account(self, username: str, password: str, device_hash: str = "") -> dict[str, Any]:
|
||||
return self.accounts.login(username, password, device_hash)
|
||||
|
||||
@@ -41,6 +41,7 @@ class AccountHttpMixin:
|
||||
str(body.get("username") or ""),
|
||||
str(body.get("password") or ""),
|
||||
token_hash(device_raw),
|
||||
str(body.get("invite_code") or ""),
|
||||
)
|
||||
self._send_authenticated_session(result, HTTPStatus.CREATED, device_raw)
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
|
||||
@@ -5,6 +5,9 @@ from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
|
||||
INVITE_CONSUMED_MESSAGE = "邀请码无效或已被使用,请联系管理员重新获取。"
|
||||
|
||||
|
||||
class AccountRepositoryMixin:
|
||||
"""Original SQLite account persistence methods, moved without query changes."""
|
||||
|
||||
@@ -23,11 +26,14 @@ class AccountRepositoryMixin:
|
||||
username: str,
|
||||
password_salt: str,
|
||||
password_hash: str,
|
||||
invite_code: str = "",
|
||||
) -> dict[str, Any]:
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
try:
|
||||
with self.connect() as connection:
|
||||
role = "admin" if int(connection.execute("SELECT COUNT(*) FROM users").fetchone()[0]) == 0 else "user"
|
||||
if invite_code and not self._consume_invite_code(connection, invite_code, now):
|
||||
raise ValueError(INVITE_CONSUMED_MESSAGE)
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT INTO users
|
||||
@@ -37,10 +43,91 @@ class AccountRepositoryMixin:
|
||||
(username, password_salt, password_hash, role, now, now),
|
||||
)
|
||||
user_id = int(cursor.lastrowid)
|
||||
if invite_code:
|
||||
connection.execute(
|
||||
"UPDATE invite_codes SET used_by = ? WHERE code = ?",
|
||||
(user_id, invite_code),
|
||||
)
|
||||
except sqlite3.IntegrityError as exc:
|
||||
raise ValueError("该账号名已被使用。") from exc
|
||||
return {"id": user_id, "username": username, "role": role, "created_at": now}
|
||||
|
||||
@staticmethod
|
||||
def _consume_invite_code(
|
||||
connection: sqlite3.Connection, code: str, used_at: str
|
||||
) -> bool:
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
UPDATE invite_codes SET status = 'used', used_at = ?
|
||||
WHERE code = ? AND status = 'unused'
|
||||
""",
|
||||
(used_at, code),
|
||||
)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
def invite_code(self, code: str) -> dict[str, Any] | None:
|
||||
with self.connect() as connection:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT code, status, note, created_at, used_at, revoked_at, used_by
|
||||
FROM invite_codes WHERE code = ?
|
||||
""",
|
||||
(code,),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def create_invite_codes(
|
||||
self, codes: list[str], note: str, created_by: int
|
||||
) -> list[str]:
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
with self.connect() as connection:
|
||||
for code in codes:
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO invite_codes (code, status, note, created_by, created_at)
|
||||
VALUES (?, 'unused', ?, ?, ?)
|
||||
""",
|
||||
(code, note, created_by or None, now),
|
||||
)
|
||||
return list(codes)
|
||||
|
||||
def revoke_invite_code(self, code: str) -> bool:
|
||||
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||
with self.connect() as connection:
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
UPDATE invite_codes SET status = 'revoked', revoked_at = ?
|
||||
WHERE code = ? AND status = 'unused'
|
||||
""",
|
||||
(now, code),
|
||||
)
|
||||
return cursor.rowcount > 0
|
||||
|
||||
def list_invite_codes(self, limit: int = 100) -> list[dict[str, Any]]:
|
||||
with self.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT c.code, c.status, c.note, c.created_at, c.used_at, c.revoked_at,
|
||||
u.username AS used_by_username
|
||||
FROM invite_codes AS c
|
||||
LEFT JOIN users AS u ON u.id = c.used_by
|
||||
ORDER BY c.created_at DESC, c.code
|
||||
LIMIT ?
|
||||
""",
|
||||
(max(1, min(500, int(limit))),),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def count_invite_codes(self) -> dict[str, int]:
|
||||
with self.connect() as connection:
|
||||
rows = connection.execute(
|
||||
"SELECT status, COUNT(*) AS total FROM invite_codes GROUP BY status"
|
||||
).fetchall()
|
||||
counts = {"unused": 0, "used": 0, "revoked": 0}
|
||||
for row in rows:
|
||||
counts[str(row["status"])] = int(row["total"])
|
||||
return counts
|
||||
|
||||
def user_by_username(self, username: str) -> dict[str, Any] | None:
|
||||
with self.connect() as connection:
|
||||
row = connection.execute(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import secrets
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
@@ -82,16 +83,123 @@ class AccountService:
|
||||
MAX_GRANTS_PER_DEVICE = 5
|
||||
SWITCH_REAUTH_MESSAGE = "该账号需重新验证"
|
||||
|
||||
def register(self, username: str, password: str, device_hash: str = "") -> dict[str, Any]:
|
||||
INVITE_ALPHABET = "ACDEFGHJKLMNPQRTUVWXY34679"
|
||||
INVITE_MAX_BATCH = 20
|
||||
INVITE_LIST_LIMIT = 500
|
||||
|
||||
def register(
|
||||
self,
|
||||
username: str,
|
||||
password: str,
|
||||
device_hash: str = "",
|
||||
invite_code: str = "",
|
||||
) -> dict[str, Any]:
|
||||
username = username.strip()
|
||||
self.validate_input(username, password)
|
||||
with self.auth_lock:
|
||||
code = self.checked_invite_code(invite_code)
|
||||
salt, password_digest = hash_password(password)
|
||||
user = self.database.create_user(username, salt, password_digest)
|
||||
user = self.database.create_user(username, salt, password_digest, code)
|
||||
result = self.create_session(user)
|
||||
self.remember_account(device_hash, int(user["id"]), fresh=True)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def normalize_invite_code(cls, value: str) -> str:
|
||||
raw = "".join(
|
||||
character
|
||||
for character in str(value or "").upper()
|
||||
if character.isalnum()
|
||||
)
|
||||
if raw.startswith("XB") and len(raw) == 14:
|
||||
body = raw[2:]
|
||||
return f"XB-{body[0:4]}-{body[4:8]}-{body[8:12]}"
|
||||
return raw[:64]
|
||||
|
||||
@staticmethod
|
||||
def mask_invite_code(code: str) -> str:
|
||||
groups = str(code or "").split("-")
|
||||
if len(groups) < 3:
|
||||
return str(code or "")
|
||||
return f"{groups[0]}-{groups[1]}-••••"
|
||||
|
||||
@staticmethod
|
||||
def invite_handle(code: str) -> str:
|
||||
return hashlib.sha256(str(code or "").encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
def checked_invite_code(self, invite_code: str) -> str:
|
||||
if self.database.count_users() == 0:
|
||||
return ""
|
||||
code = self.normalize_invite_code(invite_code)
|
||||
if not code:
|
||||
raise ValueError("请填写邀请码,注册需要管理员发放的一次性邀请码。")
|
||||
record = self.database.invite_code(code)
|
||||
status = str((record or {}).get("status") or "")
|
||||
if not record:
|
||||
raise ValueError("邀请码不存在,请向管理员确认。")
|
||||
if status == "used":
|
||||
raise ValueError("该邀请码已被使用。")
|
||||
if status != "unused":
|
||||
raise ValueError("该邀请码已作废。")
|
||||
return code
|
||||
|
||||
def generate_invite_codes(
|
||||
self, count: int, note: str = "", created_by: int = 0
|
||||
) -> list[dict[str, str]]:
|
||||
try:
|
||||
total = int(count or 1)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError("生成数量不正确。") from exc
|
||||
if total < 1 or total > self.INVITE_MAX_BATCH:
|
||||
raise ValueError(f"每次最多生成 {self.INVITE_MAX_BATCH} 个邀请码。")
|
||||
codes: list[str] = []
|
||||
while len(codes) < total:
|
||||
body = "".join(secrets.choice(self.INVITE_ALPHABET) for _ in range(12))
|
||||
code = f"XB-{body[0:4]}-{body[4:8]}-{body[8:12]}"
|
||||
if code in codes or self.database.invite_code(code):
|
||||
continue
|
||||
codes.append(code)
|
||||
self.database.create_invite_codes(codes, str(note or "").strip()[:60], created_by)
|
||||
return [{"code": code, "code_id": self.invite_handle(code)} for code in codes]
|
||||
|
||||
def _stored_invite_code(self, reference: str) -> str:
|
||||
normalized = self.normalize_invite_code(reference)
|
||||
if normalized and self.database.invite_code(normalized):
|
||||
return normalized
|
||||
handle = str(reference or "").strip().lower()
|
||||
for row in self.database.list_invite_codes(self.INVITE_LIST_LIMIT):
|
||||
if self.invite_handle(str(row["code"])) == handle:
|
||||
return str(row["code"])
|
||||
return ""
|
||||
|
||||
def revoke_invite_code(self, reference: str) -> None:
|
||||
code = self._stored_invite_code(reference)
|
||||
record = self.database.invite_code(code) if code else None
|
||||
if not record:
|
||||
raise ValueError("邀请码不存在。")
|
||||
if str(record.get("status")) == "used":
|
||||
raise ValueError("该邀请码已被使用,无法作废。")
|
||||
if not self.database.revoke_invite_code(code):
|
||||
raise ValueError("该邀请码已作废。")
|
||||
|
||||
def invite_overview(self, limit: int = 100) -> dict[str, Any]:
|
||||
codes = []
|
||||
for row in self.database.list_invite_codes(limit):
|
||||
code = str(row["code"])
|
||||
codes.append(
|
||||
{
|
||||
"code_id": self.invite_handle(code),
|
||||
"code_masked": self.mask_invite_code(code),
|
||||
"status": str(row["status"]),
|
||||
"note": str(row.get("note") or ""),
|
||||
"created_at": str(row.get("created_at") or ""),
|
||||
"used_at": str(row.get("used_at") or ""),
|
||||
"revoked_at": str(row.get("revoked_at") or ""),
|
||||
"used_by_username": str(row.get("used_by_username") or ""),
|
||||
}
|
||||
)
|
||||
return {"summary": self.database.count_invite_codes(), "codes": codes}
|
||||
|
||||
def login(self, username: str, password: str, device_hash: str = "") -> dict[str, Any]:
|
||||
username = username.strip()
|
||||
if not username or not password:
|
||||
|
||||
@@ -128,6 +128,7 @@ class SystemServiceMixin:
|
||||
"base_url": profile["base_url"],
|
||||
"model": profile["model"],
|
||||
"configured": self._profile_configured(profile),
|
||||
"api_key_last4": profile["api_key"][-4:],
|
||||
}
|
||||
)
|
||||
return {
|
||||
|
||||
@@ -11,6 +11,24 @@ PUBLIC_POST_HANDLERS = {
|
||||
"/api/auth/forget": "auth_forget",
|
||||
}
|
||||
|
||||
# Service-to-service bridge for the data hub console. These paths are guarded by
|
||||
# the shared HUB_ADMIN_TOKEN header instead of a browser session, so they stay
|
||||
# out of the user-facing route registry on purpose.
|
||||
HUB_SERVICE_HANDLERS = {
|
||||
"/api/hub-admin/session": "hub_session_check",
|
||||
"/api/hub-admin/session/logout": "hub_session_logout",
|
||||
"/api/hub-admin/password/check": "hub_password_check",
|
||||
"/api/hub-admin/status": "hub_system_status",
|
||||
"/api/hub-admin/settings/save": "hub_save_settings",
|
||||
"/api/hub-admin/settings/test": "hub_test_model",
|
||||
"/api/hub-admin/models/fetch": "hub_fetch_models",
|
||||
"/api/hub-admin/members": "hub_members",
|
||||
"/api/hub-admin/membership/save": "hub_save_membership",
|
||||
"/api/hub-admin/invites": "hub_invites",
|
||||
"/api/hub-admin/invites/create": "hub_create_invites",
|
||||
"/api/hub-admin/invites/revoke": "hub_revoke_invite",
|
||||
}
|
||||
|
||||
AUTHENTICATED_POST_HANDLERS = {
|
||||
"/api/auth/logout": "auth_logout",
|
||||
"/api/account/birth-profile": "save_birth_profile",
|
||||
@@ -81,6 +99,10 @@ class ApplicationHttpDispatchMixin:
|
||||
|
||||
def do_POST(self) -> None:
|
||||
parsed = urlparse(self.path)
|
||||
if parsed.path in HUB_SERVICE_HANDLERS:
|
||||
if self.require_service_token():
|
||||
self._dispatch_named_handler(parsed.path, HUB_SERVICE_HANDLERS)
|
||||
return
|
||||
if self._dispatch_named_handler(parsed.path, PUBLIC_POST_HANDLERS):
|
||||
return
|
||||
if not self.require_auth() or not self.require_csrf():
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import secrets
|
||||
from collections.abc import Iterable
|
||||
from http import HTTPStatus
|
||||
@@ -62,6 +63,14 @@ class HttpTransportMixin:
|
||||
return False
|
||||
return True
|
||||
|
||||
def require_service_token(self) -> bool:
|
||||
expected = str(os.environ.get("HUB_ADMIN_TOKEN") or "").strip()
|
||||
supplied = self.headers.get("X-Hub-Admin-Token", "")
|
||||
if not expected or not supplied or not secrets.compare_digest(supplied, expected):
|
||||
self.send_json({"error": "服务令牌校验失败。"}, HTTPStatus.UNAUTHORIZED)
|
||||
return False
|
||||
return True
|
||||
|
||||
def require_admin(self) -> bool:
|
||||
if str(getattr(self, "auth_user", {}).get("role") or "user") != "admin":
|
||||
self.send_json({"error": "需要管理员权限。"}, HTTPStatus.FORBIDDEN)
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from backend.features.accounts.security import token_hash, verify_password
|
||||
|
||||
|
||||
class HubAdminHttpMixin:
|
||||
"""Service-to-service bridge used by the data hub console (port 8766).
|
||||
|
||||
Every handler here is reached only after `require_service_token`, so the
|
||||
shared `HUB_ADMIN_TOKEN` is the single trust boundary and no browser
|
||||
session or CSRF token is involved. The data hub still verifies the site
|
||||
session of the operator through `hub_session_check` before it exposes any
|
||||
of these results to a page.
|
||||
"""
|
||||
|
||||
def _hub_body(self) -> dict:
|
||||
return self.read_json_body(allow_empty=True)
|
||||
|
||||
def _hub_failure(self, exc: Exception) -> None:
|
||||
self.send_json({"error": str(exc)}, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
def hub_session_check(self) -> None:
|
||||
try:
|
||||
body = self._hub_body()
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
self._hub_failure(exc)
|
||||
return
|
||||
raw_token = str(body.get("session_token") or "")
|
||||
user = (
|
||||
self.application_service.database.session_user(token_hash(raw_token))
|
||||
if raw_token
|
||||
else None
|
||||
)
|
||||
if not user:
|
||||
self.send_json({"ok": True, "authenticated": False})
|
||||
return
|
||||
self.send_json(
|
||||
{
|
||||
"ok": True,
|
||||
"authenticated": True,
|
||||
"user": {
|
||||
"id": int(user["id"]),
|
||||
"username": str(user["username"]),
|
||||
"role": str(user.get("role") or "user"),
|
||||
"is_admin": str(user.get("role") or "user") == "admin",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
def hub_session_logout(self) -> None:
|
||||
try:
|
||||
body = self._hub_body()
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
self._hub_failure(exc)
|
||||
return
|
||||
raw_token = str(body.get("session_token") or "")
|
||||
if raw_token:
|
||||
self.application_service.database.delete_session(token_hash(raw_token))
|
||||
self.send_json({"ok": True})
|
||||
|
||||
def hub_password_check(self) -> None:
|
||||
try:
|
||||
body = self._hub_body()
|
||||
user_id = int(body.get("user_id") or 0)
|
||||
except (TypeError, ValueError, json.JSONDecodeError) as exc:
|
||||
self._hub_failure(exc)
|
||||
return
|
||||
stored = self.application_service.database.user_password(user_id)
|
||||
verified = bool(
|
||||
stored
|
||||
and verify_password(
|
||||
str(body.get("password") or ""),
|
||||
str(stored.get("password_salt") or ""),
|
||||
str(stored.get("password_hash") or ""),
|
||||
)
|
||||
)
|
||||
self.send_json({"ok": True, "verified": verified})
|
||||
|
||||
def hub_system_status(self) -> None:
|
||||
service = self.application_service
|
||||
self.send_json({"ok": True, **service.system_status(), "users": service.admin_users()})
|
||||
|
||||
def hub_save_settings(self) -> None:
|
||||
try:
|
||||
result = self.application_service.save_system_settings(self._hub_body())
|
||||
self.send_json({"ok": True, **result})
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
self._hub_failure(exc)
|
||||
|
||||
def hub_test_model(self) -> None:
|
||||
try:
|
||||
body = self._hub_body()
|
||||
result = self.application_service.test_system_llm_profile(
|
||||
str(body.get("model_id") or ""), body.get("profile") or {}
|
||||
)
|
||||
self.send_json({"ok": True, "result": result})
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
self._hub_failure(exc)
|
||||
|
||||
def hub_fetch_models(self) -> None:
|
||||
try:
|
||||
body = self._hub_body()
|
||||
models = self.application_service.fetch_llm_models(
|
||||
str(body.get("base_url") or ""),
|
||||
str(body.get("api_key") or ""),
|
||||
)
|
||||
self.send_json({"ok": True, "models": models})
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
self._hub_failure(exc)
|
||||
|
||||
def hub_members(self) -> None:
|
||||
service = self.application_service
|
||||
self.send_json(
|
||||
{
|
||||
"ok": True,
|
||||
"users": service.admin_users(),
|
||||
"membership": service.system_status()["membership"],
|
||||
}
|
||||
)
|
||||
|
||||
def hub_save_membership(self) -> None:
|
||||
try:
|
||||
service = self.application_service
|
||||
service.update_membership(self._hub_body())
|
||||
self.send_json({"ok": True, "users": service.admin_users()})
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
self._hub_failure(exc)
|
||||
|
||||
def hub_invites(self) -> None:
|
||||
self.send_json({"ok": True, **self.application_service.accounts.invite_overview()})
|
||||
|
||||
def hub_create_invites(self) -> None:
|
||||
try:
|
||||
body = self._hub_body()
|
||||
accounts = self.application_service.accounts
|
||||
codes = accounts.generate_invite_codes(
|
||||
body.get("count") or 1,
|
||||
str(body.get("note") or ""),
|
||||
int(body.get("created_by") or 0),
|
||||
)
|
||||
self.send_json({"ok": True, "created": codes, **accounts.invite_overview()})
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
self._hub_failure(exc)
|
||||
|
||||
def hub_revoke_invite(self) -> None:
|
||||
try:
|
||||
body = self._hub_body()
|
||||
accounts = self.application_service.accounts
|
||||
accounts.revoke_invite_code(str(body.get("code_id") or body.get("code") or ""))
|
||||
self.send_json({"ok": True, **accounts.invite_overview()})
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
self._hub_failure(exc)
|
||||
@@ -6,6 +6,7 @@ from urllib.parse import urlparse
|
||||
|
||||
from backend.bootstrap.config import validate_text
|
||||
from backend.features.screener.compiler import LLMCompilerError, test_llm_connection
|
||||
from backend.llm import transport as llm_transport
|
||||
|
||||
|
||||
class LLMServiceMixin:
|
||||
@@ -208,6 +209,37 @@ class LLMServiceMixin:
|
||||
start.isoformat(timespec="seconds"),
|
||||
)
|
||||
|
||||
def fetch_llm_models(self, base_url: str, api_key: str) -> list[str]:
|
||||
base_url = str(base_url or "").strip().rstrip("/")
|
||||
parsed = urlparse(base_url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
raise ValueError("Base URL 格式不正确。")
|
||||
key = str(api_key or "").strip()
|
||||
if not key:
|
||||
key = self._stored_api_key(base_url)
|
||||
if not key:
|
||||
raise ValueError("该供应商尚未保存 API Key,请先填写后再拉取模型列表。")
|
||||
try:
|
||||
return llm_transport.list_models(
|
||||
api_key=key,
|
||||
base_url=base_url,
|
||||
timeout=15,
|
||||
user_agent="XiaobaiReviewWeb/0.5",
|
||||
)
|
||||
except llm_transport.OpenAIHTTPError as exc:
|
||||
raise ValueError(exc.describe("模型列表拉取失败")) from exc
|
||||
except llm_transport.OpenAITransportError as exc:
|
||||
raise ValueError(f"模型列表拉取失败:{exc}") from exc
|
||||
|
||||
def _stored_api_key(self, base_url: str) -> str:
|
||||
for item in self._system_credentials.get("llm_models") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
stored = str(item.get("base_url") or "").strip().rstrip("/")
|
||||
if stored == base_url and item.get("api_key"):
|
||||
return str(item["api_key"])
|
||||
return ""
|
||||
|
||||
def test_system_llm_profile(self, model_id: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
current = next(
|
||||
(
|
||||
|
||||
@@ -67,6 +67,37 @@ def chat_completion(
|
||||
)
|
||||
|
||||
|
||||
def list_models(
|
||||
*,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
timeout: int,
|
||||
user_agent: str,
|
||||
) -> list[str]:
|
||||
request = urllib.request.Request(
|
||||
f"{base_url.rstrip('/')}/models",
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"User-Agent": user_agent,
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise OpenAIHTTPError(exc.code, _http_error_detail(exc)) from exc
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError) as exc:
|
||||
raise OpenAITransportError(str(exc)) from exc
|
||||
items = payload.get("data") if isinstance(payload, dict) else payload
|
||||
models = []
|
||||
for item in items or []:
|
||||
name = str((item or {}).get("id") or "") if isinstance(item, dict) else str(item or "")
|
||||
if name and name not in models:
|
||||
models.append(name)
|
||||
return models
|
||||
|
||||
|
||||
def stream_chat_completion(
|
||||
*,
|
||||
api_key: str,
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
# Start later (总工部署时) with:
|
||||
# docker compose -f compose.yaml -f compose.datahub.yaml up -d
|
||||
#
|
||||
# Required .env keys: DATAHUB_ENCRYPTION_KEY, DATAHUB_TOKEN, DATAHUB_ADMIN_PASSWORD, TUSHARE_TOKEN
|
||||
# Required .env keys: DATAHUB_ENCRYPTION_KEY, DATAHUB_TOKEN, HUB_ADMIN_TOKEN, TUSHARE_TOKEN
|
||||
# HUB_ADMIN_TOKEN 必须与主站 .env 里的同名变量一致:控制台靠它验证主站管理员会话。
|
||||
|
||||
services:
|
||||
xiaobai-datahub:
|
||||
@@ -18,7 +19,9 @@ services:
|
||||
environment:
|
||||
DATAHUB_ENCRYPTION_KEY: "${DATAHUB_ENCRYPTION_KEY:?DATAHUB_ENCRYPTION_KEY must be set}"
|
||||
DATAHUB_TOKEN: "${DATAHUB_TOKEN:?DATAHUB_TOKEN must be set}"
|
||||
DATAHUB_ADMIN_PASSWORD: "${DATAHUB_ADMIN_PASSWORD:?DATAHUB_ADMIN_PASSWORD must be set}"
|
||||
HUB_ADMIN_TOKEN: "${HUB_ADMIN_TOKEN:?HUB_ADMIN_TOKEN must be set}"
|
||||
REVIEW_BASE_URL: "${REVIEW_BASE_URL:-http://xiaobai-review:8765}"
|
||||
REVIEW_PUBLIC_URL: "${REVIEW_PUBLIC_URL:-}"
|
||||
TUSHARE_TOKEN: "${TUSHARE_TOKEN:-}"
|
||||
IFIND_REFRESH_TOKEN: "${IFIND_REFRESH_TOKEN:-}"
|
||||
IFIND_ACCESS_TOKEN: "${IFIND_ACCESS_TOKEN:-}"
|
||||
|
||||
@@ -13,6 +13,10 @@ services:
|
||||
- ./.env
|
||||
environment:
|
||||
APP_ENCRYPTION_KEY: "${APP_ENCRYPTION_KEY:?APP_ENCRYPTION_KEY must be set in .env}"
|
||||
# Shared secret for /api/hub-admin/*: the data hub console (8766) uses it to
|
||||
# verify this site's admin sessions and to read/write the model pool,
|
||||
# members and invite codes. Unset means the bridge refuses every call.
|
||||
HUB_ADMIN_TOKEN: "${HUB_ADMIN_TOKEN:-}"
|
||||
# Provider credentials are consumed only by xiaobai-datahub.
|
||||
TUSHARE_TOKEN: ""
|
||||
IFIND_REFRESH_TOKEN: ""
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
"api_exact_paths": 56,
|
||||
"api_prefixes": 0,
|
||||
"api_patterns": 11,
|
||||
"database_tables": 37,
|
||||
"database_tables": 38,
|
||||
"frontend_page_fragments": 12
|
||||
},
|
||||
"pages": [
|
||||
@@ -193,6 +193,7 @@
|
||||
"heaven_readings",
|
||||
"job_runs",
|
||||
"account_switch_grants",
|
||||
"invite_codes",
|
||||
"schema_migrations"
|
||||
],
|
||||
"background_job_methods": [
|
||||
@@ -478,8 +479,8 @@
|
||||
},
|
||||
{
|
||||
"path": "frontend/index.html",
|
||||
"bytes": 48403,
|
||||
"lines": 665
|
||||
"bytes": 46900,
|
||||
"lines": 638
|
||||
},
|
||||
{
|
||||
"path": "backend/data/providers/tushare_industries.py",
|
||||
@@ -556,11 +557,6 @@
|
||||
"bytes": 14942,
|
||||
"lines": 235
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/admin.js",
|
||||
"bytes": 14836,
|
||||
"lines": 283
|
||||
},
|
||||
{
|
||||
"path": "backend/features/screener/data_sync.py",
|
||||
"bytes": 14743,
|
||||
@@ -573,8 +569,8 @@
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/session.js",
|
||||
"bytes": 13219,
|
||||
"lines": 289
|
||||
"bytes": 13633,
|
||||
"lines": 296
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/insights_auction_data.py",
|
||||
@@ -583,8 +579,8 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/features/system/service.py",
|
||||
"bytes": 12180,
|
||||
"lines": 265
|
||||
"bytes": 12242,
|
||||
"lines": 266
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/insights_auction.py",
|
||||
@@ -651,16 +647,16 @@
|
||||
"bytes": 6547,
|
||||
"lines": 220
|
||||
},
|
||||
{
|
||||
"path": "backend/application.py",
|
||||
"bytes": 6500,
|
||||
"lines": 164
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/sentiment/page.html",
|
||||
"bytes": 6488,
|
||||
"lines": 81
|
||||
},
|
||||
{
|
||||
"path": "backend/application.py",
|
||||
"bytes": 6399,
|
||||
"lines": 161
|
||||
},
|
||||
{
|
||||
"path": "frontend/pages/market/stock-detail.js",
|
||||
"bytes": 6325,
|
||||
@@ -701,6 +697,11 @@
|
||||
"bytes": 5350,
|
||||
"lines": 74
|
||||
},
|
||||
{
|
||||
"path": "backend/http/dispatch.py",
|
||||
"bytes": 5281,
|
||||
"lines": 139
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/feedback.js",
|
||||
"bytes": 5157,
|
||||
@@ -721,6 +722,11 @@
|
||||
"bytes": 4406,
|
||||
"lines": 124
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/admin.js",
|
||||
"bytes": 4376,
|
||||
"lines": 107
|
||||
},
|
||||
{
|
||||
"path": "backend/features/market/routes.py",
|
||||
"bytes": 4276,
|
||||
@@ -736,11 +742,6 @@
|
||||
"bytes": 4242,
|
||||
"lines": 129
|
||||
},
|
||||
{
|
||||
"path": "backend/http/dispatch.py",
|
||||
"bytes": 4196,
|
||||
"lines": 117
|
||||
},
|
||||
{
|
||||
"path": "frontend/shared/table.js",
|
||||
"bytes": 3790,
|
||||
@@ -783,8 +784,8 @@
|
||||
},
|
||||
{
|
||||
"path": "backend/features/accounts/application.py",
|
||||
"bytes": 2514,
|
||||
"lines": 63
|
||||
"bytes": 2597,
|
||||
"lines": 69
|
||||
},
|
||||
{
|
||||
"path": "backend/jobs/service.py",
|
||||
|
||||
+5
-32
@@ -73,6 +73,7 @@
|
||||
<label class="form-field"><span>账号名</span><input id="authUsername" type="text" minlength="3" maxlength="30" autocomplete="username" required></label>
|
||||
<label class="form-field"><span>密码</span><input id="authPassword" type="password" minlength="8" maxlength="128" autocomplete="current-password" required></label>
|
||||
<label id="authConfirmField" class="form-field" hidden><span>确认密码</span><input id="authPasswordConfirm" type="password" minlength="8" maxlength="128" autocomplete="new-password"></label>
|
||||
<label id="authInviteField" class="form-field" hidden><span>邀请码</span><input id="authInviteCode" type="text" maxlength="32" autocomplete="off" spellcheck="false" placeholder="XB-XXXX-XXXX-XXXX"></label>
|
||||
<p id="authError" class="auth-error" hidden></p>
|
||||
<button id="authSubmitButton" class="button primary" type="submit">登录</button>
|
||||
</form>
|
||||
@@ -156,7 +157,8 @@
|
||||
</div>
|
||||
<button id="refreshButton" class="button command-button" type="button" title="刷新"><i data-lucide="refresh-cw"></i><span>刷新</span></button>
|
||||
<button id="syncButton" class="button primary command-button" type="button" title="后台刷新" hidden><i data-lucide="cloud-download"></i><span>后台刷新</span></button>
|
||||
<button id="settingsButton" class="button command-button" type="button" title="系统管理" hidden><i data-lucide="settings-2"></i><span>系统管理</span></button>
|
||||
<button id="marketAdminButton" class="button command-button" type="button" title="行情管理" hidden><i data-lucide="settings-2"></i><span>行情管理</span></button>
|
||||
<button id="settingsButton" class="button command-button" type="button" title="数据中枢" hidden><i data-lucide="database"></i><span>数据中枢</span></button>
|
||||
<div class="account-menu-shell">
|
||||
<div id="accountRoleBadges" class="account-role-badges" aria-label="账号身份">
|
||||
<span id="accountAdminBadge" class="account-role-badge admin-role-badge" title="管理员" hidden><i data-lucide="shield-check"></i><span>管理员</span></span>
|
||||
@@ -592,18 +594,11 @@
|
||||
|
||||
<dialog id="adminDialog" class="settings-dialog admin-dialog" aria-labelledby="adminDialogTitle">
|
||||
<div class="dialog-header">
|
||||
<div><span class="dialog-eyebrow">管理员</span><h2 id="adminDialogTitle">系统配置</h2></div>
|
||||
<div><span class="dialog-eyebrow">管理员</span><h2 id="adminDialogTitle">行情管理</h2></div>
|
||||
<button id="closeAdminDialog" class="icon-button" type="button" aria-label="关闭" title="关闭"><i data-lucide="x"></i></button>
|
||||
</div>
|
||||
<div id="adminConnectionStatus" class="connection-status" aria-live="polite">正在读取系统状态</div>
|
||||
<div class="admin-section-picker">
|
||||
<label for="adminSectionSelect">管理项目</label>
|
||||
<select id="adminSectionSelect">
|
||||
<option value="market">行情管理</option>
|
||||
<option value="models">模型池</option>
|
||||
<option value="members">会员管理</option>
|
||||
</select>
|
||||
</div>
|
||||
<p class="form-hint">模型池、会员与邀请码已统一在数据中枢管理,点击顶栏「数据中枢」进入。</p>
|
||||
<div class="admin-panel" data-admin-panel="market">
|
||||
<form id="systemMarketForm" class="settings-section">
|
||||
<div class="settings-section-heading"><h3>公共行情</h3><span id="systemDataStatus">待检查</span></div>
|
||||
@@ -624,28 +619,6 @@
|
||||
<div class="dialog-actions"><button id="backfillButton" class="button" type="button">开始回补</button></div>
|
||||
</section>
|
||||
</div>
|
||||
<div class="admin-panel" data-admin-panel="models" hidden>
|
||||
<form id="systemModelsForm" class="settings-section">
|
||||
<div class="settings-section-heading"><h3>平台模型池</h3><span>会员共享</span></div>
|
||||
<div class="model-role-selectors">
|
||||
<label class="form-field"><span>主模型</span><select id="platformPrimaryModelSelect"></select></label>
|
||||
<label class="form-field"><span>辅助模型</span><select id="platformFallbackModelSelect"><option value="">不启用辅助模型</option></select></label>
|
||||
</div>
|
||||
<div id="modelPoolList" class="model-pool-list"></div>
|
||||
<div class="dialog-actions admin-inline-actions"><button id="addPlatformModel" class="button" type="button"><i data-lucide="plus"></i>添加模型</button><button class="button primary" type="submit">保存模型池</button></div>
|
||||
</form>
|
||||
</div>
|
||||
<div class="admin-panel" data-admin-panel="members" hidden>
|
||||
<form id="membershipSettingsForm" class="settings-section membership-limit-form">
|
||||
<div class="settings-section-heading"><h3>会员调用额度</h3><span>每日自动重置</span></div>
|
||||
<label class="form-field compact-number-field"><span>会员每日智能分析上限</span><input id="memberDailyLimit" type="number" min="1" max="1000" value="50"></label>
|
||||
<div class="dialog-actions"><button class="button primary" type="submit">保存调用额度</button></div>
|
||||
</form>
|
||||
<section class="settings-section">
|
||||
<div class="settings-section-heading"><h3>会员账号</h3><span>手动开通与续期</span></div>
|
||||
<div id="adminUsersList" class="admin-users-list"></div>
|
||||
</section>
|
||||
</div>
|
||||
</dialog>
|
||||
|
||||
<div id="loadingOverlay" class="loading-overlay" hidden>
|
||||
|
||||
+11
-2
@@ -15,6 +15,7 @@
|
||||
error: "",
|
||||
username: "",
|
||||
password: "",
|
||||
inviteCode: "",
|
||||
passwordVisible: false,
|
||||
};
|
||||
|
||||
@@ -119,6 +120,7 @@
|
||||
`<label class="form-field"><span>账号名</span><input id="loginUsername" type="text" minlength="3" maxlength="30" autocomplete="username" placeholder="请输入账号名" value="${escapeHtml(state.username)}" required></label>`,
|
||||
`<label class="form-field"><span>密码</span><span class="login-password-wrap"><input id="loginPassword" class="${invalid.trim()}" type="${passwordType}" minlength="8" maxlength="128" autocomplete="${registering ? "new-password" : "current-password"}" placeholder="请输入密码" value="${escapeHtml(state.password)}" required><button class="login-password-toggle" type="button" data-login-action="toggle-password" aria-pressed="${state.passwordVisible ? "true" : "false"}" aria-label="${state.passwordVisible ? "隐藏密码" : "显示密码"}">${passwordToggle}</button></span></label>`,
|
||||
`<label class="form-field" id="loginConfirmField"${registering ? "" : " hidden"}><span>确认密码</span><input id="loginPasswordConfirm" type="password" minlength="8" maxlength="128" autocomplete="new-password"${registering ? " required" : ""}></label>`,
|
||||
`<label class="form-field" id="loginInviteField"${registering ? "" : " hidden"}><span>邀请码</span><input id="loginInviteCode" type="text" maxlength="32" autocomplete="off" spellcheck="false" placeholder="XB-XXXX-XXXX-XXXX" value="${escapeHtml(state.inviteCode)}"${registering ? " required" : ""}></label>`,
|
||||
state.error ? `<p class="login-error">${escapeHtml(state.error)}</p>` : '<p class="login-error" hidden></p>',
|
||||
`<button class="button primary login-submit" type="submit"${state.loading ? " disabled" : ""}>`,
|
||||
state.loading ? '<span class="login-spinner" aria-hidden="true"></span>' : "",
|
||||
@@ -289,9 +291,14 @@
|
||||
event.preventDefault();
|
||||
const username = document.querySelector("#loginUsername").value.trim();
|
||||
const password = document.querySelector("#loginPassword").value;
|
||||
const registering = state.mode === "register";
|
||||
const inviteCode = registering
|
||||
? document.querySelector("#loginInviteCode").value.trim()
|
||||
: "";
|
||||
state.username = username;
|
||||
state.password = password;
|
||||
if (state.mode === "register" && password !== document.querySelector("#loginPasswordConfirm").value) {
|
||||
state.inviteCode = inviteCode;
|
||||
if (registering && password !== document.querySelector("#loginPasswordConfirm").value) {
|
||||
setError("两次输入的密码不一致。");
|
||||
render();
|
||||
return;
|
||||
@@ -300,7 +307,9 @@
|
||||
setError("");
|
||||
render();
|
||||
try {
|
||||
await api.request(`/api/auth/${state.mode}`, "POST", { username, password });
|
||||
await api.request(`/api/auth/${state.mode}`, "POST", registering
|
||||
? { username, password, invite_code: inviteCode }
|
||||
: { username, password });
|
||||
await celebrateLogin();
|
||||
enterApp();
|
||||
} catch (error) {
|
||||
|
||||
@@ -52,8 +52,7 @@
|
||||
{ key: "system/profile", label: "账号资料", icon: "user" },
|
||||
{ key: "system/password", label: "修改密码", icon: "lock" },
|
||||
{ key: "system/membership", label: "会员状态", icon: "gem" },
|
||||
{ key: "system/admin", label: "系统设置", icon: "sliders-horizontal", adminOnly: true },
|
||||
{ key: "system/members", label: "会员管理", icon: "users", adminOnly: true }
|
||||
{ key: "system/admin", label: "行情管理", icon: "sliders-horizontal", adminOnly: true }
|
||||
]
|
||||
}
|
||||
},
|
||||
|
||||
+26
-361
@@ -198,11 +198,7 @@
|
||||
system: {
|
||||
account: null,
|
||||
admin: null,
|
||||
adminTab: "market",
|
||||
models: [],
|
||||
accounts: [],
|
||||
editingModelId: "",
|
||||
editingUserId: "",
|
||||
},
|
||||
};
|
||||
|
||||
@@ -3137,7 +3133,6 @@
|
||||
"system/password": setupSystemPage,
|
||||
"system/membership": setupSystemPage,
|
||||
"system/admin": setupSystemPage,
|
||||
"system/members": setupSystemPage,
|
||||
};
|
||||
|
||||
const COMPLEX_LOADERS = {
|
||||
@@ -3160,7 +3155,6 @@
|
||||
"system/password": loadSystem,
|
||||
"system/membership": loadSystem,
|
||||
"system/admin": loadSystem,
|
||||
"system/members": loadSystem,
|
||||
};
|
||||
|
||||
function isComplexPage(key) {
|
||||
@@ -4938,7 +4932,7 @@
|
||||
return '<div class="m-state m-motion-rise-in" data-system-page="forbidden">' +
|
||||
'<span class="m-state-icon">' + icon("lock", 26) + "</span>" +
|
||||
"<p>仅管理员可访问</p>" +
|
||||
"<small>系统设置和会员管理需要管理员权限。</small>" +
|
||||
"<small>行情管理与数据中枢需要管理员权限。</small>" +
|
||||
"</div>";
|
||||
}
|
||||
|
||||
@@ -4974,7 +4968,6 @@
|
||||
state.sort = { key: "", dir: null };
|
||||
state.sortTable = { cols: null, reapply: null };
|
||||
state.detail = null;
|
||||
if (key === "system/admin") state.system.adminTab = state.system.adminTab || "market";
|
||||
document.getElementById("m-view").classList.add("m-view-feature");
|
||||
global.MobileRouter.updateHeader({ title: findLabel(key) || key, back: true, actions: "" });
|
||||
document.getElementById("m-view").innerHTML = complexFrame(key, complexScroll(skeletonHtml(6)));
|
||||
@@ -4982,7 +4975,7 @@
|
||||
|
||||
function loadSystem() {
|
||||
const key = state.key;
|
||||
if ((key === "system/admin" || key === "system/members") && !global.MobileSession.isAdmin()) {
|
||||
if (key === "system/admin" && !global.MobileSession.isAdmin()) {
|
||||
systemFill(systemForbiddenHtml());
|
||||
return;
|
||||
}
|
||||
@@ -4991,15 +4984,12 @@
|
||||
return;
|
||||
}
|
||||
const seq = nextSeq();
|
||||
const url = (key === "system/admin" || key === "system/members") ? "/api/admin/settings" : "/api/account/status";
|
||||
global.MobileAPI.request(url).then(function (payload) {
|
||||
const url = key === "system/admin" ? "/api/admin/settings" : "/api/account/status";
|
||||
return global.MobileAPI.request(url).then(function (payload) {
|
||||
if (seq !== state.seq || state.key !== key) return;
|
||||
if (key === "system/admin" || key === "system/members") {
|
||||
if (key === "system/admin") {
|
||||
state.system.admin = payload || {};
|
||||
state.system.models = ((payload.llm && payload.llm.models) || []).map(function (item) {
|
||||
return Object.assign({}, item);
|
||||
});
|
||||
renderSystemAdmin(key);
|
||||
renderSystemAdmin();
|
||||
} else {
|
||||
state.system.account = payload || {};
|
||||
if (key === "system/profile") renderSystemProfile();
|
||||
@@ -5089,8 +5079,7 @@
|
||||
{ key: "system/membership", icon: "gem", label: "会员状态", hint: "有效期与智能分析额度" }
|
||||
];
|
||||
const adminRows = [
|
||||
{ key: "system/admin", icon: "sliders-horizontal", label: "系统设置", hint: "行情数据 · 模型池" },
|
||||
{ key: "system/members", icon: "users", label: "会员管理", hint: "开通 · 续期 · 额度" }
|
||||
{ key: "system/admin", icon: "sliders-horizontal", label: "行情管理", hint: "数据源状态 · 后台刷新 · 历史回补" }
|
||||
];
|
||||
const html =
|
||||
'<div class="m-sys-home" data-system-page="home">' +
|
||||
@@ -5104,7 +5093,7 @@
|
||||
'<h3 class="m-sys-group-title">偏好</h3>' +
|
||||
'<div class="m-card m-sys-list">' + systemThemeRowHtml() + "</div>" +
|
||||
(global.MobileSession.isAdmin()
|
||||
? '<h3 class="m-sys-group-title">管理员专区</h3><div class="m-card m-sys-list">' + adminRows.map(systemRowHtml).join("") + "</div>"
|
||||
? '<h3 class="m-sys-group-title">管理员专区</h3><div class="m-card m-sys-list">' + adminRows.map(systemRowHtml).join("") + dataHubRowHtml() + "</div>"
|
||||
: "") +
|
||||
'<h3 class="m-sys-group-title">其他</h3>' +
|
||||
'<div class="m-card m-sys-list">' +
|
||||
@@ -5117,7 +5106,7 @@
|
||||
'<span class="m-sys-row-body"><strong>退出登录</strong><small>退出后需要重新登录</small></span>' +
|
||||
'<span class="m-sys-row-chevron">' + icon("chevron-right", 16) + "</span></button>" +
|
||||
"</div>" +
|
||||
'<p class="m-sys-foot">' + (global.MobileSession.isAdmin() ? "小白复盘 · 内网个人版" : "系统设置与会员管理仅管理员可见") + "</p>" +
|
||||
'<p class="m-sys-foot">' + (global.MobileSession.isAdmin() ? "小白复盘 · 内网个人版" : "行情管理与数据中枢仅管理员可见") + "</p>" +
|
||||
"</div>";
|
||||
document.getElementById("m-view").innerHTML = html;
|
||||
}
|
||||
@@ -5207,14 +5196,6 @@
|
||||
systemFill(html);
|
||||
}
|
||||
|
||||
function adminTabHtml() {
|
||||
const tab = state.system.adminTab === "models" ? "models" : "market";
|
||||
return '<div class="m-source-tabs" role="tablist" aria-label="系统设置分类">' +
|
||||
'<button class="m-source-tab' + (tab === "market" ? " active" : "") + '" type="button" role="tab" data-system-admin-tab="market">行情管理</button>' +
|
||||
'<button class="m-source-tab' + (tab === "models" ? " active" : "") + '" type="button" role="tab" data-system-admin-tab="models">模型池</button>' +
|
||||
"</div>";
|
||||
}
|
||||
|
||||
function statusDot(ok) {
|
||||
return '<span class="m-sys-dot' + (ok ? " m-sys-dot--ok" : "") + '"></span>';
|
||||
}
|
||||
@@ -5228,16 +5209,10 @@
|
||||
return " 未配置";
|
||||
}
|
||||
|
||||
function renderSystemAdmin(key) {
|
||||
if (key === "system/members") {
|
||||
renderSystemMembers();
|
||||
return;
|
||||
}
|
||||
function renderSystemAdmin() {
|
||||
const payload = state.system.admin || {};
|
||||
const data = payload.data || {};
|
||||
const ifind = data.ifind || {};
|
||||
const llm = payload.llm || {};
|
||||
const tab = state.system.adminTab === "models" ? "models" : "market";
|
||||
const marketHtml =
|
||||
'<div class="m-sys-body" data-system-admin-panel="market">' +
|
||||
'<div class="m-card m-sys-section"><strong>数据源状态</strong>' +
|
||||
@@ -5263,178 +5238,28 @@
|
||||
formFieldHtml("结束日期", dateInputHtml("m-sys-backfill-end", ""), false) +
|
||||
'<button class="m-btn-outline" type="button" data-system-backfill>开始回补</button>' +
|
||||
'<p class="m-sys-hint">回补用于补齐缺失的历史行情,开始前会再次确认;回补期间页面可正常使用。</p></div>' +
|
||||
'<div class="m-card m-sys-section"><strong>模型池 · 会员 · 邀请码</strong>' +
|
||||
'<p class="m-sys-hint">这些配置已统一在数据中枢管理。</p>' +
|
||||
'<button class="m-btn-outline" type="button" data-system-datahub>打开数据中枢</button></div>' +
|
||||
"</div>";
|
||||
const models = state.system.models || [];
|
||||
const modelsHtml =
|
||||
'<div class="m-sys-body" data-system-admin-panel="models">' +
|
||||
'<div class="m-card m-sys-section"><strong>模型分工</strong>' +
|
||||
formFieldHtml("主模型", '<select id="m-sys-primary-model"></select>', false) +
|
||||
formFieldHtml("辅助模型", '<select id="m-sys-fallback-model"></select>', false) +
|
||||
'<p class="m-sys-hint">主模型不可用时自动改用辅助模型</p>' +
|
||||
'<button class="m-btn-primary" type="button" data-system-save-models>保存分工</button></div>' +
|
||||
'<div class="m-card m-sys-section"><strong>模型池 · ' + models.length + " 个</strong>" +
|
||||
'<div id="m-sys-model-list">' + renderModelPoolHtml(models) + "</div>" +
|
||||
'<p class="m-sys-hint">点任意模型卡片进入编辑:改名称、地址、密钥、测试连接或删除</p>' +
|
||||
'<button class="m-btn-primary" type="button" data-system-add-model>+ 添加模型</button></div>' +
|
||||
"</div>";
|
||||
const page = document.querySelector(".m-page");
|
||||
if (!page) return;
|
||||
page.innerHTML = adminTabHtml() + '<div class="m-scroll" id="m-scroll">' + (tab === "models" ? modelsHtml : marketHtml) + "</div>";
|
||||
if (tab === "models") updateSystemModelRoleOptions(llm.primary_model_id || "", llm.fallback_model_id || "");
|
||||
systemFill(marketHtml);
|
||||
}
|
||||
|
||||
function hostOfUrl(url) {
|
||||
try {
|
||||
return new URL(url).host;
|
||||
} catch (error) {
|
||||
return String(url || "").replace(/^https?:\/\//, "").split("/")[0] || "--";
|
||||
}
|
||||
function dataHubConsoleUrl() {
|
||||
if (global.XIAOBAI_DATAHUB_URL) return String(global.XIAOBAI_DATAHUB_URL);
|
||||
return global.location.protocol + "//" + global.location.hostname + ":8766/admin/";
|
||||
}
|
||||
|
||||
function renderModelPoolHtml(models) {
|
||||
if (!models.length) {
|
||||
return '<div class="m-state"><p>模型池为空,请先添加模型</p></div>';
|
||||
function openDataHubConsole() {
|
||||
if (!global.MobileSession.isAdmin()) return;
|
||||
global.open(dataHubConsoleUrl(), "_blank", "noopener");
|
||||
}
|
||||
const llm = (state.system.admin && state.system.admin.llm) || {};
|
||||
return models.map(function (item) {
|
||||
const badges = [];
|
||||
if (item.id === llm.primary_model_id) badges.push('<span class="m-sys-badge m-sys-badge--admin">主模型</span>');
|
||||
if (item.id === llm.fallback_model_id) badges.push('<span class="m-sys-badge">辅助</span>');
|
||||
badges.push('<span class="m-sys-badge' + (item.configured ? " m-sys-badge--ok" : "") + '">' + (item.configured ? "已配置" : "待配置") + "</span>");
|
||||
return '<button class="m-sys-model-card" type="button" data-model-id="' + escapeHtml(item.id) + '" data-system-edit-model>' +
|
||||
"<div><strong>" + escapeHtml(item.name || "未命名模型") + "</strong>" +
|
||||
'<div class="m-sys-badges">' + badges.join("") + "</div>" +
|
||||
'<p class="m-sys-hint">' + escapeHtml(hostOfUrl(item.base_url) + " · " + (item.model || "未填写标识")) + "</p></div>" +
|
||||
|
||||
function dataHubRowHtml() {
|
||||
return '<button class="m-sys-row" type="button" data-system-datahub>' +
|
||||
'<span class="m-sys-row-icon">' + icon("database", 18) + "</span>" +
|
||||
'<span class="m-sys-row-body"><strong>数据中枢</strong><small>数据源 · 模型池 · 会员与邀请码</small></span>' +
|
||||
'<span class="m-sys-row-chevron">' + icon("chevron-right", 16) + "</span></button>";
|
||||
}).join("");
|
||||
}
|
||||
|
||||
function collectSystemModelPool() {
|
||||
return (state.system.models || []).map(function (item) {
|
||||
return {
|
||||
id: item.id,
|
||||
name: item.name || "",
|
||||
base_url: item.base_url || "",
|
||||
model: item.model || "",
|
||||
api_key: item.api_key || "",
|
||||
configured: Boolean(item.configured)
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function updateSystemModelRoleOptions(primaryId, fallbackId) {
|
||||
const models = collectSystemModelPool();
|
||||
const options = models.map(function (item) {
|
||||
return '<option value="' + escapeHtml(item.id) + '">' + escapeHtml(item.name || item.model || "未命名模型") + "</option>";
|
||||
}).join("");
|
||||
const primary = document.getElementById("m-sys-primary-model");
|
||||
const fallback = document.getElementById("m-sys-fallback-model");
|
||||
if (!primary || !fallback) return;
|
||||
primary.innerHTML = models.length ? options : '<option value="">暂无模型</option>';
|
||||
fallback.innerHTML = '<option value="">不启用辅助模型</option>' + options;
|
||||
const keepPrimary = models.some(function (item) { return item.id === primaryId; }) ? primaryId : (models[0] && models[0].id) || "";
|
||||
primary.value = keepPrimary;
|
||||
fallback.value = models.some(function (item) { return item.id === fallbackId; }) && fallbackId !== keepPrimary ? fallbackId : "";
|
||||
}
|
||||
|
||||
function openModelEditSheet(modelId) {
|
||||
const models = state.system.models || [];
|
||||
const item = models.find(function (row) { return row.id === modelId; }) || {};
|
||||
state.system.editingModelId = modelId;
|
||||
openSheet(
|
||||
'<div class="m-sheet-head"><h2>编辑模型</h2>' +
|
||||
'<button class="m-sheet-close" type="button" data-sheet-close aria-label="关闭">' + icon("close", 20) + "</button></div>" +
|
||||
'<div class="m-sheet-body" data-model-id="' + escapeHtml(modelId) + '">' +
|
||||
formFieldHtml("显示名称", '<input data-model-field="name" maxlength="50" value="' + escapeHtml(item.name || "") + '">', true) +
|
||||
formFieldHtml("API Base URL", '<input data-model-field="base_url" type="url" value="' + escapeHtml(item.base_url || "https://api.openai.com/v1") + '">', true) +
|
||||
formFieldHtml("模型标识", '<input data-model-field="model" maxlength="100" value="' + escapeHtml(item.model || "") + '">', true) +
|
||||
formFieldHtml("API Key", '<input data-model-field="api_key" type="password" autocomplete="off" maxlength="300" placeholder="' + (item.configured ? "留空则保留已保存的 Key" : "输入 API Key") + '">', !item.configured) +
|
||||
'<div class="m-sys-test-row"><button class="m-btn-outline" type="button" data-system-test-model>测试连接</button>' +
|
||||
'<span data-model-test-status>未测试</span></div>' +
|
||||
'<div class="m-sys-sheet-actions">' +
|
||||
'<button class="m-btn-outline-danger" type="button" data-system-delete-model>删除模型</button>' +
|
||||
'<button class="m-btn-primary" type="button" data-system-save-model>保存</button>' +
|
||||
"</div></div>"
|
||||
);
|
||||
}
|
||||
|
||||
function readModelSheetFields(root) {
|
||||
function fieldValue(name) {
|
||||
const input = root.querySelector("[data-model-field='" + name + "']");
|
||||
return String(input && input.value != null ? input.value : "").trim();
|
||||
}
|
||||
return {
|
||||
name: fieldValue("name"),
|
||||
base_url: fieldValue("base_url"),
|
||||
model: fieldValue("model"),
|
||||
api_key: fieldValue("api_key")
|
||||
};
|
||||
}
|
||||
|
||||
function renderSystemMembers() {
|
||||
const payload = state.system.admin || {};
|
||||
const membership = payload.membership || {};
|
||||
const users = payload.users || [];
|
||||
const userHtml = users.map(function (user) {
|
||||
const admin = user.role === "admin";
|
||||
const member = Boolean(user.membership_subscribed);
|
||||
const badges = [];
|
||||
if (admin) badges.push('<span class="m-sys-badge m-sys-badge--admin">管理员</span>');
|
||||
if (member) badges.push('<span class="m-sys-badge m-sys-badge--ok">会员</span>');
|
||||
else badges.push('<span class="m-sys-badge">普通用户</span>');
|
||||
const expiry = member
|
||||
? (user.membership_expires_at ? "有效至 " + membershipDateLabel(user.membership_expires_at) : "永久有效")
|
||||
: user.membership_status === "suspended"
|
||||
? "会员已停用"
|
||||
: "尚未开通";
|
||||
return '<div class="m-sys-user-row" data-admin-user="' + number(user.id) + '">' +
|
||||
'<div class="m-sys-row-body"><strong>' + escapeHtml(user.username) + "</strong>" +
|
||||
'<div class="m-sys-badges">' + badges.join("") + "</div>" +
|
||||
'<p class="m-sys-hint">' + escapeHtml(expiry) + " · 今日已用 " + number(user.used_today) + " 次</p></div>" +
|
||||
'<button class="m-btn-outline" type="button" data-system-open-member>管理</button></div>';
|
||||
}).join("") || '<div class="m-state"><p>暂无注册用户</p></div>';
|
||||
const html =
|
||||
'<div class="m-sys-body" data-system-page="members">' +
|
||||
'<div class="m-card m-sys-section"><strong>全局额度</strong>' +
|
||||
formFieldHtml("会员每日智能分析上限", '<input id="m-sys-member-limit" type="number" min="1" max="1000" value="' + (number(membership.member_daily_limit) || 50) + '">', false) +
|
||||
'<p class="m-sys-hint">对所有会员生效,每日 0 点自动重置。</p>' +
|
||||
'<button class="m-btn-primary" type="button" data-system-save-limit>保存额度</button></div>' +
|
||||
'<div class="m-card m-sys-section"><strong>会员账号 · ' + users.length + " 个</strong>" +
|
||||
userHtml +
|
||||
'<p class="m-sys-hint">点「管理」为对应账号开通、续期或停用会员。</p></div>' +
|
||||
"</div>";
|
||||
systemFill(html);
|
||||
}
|
||||
|
||||
function openMemberManageSheet(userId) {
|
||||
const users = (state.system.admin && state.system.admin.users) || [];
|
||||
const user = users.find(function (item) { return String(item.id) === String(userId); });
|
||||
if (!user) return;
|
||||
state.system.editingUserId = String(userId);
|
||||
const member = Boolean(user.membership_subscribed);
|
||||
const expiry = member
|
||||
? (user.membership_expires_at ? "有效至 " + membershipDateLabel(user.membership_expires_at) : "永久有效")
|
||||
: user.membership_status === "suspended" ? "会员已停用" : "尚未开通";
|
||||
openSheet(
|
||||
'<div class="m-sheet-head"><h2>管理会员 · ' + escapeHtml(user.username) + "</h2>" +
|
||||
'<button class="m-sheet-close" type="button" data-sheet-close aria-label="关闭">' + icon("close", 20) + "</button></div>" +
|
||||
'<div class="m-sheet-body" data-admin-user="' + number(user.id) + '">' +
|
||||
'<p class="m-sys-lead">当前状态:' + escapeHtml(expiry) + " · 今日已用 " + number(user.used_today) + " 次</p>" +
|
||||
formFieldHtml("会员状态", '<select data-member-status>' +
|
||||
'<option value="inactive"' + (user.membership_status === "inactive" ? " selected" : "") + ">未开通</option>" +
|
||||
'<option value="active"' + (user.membership_status === "active" ? " selected" : "") + ">有效</option>" +
|
||||
'<option value="suspended"' + (user.membership_status === "suspended" ? " selected" : "") + ">停用</option>" +
|
||||
"</select>", false) +
|
||||
formFieldHtml("开通 / 续期时长", '<select data-member-duration><option value="">选择时长</option>' +
|
||||
'<option value="1_month">1个月</option><option value="3_months">3个月</option>' +
|
||||
'<option value="12_months">12个月</option><option value="3_years">3年</option>' +
|
||||
'<option value="permanent">永久</option></select>', false) +
|
||||
'<p class="m-sys-hint">从当前时间开始顺延;已有会员则叠加续期。</p>' +
|
||||
'<div class="m-sys-sheet-actions">' +
|
||||
'<button class="m-btn-outline" type="button" data-sheet-close>取消</button>' +
|
||||
'<button class="m-btn-primary" type="button" data-system-save-member>应用</button>' +
|
||||
"</div></div>"
|
||||
);
|
||||
}
|
||||
|
||||
function setFieldError(inputId, message) {
|
||||
@@ -5571,97 +5396,6 @@
|
||||
});
|
||||
}
|
||||
|
||||
function saveSystemModels() {
|
||||
const button = document.querySelector("[data-system-save-models]");
|
||||
if (button) button.disabled = true;
|
||||
global.MobileAPI.request("/api/admin/settings", "POST", {
|
||||
primary_model_id: (document.getElementById("m-sys-primary-model") || {}).value || "",
|
||||
fallback_model_id: (document.getElementById("m-sys-fallback-model") || {}).value || "",
|
||||
}).then(function () {
|
||||
showToast("模型分工已保存");
|
||||
loadSystem();
|
||||
}).catch(function (error) {
|
||||
showToast(error && error.message ? error.message : "模型分工保存失败");
|
||||
}).then(function () {
|
||||
if (button) button.disabled = false;
|
||||
});
|
||||
}
|
||||
|
||||
function persistSystemModels(models, message) {
|
||||
return global.MobileAPI.request("/api/admin/settings", "POST", { models: models }).then(function () {
|
||||
showToast(message || "模型池已保存");
|
||||
closeSheet();
|
||||
loadSystem();
|
||||
});
|
||||
}
|
||||
|
||||
function saveEditedSystemModel() {
|
||||
const sheet = document.querySelector(".m-sheet-body[data-model-id]");
|
||||
if (!sheet) return;
|
||||
const id = sheet.dataset.modelId;
|
||||
const fields = readModelSheetFields(sheet);
|
||||
const models = collectSystemModelPool().map(function (item) {
|
||||
if (item.id !== id) return item;
|
||||
return Object.assign({}, item, fields);
|
||||
});
|
||||
persistSystemModels(models, "模型已保存").catch(function (error) {
|
||||
showToast(error && error.message ? error.message : "模型保存失败");
|
||||
});
|
||||
}
|
||||
|
||||
function addSystemModel() {
|
||||
const models = collectSystemModelPool();
|
||||
const id = "model-" + Date.now() + "-" + Math.floor(Math.random() * 10000);
|
||||
const created = { id: id, name: "模型 " + (models.length + 1), base_url: "https://api.openai.com/v1", model: "", api_key: "", configured: false };
|
||||
state.system.models = models.concat([created]);
|
||||
const list = document.getElementById("m-sys-model-list");
|
||||
if (list) list.innerHTML = renderModelPoolHtml(state.system.models);
|
||||
updateSystemModelRoleOptions(id, (document.getElementById("m-sys-fallback-model") || {}).value || "");
|
||||
openModelEditSheet(id);
|
||||
}
|
||||
|
||||
function deleteSystemModel(row) {
|
||||
if (!row) return;
|
||||
const id = row.dataset.modelId;
|
||||
const primary = (document.getElementById("m-sys-primary-model") || {}).value;
|
||||
const fallback = (document.getElementById("m-sys-fallback-model") || {}).value;
|
||||
const llm = (state.system.admin && state.system.admin.llm) || {};
|
||||
if (id === primary || id === fallback || id === llm.primary_model_id || id === llm.fallback_model_id) {
|
||||
showToast("请先为主模型或辅助模型选择其他模型,再删除当前模型");
|
||||
return;
|
||||
}
|
||||
openConfirmSheet("删除模型?", "删除后该模型将从模型池移除,此操作不可恢复。", {
|
||||
danger: true,
|
||||
centered: true,
|
||||
confirmLabel: "删除",
|
||||
onConfirm: function () {
|
||||
const models = collectSystemModelPool().filter(function (item) { return item.id !== id; });
|
||||
persistSystemModels(models, "模型已删除").catch(function (error) {
|
||||
showToast(error && error.message ? error.message : "删除失败");
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function testSystemModel(row) {
|
||||
if (!row) return;
|
||||
const status = document.querySelector("[data-model-test-status]");
|
||||
const button = document.querySelector("[data-system-test-model]");
|
||||
const sheet = document.querySelector(".m-sheet-body[data-model-id]");
|
||||
const id = (sheet && sheet.dataset.modelId) || row.dataset.modelId;
|
||||
const fields = sheet ? readModelSheetFields(sheet) : {};
|
||||
const profile = Object.assign({}, collectSystemModelPool().find(function (item) { return item.id === id; }) || {}, fields, { id: id });
|
||||
if (button) button.disabled = true;
|
||||
if (status) status.textContent = "连接中";
|
||||
global.MobileAPI.request("/api/admin/settings/test", "POST", { model_id: id, profile: profile }).then(function (payload) {
|
||||
if (status) status.textContent = "上次测试:成功 · " + number(payload.result && payload.result.latency_ms) + "ms";
|
||||
}).catch(function (error) {
|
||||
if (status) status.textContent = error && error.message ? error.message : "测试失败";
|
||||
}).then(function () {
|
||||
if (button) button.disabled = false;
|
||||
});
|
||||
}
|
||||
|
||||
function startSystemRefresh() {
|
||||
const button = document.querySelector("[data-system-refresh]");
|
||||
if (button) button.disabled = true;
|
||||
@@ -5699,56 +5433,6 @@
|
||||
});
|
||||
}
|
||||
|
||||
function saveSystemMemberLimit() {
|
||||
const button = document.querySelector("[data-system-save-limit]");
|
||||
if (button) button.disabled = true;
|
||||
global.MobileAPI.request("/api/admin/settings", "POST", {
|
||||
member_daily_limit: number((document.getElementById("m-sys-member-limit") || {}).value),
|
||||
}).then(function () {
|
||||
showToast("会员调用额度已保存");
|
||||
loadSystem();
|
||||
}).catch(function (error) {
|
||||
showToast(error && error.message ? error.message : "会员调用额度保存失败");
|
||||
}).then(function () {
|
||||
if (button) button.disabled = false;
|
||||
});
|
||||
}
|
||||
|
||||
function saveSystemMember(card) {
|
||||
if (!card) return;
|
||||
const status = (card.querySelector("[data-member-status]") || {}).value;
|
||||
const apply = function () {
|
||||
const button = card.querySelector("[data-system-save-member]");
|
||||
if (button) button.disabled = true;
|
||||
global.MobileAPI.request("/api/admin/membership", "POST", {
|
||||
user_id: card.dataset.adminUser,
|
||||
status: status,
|
||||
duration: (card.querySelector("[data-member-duration]") || {}).value,
|
||||
}).then(function (payload) {
|
||||
if (state.system.admin) state.system.admin.users = payload.users || [];
|
||||
closeSheet();
|
||||
showToast("会员状态已更新");
|
||||
renderSystemMembers();
|
||||
}).catch(function (error) {
|
||||
showToast(error && error.message ? error.message : "会员状态保存失败");
|
||||
}).then(function () {
|
||||
if (button) button.disabled = false;
|
||||
});
|
||||
};
|
||||
if (status === "suspended") {
|
||||
openConfirmSheet("停用该会员?", "停用后该账号将无法使用会员智能功能,可稍后重新开通。", {
|
||||
danger: true,
|
||||
centered: true,
|
||||
confirmLabel: "停用",
|
||||
onConfirm: apply
|
||||
});
|
||||
return;
|
||||
}
|
||||
apply();
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------- events */
|
||||
|
||||
function bindEvents() {
|
||||
window.addEventListener("hashchange", closeSheet);
|
||||
|
||||
@@ -5876,12 +5560,6 @@
|
||||
return;
|
||||
}
|
||||
|
||||
const systemAdminTab = event.target.closest("[data-system-admin-tab]");
|
||||
if (systemAdminTab) {
|
||||
state.system.adminTab = systemAdminTab.dataset.systemAdminTab === "models" ? "models" : "market";
|
||||
renderSystemAdmin("system/admin");
|
||||
return;
|
||||
}
|
||||
if (event.target.closest("[data-system-save-birth]")) { saveSystemBirth(); return; }
|
||||
if (event.target.closest("[data-system-delete-birth]")) { deleteSystemBirth(); return; }
|
||||
if (event.target.closest("[data-system-save-password]")) { saveSystemPassword(); return; }
|
||||
@@ -5889,22 +5567,9 @@
|
||||
if (event.target.closest("[data-system-logout]")) { logoutSystemAccount(); return; }
|
||||
if (event.target.closest("[data-system-save-market]")) { saveSystemMarket(); return; }
|
||||
if (event.target.closest("[data-system-toggle-refresh]")) { toggleSystemRefresh(); return; }
|
||||
if (event.target.closest("[data-system-save-models]")) { saveSystemModels(); return; }
|
||||
if (event.target.closest("[data-system-save-model]")) { saveEditedSystemModel(); return; }
|
||||
if (event.target.closest("[data-system-add-model]")) { addSystemModel(); return; }
|
||||
const editModel = event.target.closest("[data-system-edit-model]");
|
||||
if (editModel) { openModelEditSheet(editModel.dataset.modelId); return; }
|
||||
const deleteModel = event.target.closest("[data-system-delete-model]");
|
||||
if (deleteModel) { deleteSystemModel(deleteModel.closest("[data-model-id]")); return; }
|
||||
const testModel = event.target.closest("[data-system-test-model]");
|
||||
if (testModel) { testSystemModel(testModel.closest("[data-model-id]")); return; }
|
||||
if (event.target.closest("[data-system-refresh]")) { startSystemRefresh(); return; }
|
||||
if (event.target.closest("[data-system-backfill]")) { startSystemBackfill(); return; }
|
||||
if (event.target.closest("[data-system-save-limit]")) { saveSystemMemberLimit(); return; }
|
||||
const openMember = event.target.closest("[data-system-open-member]");
|
||||
if (openMember) { openMemberManageSheet(openMember.closest("[data-admin-user]").dataset.adminUser); return; }
|
||||
const saveMember = event.target.closest("[data-system-save-member]");
|
||||
if (saveMember) { saveSystemMember(saveMember.closest("[data-admin-user]")); return; }
|
||||
if (event.target.closest("[data-system-datahub]")) { openDataHubConsole(); return; }
|
||||
|
||||
// 我的复盘:新增/编辑/删除/筛选/提交等操作
|
||||
const reviewAddTrade = event.target.closest("[data-review-add-trade]");
|
||||
|
||||
@@ -263,6 +263,7 @@
|
||||
'<label class="m-form-field"><span>账号名</span><input id="m-auth-username" type="text" minlength="3" maxlength="30" autocomplete="username" required></label>',
|
||||
'<label class="m-form-field"><span>密码</span><input id="m-auth-password" type="password" minlength="8" maxlength="128" autocomplete="current-password" required></label>',
|
||||
'<label class="m-form-field" id="m-auth-confirm-field" hidden><span>确认密码</span><input id="m-auth-confirm" type="password" minlength="8" maxlength="128" autocomplete="new-password"></label>',
|
||||
'<label class="m-form-field" id="m-auth-invite-field" hidden><span>邀请码</span><input id="m-auth-invite" type="text" maxlength="32" autocomplete="off" spellcheck="false" placeholder="XB-XXXX-XXXX-XXXX"></label>',
|
||||
'<p class="m-auth-error" id="m-auth-error" hidden></p>',
|
||||
'<button class="m-btn-primary" id="m-auth-submit" type="submit"><span class="m-btn-spinner" aria-hidden="true" hidden></span><span class="m-btn-label">登录</span></button>',
|
||||
"</form>",
|
||||
@@ -280,6 +281,8 @@
|
||||
});
|
||||
document.getElementById("m-auth-confirm-field").hidden = authMode !== "register";
|
||||
document.getElementById("m-auth-confirm").required = authMode === "register";
|
||||
document.getElementById("m-auth-invite-field").hidden = authMode !== "register";
|
||||
document.getElementById("m-auth-invite").required = authMode === "register";
|
||||
document.getElementById("m-auth-password").autocomplete = authMode === "register" ? "new-password" : "current-password";
|
||||
const submitLabel = document.querySelector("#m-auth-submit .m-btn-label");
|
||||
if (submitLabel) submitLabel.textContent = authMode === "register" ? "注册并进入" : "登录";
|
||||
@@ -346,7 +349,7 @@
|
||||
label.textContent = authMode === "register" ? "注册中…" : "登录中…";
|
||||
try {
|
||||
if (authMode === "register") {
|
||||
await global.MobileSession.register(username, password);
|
||||
await global.MobileSession.register(username, password, document.getElementById("m-auth-invite").value.trim());
|
||||
} else {
|
||||
await global.MobileSession.login(username, password);
|
||||
}
|
||||
|
||||
@@ -24,10 +24,11 @@
|
||||
return applySession(payload);
|
||||
}
|
||||
|
||||
async function register(username, password) {
|
||||
async function register(username, password, inviteCode) {
|
||||
const payload = await global.MobileAPI.request("/api/auth/register", "POST", {
|
||||
username: username,
|
||||
password: password
|
||||
password: password,
|
||||
invite_code: inviteCode || ""
|
||||
});
|
||||
return applySession(payload);
|
||||
}
|
||||
|
||||
Vendored
+1
-1
@@ -2065,7 +2065,7 @@ body[data-active-view="reviewWorkspaceView"] .workspace-view {
|
||||
}
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] :is(.global-search-results, .assistant-messages, .trade-log-form, .alerts-toolbar, .admin-section-picker) {
|
||||
:root[data-theme="dark"] :is(.global-search-results, .assistant-messages, .trade-log-form, .alerts-toolbar) {
|
||||
border-color: var(--line-soft);
|
||||
|
||||
background: var(--surface-muted);
|
||||
|
||||
+12
-188
@@ -39,21 +39,26 @@ async function openAdminSettings(refreshOnly = false) {
|
||||
const payload = await apiRequest("/api/admin/settings");
|
||||
const data = payload.data || {};
|
||||
const ifind = data.ifind || {};
|
||||
const llm = payload.llm || {};
|
||||
const membership = payload.membership || {};
|
||||
status.textContent = `数据中枢 ${data.configured ? "已连接" : "未连接"} · iFinD ${ifind.configured ? "已配置" : "未配置"} · ${number(data.snapshot_dates)} 个交易日`;
|
||||
status.classList.toggle("connected", Boolean(data.configured));
|
||||
setText("systemDataStatus", data.background_refresh_enabled ? "后台刷新已启用" : "后台刷新已暂停");
|
||||
renderDatahubRouteStatus(data.datahub || {});
|
||||
document.querySelector("#systemBackgroundRefresh").checked = Boolean(data.background_refresh_enabled);
|
||||
document.querySelector("#memberDailyLimit").value = number(membership.member_daily_limit) || 50;
|
||||
renderModelPool(llm.models || [], llm.primary_model_id || "", llm.fallback_model_id || "");
|
||||
renderAdminUsers(payload.users || []);
|
||||
} catch (error) {
|
||||
status.textContent = error.message || "系统配置读取失败";
|
||||
}
|
||||
}
|
||||
|
||||
function dataHubConsoleUrl() {
|
||||
if (window.XIAOBAI_DATAHUB_URL) return String(window.XIAOBAI_DATAHUB_URL);
|
||||
return `${window.location.protocol}//${window.location.hostname}:8766/admin/`;
|
||||
}
|
||||
|
||||
function openDataHubConsole() {
|
||||
if (state.user?.role !== "admin") return;
|
||||
window.open(dataHubConsoleUrl(), "_blank", "noopener");
|
||||
}
|
||||
|
||||
function renderDatahubRouteStatus(hub) {
|
||||
const box = document.querySelector("#datahubRouteStatus");
|
||||
if (!box) return;
|
||||
@@ -74,129 +79,6 @@ function renderDatahubRouteStatus(hub) {
|
||||
}
|
||||
}
|
||||
|
||||
function selectAdminPanel(panel) {
|
||||
const selected = ["market", "models", "members"].includes(panel) ? panel : "market";
|
||||
document.querySelector("#adminSectionSelect").value = selected;
|
||||
document.querySelectorAll("[data-admin-panel]").forEach((item) => {
|
||||
item.hidden = item.dataset.adminPanel !== selected;
|
||||
});
|
||||
}
|
||||
|
||||
function renderModelPool(models, primaryId = "", fallbackId = "") {
|
||||
state.adminModels = models.map((item) => ({ ...item, api_key: item.api_key || "" }));
|
||||
const container = document.querySelector("#modelPoolList");
|
||||
container.innerHTML = state.adminModels.map((item, index) => `
|
||||
<article class="model-pool-row" data-model-id="${escapeHtml(item.id)}">
|
||||
<div class="model-pool-heading"><strong>${escapeHtml(item.name || `模型 ${index + 1}`)}</strong><span>${item.configured ? "已保存密钥" : "待配置"}</span></div>
|
||||
<div class="model-pool-fields">
|
||||
<label class="form-field"><span>显示名称 *</span><input data-model-field="name" maxlength="50" value="${escapeHtml(item.name || "")}" required></label>
|
||||
<label class="form-field"><span>API Base URL *</span><input data-model-field="base_url" type="url" value="${escapeHtml(item.base_url || "https://api.openai.com/v1")}" required></label>
|
||||
<label class="form-field"><span>模型标识 *</span><input data-model-field="model" maxlength="100" value="${escapeHtml(item.model || "")}" required></label>
|
||||
<label class="form-field"><span>API Key${item.configured ? "" : " *"}</span><input data-model-field="api_key" type="password" autocomplete="off" maxlength="300" placeholder="${item.configured ? "留空保留已保存的 Key" : "输入 API Key"}" ${item.configured ? "" : "required"}></label>
|
||||
</div>
|
||||
<div class="model-test-row"><button class="button" type="button" data-test-model>测试连接</button><span class="model-test-status" aria-live="polite">未测试</span><button class="icon-button model-delete-button" type="button" data-delete-model aria-label="删除模型" title="删除模型"><i data-lucide="trash-2"></i></button></div>
|
||||
</article>
|
||||
`).join("") || emptyStateHtml("模型池为空,请先添加模型");
|
||||
updateModelRoleOptions(primaryId, fallbackId);
|
||||
container.querySelectorAll("[data-test-model]").forEach((button) => button.addEventListener("click", () => testPlatformModel(button.closest("[data-model-id]"))));
|
||||
container.querySelectorAll("[data-delete-model]").forEach((button) => button.addEventListener("click", () => deletePlatformModel(button.closest("[data-model-id]"))));
|
||||
container.querySelectorAll("[data-model-field='name']").forEach((input) => input.addEventListener("input", updateModelRoleLabels));
|
||||
refreshIcons();
|
||||
}
|
||||
|
||||
function collectModelPool() {
|
||||
const saved = new Map(state.adminModels.map((item) => [item.id, item]));
|
||||
return [...document.querySelectorAll("#modelPoolList [data-model-id]")].map((row) => ({
|
||||
id: row.dataset.modelId,
|
||||
name: row.querySelector("[data-model-field='name']").value.trim(),
|
||||
base_url: row.querySelector("[data-model-field='base_url']").value.trim(),
|
||||
model: row.querySelector("[data-model-field='model']").value.trim(),
|
||||
api_key: row.querySelector("[data-model-field='api_key']").value.trim(),
|
||||
configured: Boolean(saved.get(row.dataset.modelId)?.configured),
|
||||
}));
|
||||
}
|
||||
|
||||
function updateModelRoleOptions(primaryId = document.querySelector("#platformPrimaryModelSelect").value, fallbackId = document.querySelector("#platformFallbackModelSelect").value) {
|
||||
const models = collectModelPool();
|
||||
const options = models.map((item) => `<option value="${escapeHtml(item.id)}">${escapeHtml(item.name || item.model || "未命名模型")}</option>`).join("");
|
||||
const primary = document.querySelector("#platformPrimaryModelSelect");
|
||||
const fallback = document.querySelector("#platformFallbackModelSelect");
|
||||
primary.innerHTML = models.length ? options : '<option value="">暂无模型</option>';
|
||||
fallback.innerHTML = `<option value="">不启用辅助模型</option>${options}`;
|
||||
primary.value = models.some((item) => item.id === primaryId) ? primaryId : models[0]?.id || "";
|
||||
fallback.value = models.some((item) => item.id === fallbackId) && fallbackId !== primary.value ? fallbackId : "";
|
||||
}
|
||||
|
||||
function updateModelRoleLabels() {
|
||||
updateModelRoleOptions();
|
||||
}
|
||||
|
||||
function addPlatformModel() {
|
||||
const models = collectModelPool();
|
||||
const id = `model-${Date.now()}-${Math.floor(Math.random() * 10000)}`;
|
||||
models.push({ id, name: `模型 ${models.length + 1}`, base_url: "https://api.openai.com/v1", model: "", api_key: "", configured: false });
|
||||
renderModelPool(models, document.querySelector("#platformPrimaryModelSelect").value || id, document.querySelector("#platformFallbackModelSelect").value);
|
||||
document.querySelector(`[data-model-id="${CSS.escape(id)}"] [data-model-field="name"]`)?.focus();
|
||||
}
|
||||
|
||||
function deletePlatformModel(row) {
|
||||
if (!row) return;
|
||||
const id = row.dataset.modelId;
|
||||
const primary = document.querySelector("#platformPrimaryModelSelect").value;
|
||||
const fallback = document.querySelector("#platformFallbackModelSelect").value;
|
||||
if (id === primary || id === fallback) {
|
||||
showToast("请先为主模型或辅助模型选择其他模型,再删除当前模型");
|
||||
return;
|
||||
}
|
||||
const models = collectModelPool().filter((item) => item.id !== id);
|
||||
renderModelPool(models, primary, fallback);
|
||||
}
|
||||
|
||||
function renderAdminUsers(users) {
|
||||
const container = document.querySelector("#adminUsersList");
|
||||
container.innerHTML = users.map((user) => {
|
||||
const admin = user.role === "admin";
|
||||
const member = Boolean(user.membership_subscribed);
|
||||
const identityLabels = [admin ? "管理员" : "", member ? "会员有效" : "普通用户"].filter(Boolean).join(" · ");
|
||||
const expiry = member
|
||||
? (user.membership_expires_at ? `有效至 ${membershipDateDisplay(user.membership_expires_at)}` : "永久有效")
|
||||
: user.membership_status === "suspended"
|
||||
? "会员已停用"
|
||||
: user.membership_status === "active" && user.membership_expires_at
|
||||
? `已于 ${membershipDateDisplay(user.membership_expires_at)} 到期`
|
||||
: "尚未开通";
|
||||
return `<article class="admin-user-row" data-admin-user="${number(user.id)}">
|
||||
<div class="admin-user-identity"><strong>${escapeHtml(user.username)}</strong><span>${escapeHtml(identityLabels)}</span><small>${escapeHtml(expiry)}</small></div>
|
||||
<div class="admin-user-usage">今日调用 <b>${number(user.used_today)}</b></div>
|
||||
<form class="membership-form">
|
||||
<input type="hidden" name="user_id" value="${number(user.id)}">
|
||||
<label><span>状态</span><select name="status"><option value="inactive" ${user.membership_status === "inactive" ? "selected" : ""}>未开通</option><option value="active" ${user.membership_status === "active" ? "selected" : ""}>有效</option><option value="suspended" ${user.membership_status === "suspended" ? "selected" : ""}>停用</option></select></label>
|
||||
<label><span>开通 / 续期时长</span><select name="duration"><option value="">选择时长</option><option value="1_month">1个月</option><option value="3_months">3个月</option><option value="12_months">12个月</option><option value="3_years">3年</option><option value="permanent">永久</option></select></label>
|
||||
<div class="membership-expiry"><span>当前到期</span><strong>${escapeHtml(expiry)}</strong></div>
|
||||
<button class="button" type="submit">应用</button>
|
||||
</form>
|
||||
</article>`;
|
||||
}).join("") || emptyStateHtml("暂无注册用户");
|
||||
container.querySelectorAll(".membership-form").forEach((form) => form.addEventListener("submit", saveMembership));
|
||||
}
|
||||
|
||||
async function saveMembership(event) {
|
||||
event.preventDefault();
|
||||
const form = event.currentTarget;
|
||||
const data = Object.fromEntries(new FormData(form).entries());
|
||||
const button = form.querySelector("button[type='submit']");
|
||||
button.disabled = true;
|
||||
try {
|
||||
const payload = await apiRequest("/api/admin/membership", "POST", data);
|
||||
renderAdminUsers(payload.users || []);
|
||||
showToast("会员状态已更新");
|
||||
} catch (error) {
|
||||
showToast(error.message || "会员状态保存失败");
|
||||
} finally {
|
||||
button.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function saveMarketSettings(event) {
|
||||
event.preventDefault();
|
||||
const button = event.currentTarget.querySelector("button[type='submit']");
|
||||
@@ -214,70 +96,12 @@ async function saveMarketSettings(event) {
|
||||
}
|
||||
}
|
||||
|
||||
async function saveModelPool(event) {
|
||||
event.preventDefault();
|
||||
const button = event.currentTarget.querySelector("button[type='submit']");
|
||||
button.disabled = true;
|
||||
try {
|
||||
await apiRequest("/api/admin/settings", "POST", {
|
||||
models: collectModelPool(),
|
||||
primary_model_id: document.querySelector("#platformPrimaryModelSelect").value,
|
||||
fallback_model_id: document.querySelector("#platformFallbackModelSelect").value,
|
||||
});
|
||||
showToast("模型池已保存");
|
||||
await openAdminSettings(true);
|
||||
} catch (error) {
|
||||
showToast(error.message || "模型池保存失败");
|
||||
} finally {
|
||||
button.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function saveMembershipSettings(event) {
|
||||
event.preventDefault();
|
||||
const button = event.currentTarget.querySelector("button[type='submit']");
|
||||
button.disabled = true;
|
||||
try {
|
||||
await apiRequest("/api/admin/settings", "POST", {
|
||||
member_daily_limit: number(document.querySelector("#memberDailyLimit").value),
|
||||
});
|
||||
showToast("会员调用额度已保存");
|
||||
await openAdminSettings(true);
|
||||
} catch (error) {
|
||||
showToast(error.message || "会员调用额度保存失败");
|
||||
} finally {
|
||||
button.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function testPlatformModel(row) {
|
||||
if (!row) return;
|
||||
const button = row.querySelector("[data-test-model]");
|
||||
const status = row.querySelector(".model-test-status");
|
||||
const profile = collectModelPool().find((item) => item.id === row.dataset.modelId) || {};
|
||||
button.disabled = true;
|
||||
status.textContent = "连接中";
|
||||
try {
|
||||
const payload = await apiRequest("/api/admin/settings/test", "POST", { model_id: row.dataset.modelId, profile });
|
||||
status.textContent = `已连通 · ${number(payload.result.latency_ms)} ms`;
|
||||
status.className = "model-test-status success";
|
||||
} catch (error) {
|
||||
status.textContent = error.message;
|
||||
status.className = "model-test-status failure";
|
||||
} finally {
|
||||
button.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function bindAdminEvents() {
|
||||
document.querySelector("#settingsButton").addEventListener("click", () => openAdminSettings());
|
||||
document.querySelector("#settingsButton").addEventListener("click", openDataHubConsole);
|
||||
document.querySelector("#marketAdminButton").addEventListener("click", () => openAdminSettings());
|
||||
document.querySelector("#closeAdminDialog").addEventListener("click", () => elements.adminDialog.close());
|
||||
document.querySelector("#backfillButton").addEventListener("click", backfillData);
|
||||
document.querySelector("#adminSectionSelect").addEventListener("change", (event) => selectAdminPanel(event.target.value));
|
||||
document.querySelector("#systemMarketForm").addEventListener("submit", saveMarketSettings);
|
||||
document.querySelector("#systemModelsForm").addEventListener("submit", saveModelPool);
|
||||
document.querySelector("#membershipSettingsForm").addEventListener("submit", saveMembershipSettings);
|
||||
document.querySelector("#addPlatformModel").addEventListener("click", addPlatformModel);
|
||||
document.querySelector("#adminRefreshButton").addEventListener("click", startAdminRefresh);
|
||||
}
|
||||
|
||||
@@ -125,50 +125,6 @@
|
||||
width: min(1060px, -24px + 100vw);
|
||||
}
|
||||
|
||||
.admin-section-picker {
|
||||
display: grid;
|
||||
|
||||
grid-template-columns: 96px minmax(220px, 360px);
|
||||
|
||||
align-items: center;
|
||||
|
||||
gap: 12px;
|
||||
|
||||
padding: 16px 18px;
|
||||
}
|
||||
|
||||
.admin-section-picker label {
|
||||
color: var(--text-secondary);
|
||||
|
||||
font-size: 13px;
|
||||
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.admin-section-picker select {
|
||||
width: 100%;
|
||||
|
||||
min-height: 32px;
|
||||
|
||||
padding: 0px 11px;
|
||||
|
||||
border: 1px solid var(--border-strong);
|
||||
|
||||
border-radius: 8px;
|
||||
|
||||
background: var(--surface);
|
||||
|
||||
color: var(--text-primary);
|
||||
|
||||
font-size: var(--font-size-label);
|
||||
}
|
||||
|
||||
.admin-section-picker select:focus-visible {
|
||||
outline: 2px solid var(--action);
|
||||
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
.admin-panel[hidden] {
|
||||
display: none;
|
||||
}
|
||||
@@ -220,161 +176,6 @@
|
||||
font-weight: 650;
|
||||
}
|
||||
|
||||
.compact-number-field {
|
||||
width: min(260px, 100%);
|
||||
|
||||
margin-top: 14px;
|
||||
}
|
||||
|
||||
.admin-users-list {
|
||||
display: grid;
|
||||
|
||||
border-top: 1px solid var(--line);
|
||||
}
|
||||
|
||||
.admin-user-row {
|
||||
min-width: 0px;
|
||||
|
||||
display: grid;
|
||||
|
||||
grid-template-columns: 160px 100px minmax(0px, 1fr);
|
||||
|
||||
align-items: center;
|
||||
|
||||
gap: 14px;
|
||||
|
||||
padding: 13px 0px;
|
||||
|
||||
border-bottom: 1px solid var(--line);
|
||||
}
|
||||
|
||||
.admin-user-identity strong {
|
||||
display: block;
|
||||
|
||||
color: var(--text-primary);
|
||||
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.admin-user-identity small,
|
||||
.admin-user-identity span {
|
||||
display: block;
|
||||
|
||||
margin-top: 4px;
|
||||
|
||||
color: var(--text-muted);
|
||||
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.admin-user-usage {
|
||||
margin-top: 4px;
|
||||
|
||||
color: var(--text-muted);
|
||||
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.membership-form {
|
||||
padding: 0px;
|
||||
|
||||
min-width: 0px;
|
||||
|
||||
display: grid;
|
||||
|
||||
grid-template-columns: 100px minmax(170px, 0.8fr) minmax(160px, 1fr) auto;
|
||||
|
||||
align-items: end;
|
||||
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.membership-form label {
|
||||
min-width: 0px;
|
||||
|
||||
display: grid;
|
||||
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
.membership-form label span {
|
||||
color: var(--text-muted);
|
||||
|
||||
font-size: 10px;
|
||||
}
|
||||
|
||||
.membership-form input,
|
||||
.membership-form select {
|
||||
width: 100%;
|
||||
|
||||
min-width: 0px;
|
||||
|
||||
height: 32px;
|
||||
|
||||
padding: 0px 8px;
|
||||
|
||||
border: 1px solid var(--border-strong);
|
||||
|
||||
border-radius: 8px;
|
||||
|
||||
background: var(--surface);
|
||||
}
|
||||
|
||||
.membership-expiry {
|
||||
min-width: 0px;
|
||||
|
||||
display: grid;
|
||||
|
||||
gap: 5px;
|
||||
|
||||
align-self: end;
|
||||
}
|
||||
|
||||
.membership-expiry span {
|
||||
color: var(--text-muted);
|
||||
|
||||
font-size: 10px;
|
||||
}
|
||||
|
||||
.membership-expiry strong {
|
||||
min-height: 34px;
|
||||
|
||||
display: flex;
|
||||
|
||||
align-items: center;
|
||||
|
||||
color: var(--text-secondary);
|
||||
|
||||
font-size: 12px;
|
||||
|
||||
font-weight: 650;
|
||||
}
|
||||
|
||||
@media (max-width: 820px) {
|
||||
.admin-user-row {
|
||||
grid-template-columns: 1fr auto;
|
||||
}
|
||||
|
||||
.admin-user-row .membership-form {
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
|
||||
.membership-form {
|
||||
grid-template-columns: repeat(2, minmax(0px, 1fr));
|
||||
}
|
||||
|
||||
.membership-form .button {
|
||||
align-self: end;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
|
||||
.membership-form {
|
||||
grid-template-columns: minmax(0px, 1fr);
|
||||
}
|
||||
}
|
||||
|
||||
.connection-status {
|
||||
margin: 18px 18px 0px;
|
||||
|
||||
@@ -399,7 +200,6 @@
|
||||
padding: 18px;
|
||||
}
|
||||
|
||||
.settings-dialog .membership-form,
|
||||
.settings-dialog .settings-section > form,
|
||||
.settings-dialog.admin-dialog > form {
|
||||
padding: 0px;
|
||||
@@ -559,7 +359,6 @@
|
||||
padding-left: 12px;
|
||||
}
|
||||
|
||||
.settings-dialog .membership-form,
|
||||
.settings-dialog .settings-section > form,
|
||||
.settings-dialog.admin-dialog > form {
|
||||
padding: 0px;
|
||||
@@ -1173,12 +972,6 @@ button.account-role-badge:focus-visible {
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
.admin-section-picker {
|
||||
grid-template-columns: minmax(0px, 1fr);
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 520px) {
|
||||
.settings-dialog .form-field {
|
||||
width: 100%;
|
||||
@@ -1520,32 +1313,6 @@ button.account-role-badge:focus-visible {
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.admin-dialog .admin-section-picker {
|
||||
grid-template-columns: auto minmax(220px, 300px);
|
||||
|
||||
gap: 14px;
|
||||
|
||||
padding: 12px 20px;
|
||||
|
||||
border-bottom: 1px solid var(--dialog-line);
|
||||
|
||||
background: var(--surface-subtle);
|
||||
}
|
||||
|
||||
.admin-dialog .admin-section-picker label {
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.admin-dialog .admin-section-picker select {
|
||||
min-height: 32px;
|
||||
|
||||
border-color: var(--border-strong);
|
||||
|
||||
border-radius: 8px;
|
||||
|
||||
font-size: var(--font-size-label);
|
||||
}
|
||||
|
||||
.admin-dialog .admin-panel {
|
||||
max-width: 860px;
|
||||
|
||||
@@ -1556,24 +1323,6 @@ button.account-role-badge:focus-visible {
|
||||
border-top: 1px solid var(--dialog-line);
|
||||
}
|
||||
|
||||
.admin-dialog .admin-users-list {
|
||||
border-color: var(--dialog-line);
|
||||
}
|
||||
|
||||
.admin-dialog .admin-user-row {
|
||||
grid-template-columns: 150px 90px minmax(0px, 1fr);
|
||||
|
||||
gap: 12px;
|
||||
|
||||
padding: 12px 0px;
|
||||
|
||||
border-color: var(--dialog-line);
|
||||
}
|
||||
|
||||
.admin-dialog .membership-form {
|
||||
grid-template-columns: 92px minmax(135px, 0.8fr) minmax(140px, 1fr) auto;
|
||||
}
|
||||
|
||||
@media (max-width: 760px) {
|
||||
.account-settings-dialog .membership-status-grid {
|
||||
grid-template-columns: repeat(2, minmax(0px, 1fr));
|
||||
@@ -1582,22 +1331,6 @@ button.account-role-badge:focus-visible {
|
||||
.account-settings-dialog .account-birth-form {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.admin-dialog .admin-section-picker {
|
||||
grid-template-columns: 1fr;
|
||||
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.admin-dialog .admin-user-row {
|
||||
grid-template-columns: 1fr auto;
|
||||
}
|
||||
|
||||
.admin-dialog .admin-user-row .membership-form {
|
||||
grid-column: 1 / -1;
|
||||
|
||||
grid-template-columns: repeat(2, minmax(0px, 1fr));
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 460px) {
|
||||
@@ -1612,10 +1345,6 @@ button.account-role-badge:focus-visible {
|
||||
.account-settings-dialog .membership-comparison > div {
|
||||
min-width: 510px;
|
||||
}
|
||||
|
||||
.admin-dialog .admin-user-row .membership-form {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] :is(.header-command-group, .account-dropdown) {
|
||||
|
||||
@@ -1028,64 +1028,6 @@
|
||||
vertical-align: -1px;
|
||||
}
|
||||
|
||||
.model {
|
||||
padding: 12px 14px;
|
||||
|
||||
border-bottom: 1px solid var(--line-soft);
|
||||
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.model:hover {
|
||||
background: var(--table-hover);
|
||||
}
|
||||
|
||||
.model .r1 {
|
||||
display: flex;
|
||||
|
||||
align-items: center;
|
||||
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.model .r1 .nm {
|
||||
font-weight: 700;
|
||||
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.model .r1 .lv {
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.model .r2 {
|
||||
font-size: 11.5px;
|
||||
|
||||
color: var(--sub);
|
||||
|
||||
margin-top: 4px;
|
||||
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.model .r3 {
|
||||
display: flex;
|
||||
|
||||
gap: 10px;
|
||||
|
||||
margin-top: 6px;
|
||||
|
||||
font-size: 10.5px;
|
||||
|
||||
color: var(--faint);
|
||||
}
|
||||
|
||||
.model .r3 .q {
|
||||
border-bottom: 1px dashed var(--faint);
|
||||
|
||||
cursor: help;
|
||||
}
|
||||
|
||||
:root[data-theme="dark"] :is(.card-h, .performance-stage-card header, .rotation-card-head, .auction-card-head-v2, .theme-card-head-v2, .dragon-detail-header, .screener-card-head, .mentor-page-header) {
|
||||
border-color: var(--line-soft);
|
||||
|
||||
|
||||
@@ -29,22 +29,6 @@
|
||||
margin-top: 22px;
|
||||
}
|
||||
|
||||
.admin-dialog .model-role-selectors {
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.admin-dialog .model-pool-list {
|
||||
margin-top: 14px;
|
||||
}
|
||||
|
||||
.admin-dialog .model-row {
|
||||
border-color: var(--border);
|
||||
|
||||
border-radius: 8px;
|
||||
|
||||
background: var(--surface-subtle);
|
||||
}
|
||||
|
||||
@keyframes overlay-enter {
|
||||
0% {
|
||||
opacity: 0;
|
||||
|
||||
@@ -6,6 +6,8 @@ function selectAuthMode(mode) {
|
||||
const registering = state.authMode === "register";
|
||||
document.querySelector("#authConfirmField").hidden = !registering;
|
||||
document.querySelector("#authPasswordConfirm").required = registering;
|
||||
document.querySelector("#authInviteField").hidden = !registering;
|
||||
document.querySelector("#authInviteCode").required = registering;
|
||||
document.querySelector("#authPassword").autocomplete = registering ? "new-password" : "current-password";
|
||||
document.querySelector("#authSubmitButton").textContent = registering ? "注册并进入" : "登录";
|
||||
document.querySelector("#authError").hidden = true;
|
||||
@@ -24,7 +26,11 @@ async function submitAuthForm(event) {
|
||||
const button = document.querySelector("#authSubmitButton");
|
||||
button.disabled = true;
|
||||
try {
|
||||
const session = await apiRequest(`/api/auth/${state.authMode}`, "POST", { username, password });
|
||||
const registering = state.authMode === "register";
|
||||
const credentials = registering
|
||||
? { username, password, invite_code: document.querySelector("#authInviteCode").value.trim() }
|
||||
: { username, password };
|
||||
const session = await apiRequest(`/api/auth/${state.authMode}`, "POST", credentials);
|
||||
document.querySelector("#authForm").reset();
|
||||
await applyAuthenticatedSession(session);
|
||||
} catch (error) {
|
||||
@@ -42,6 +48,7 @@ async function applyAuthenticatedSession(session) {
|
||||
const isAdmin = session.user?.role === "admin";
|
||||
updateAccountIdentityBadges(session.user?.membership || {});
|
||||
document.querySelector("#settingsButton").hidden = !isAdmin;
|
||||
document.querySelector("#marketAdminButton").hidden = !isAdmin;
|
||||
document.querySelector("#syncButton").hidden = !isAdmin;
|
||||
document.querySelector("#reasonForm").hidden = !isAdmin;
|
||||
document.querySelector("#sectorPhaseManager").hidden = !isAdmin;
|
||||
|
||||
@@ -632,6 +632,7 @@ test("admin shell opens every primary workspace and global search", async ({ pag
|
||||
await expect(page.locator("#authGate")).toBeHidden();
|
||||
await openHeaderCommandMenu(page);
|
||||
await expect(page.locator("#settingsButton")).toBeVisible();
|
||||
await expect(page.locator("#marketAdminButton")).toBeVisible();
|
||||
await expect(page.locator("#syncButton")).toBeVisible();
|
||||
await page.keyboard.press("Escape");
|
||||
await page.locator("#alertButton").click();
|
||||
@@ -1753,6 +1754,7 @@ test("regular account cannot see admin controls and member features are gated",
|
||||
await mockApplication(page, session("user", false));
|
||||
await page.goto("/index.html");
|
||||
await expect(page.locator("#settingsButton")).toBeHidden();
|
||||
await expect(page.locator("#marketAdminButton")).toBeHidden();
|
||||
await expect(page.locator("#syncButton")).toBeHidden();
|
||||
await expect(page.locator("#accountVipLabel")).toHaveText("非会员");
|
||||
await page.locator('[data-view="screenerView"]').first().click();
|
||||
@@ -3332,9 +3334,9 @@ test("global dialogs share the stage 18 geometry without changing account or adm
|
||||
await page.locator("#closeSettingsDialog").click();
|
||||
|
||||
await openHeaderCommandMenu(page);
|
||||
await page.locator("#settingsButton").click();
|
||||
await page.locator("#marketAdminButton").click();
|
||||
await expect(page.locator("#adminDialog")).toHaveAttribute("aria-labelledby", "adminDialogTitle");
|
||||
await expect(page.locator("#adminSectionSelect")).toBeVisible();
|
||||
await expect(page.locator("#systemMarketForm")).toBeVisible();
|
||||
const adminBox = await page.locator("#adminDialog").boundingBox();
|
||||
expect(adminBox.width).toBeLessThanOrEqual(902);
|
||||
expect(Math.abs(adminBox.x + adminBox.width / 2 - 720)).toBeLessThanOrEqual(2);
|
||||
@@ -3510,7 +3512,7 @@ test("B-199 screener review and account surfaces fit day night viewports", async
|
||||
await page.locator("#closeSettingsDialog").click();
|
||||
|
||||
await openHeaderCommandMenu(page);
|
||||
await page.locator("#settingsButton").click();
|
||||
await page.locator("#marketAdminButton").click();
|
||||
await expect(page.locator("#adminDialog")).toBeVisible();
|
||||
await shot("admin-1600-day");
|
||||
await page.locator("#closeAdminDialog").click();
|
||||
@@ -3609,6 +3611,7 @@ test("desktop header keeps refresh, admin commands and account identity visible"
|
||||
await expect(page.locator("#refreshButton")).toBeVisible();
|
||||
await expect(page.locator("#syncButton")).toBeVisible();
|
||||
await expect(page.locator("#settingsButton")).toBeVisible();
|
||||
await expect(page.locator("#marketAdminButton")).toBeVisible();
|
||||
await expect(page.locator("#accountAdminBadge")).toBeVisible();
|
||||
await expect(page.locator("#accountVipBadge")).toBeVisible();
|
||||
await expect(page.locator("#accountButton")).toBeVisible();
|
||||
@@ -3621,7 +3624,7 @@ test("desktop header keeps refresh, admin commands and account identity visible"
|
||||
expect(geometry.nameFits, `${viewport.width} account name truncated`).toBe(true);
|
||||
await page.locator("#refreshButton").click();
|
||||
await expect(page.locator("#loadingOverlay")).toBeHidden();
|
||||
await page.locator("#settingsButton").click();
|
||||
await page.locator("#marketAdminButton").click();
|
||||
await expect(page.locator("#adminDialog")).toBeVisible();
|
||||
await page.locator("#closeAdminDialog").click();
|
||||
await page.locator("#accountButton").click();
|
||||
@@ -3668,6 +3671,7 @@ test("desktop header keeps refresh, admin commands and account identity visible"
|
||||
await expect(page.locator("#refreshButton")).toBeVisible();
|
||||
await expect(page.locator("#syncButton")).toBeHidden();
|
||||
await expect(page.locator("#settingsButton")).toBeHidden();
|
||||
await expect(page.locator("#marketAdminButton")).toBeHidden();
|
||||
await expect(page.locator("#accountAdminBadge")).toBeHidden();
|
||||
await expect(page.locator("#accountVipBadge")).toBeVisible();
|
||||
await expect(page.locator("#accountName")).toHaveText("normal_user");
|
||||
|
||||
@@ -264,8 +264,7 @@ const SYSTEM_PAGES = [
|
||||
["system/profile", "账号资料"],
|
||||
["system/password", "修改密码"],
|
||||
["system/membership", "会员状态"],
|
||||
["system/admin", "系统设置"],
|
||||
["system/members", "会员管理"],
|
||||
["system/admin", "行情管理"],
|
||||
];
|
||||
|
||||
const PLACEHOLDER_COPY = "该功能页将在后续批次实现";
|
||||
@@ -360,8 +359,7 @@ test("system management pages render real content instead of placeholders", asyn
|
||||
await expect(page.locator(".m-sys-grid")).toBeVisible();
|
||||
await navigateToFeature(page, "system/admin");
|
||||
await expect(page.locator("#m-sys-token")).toBeVisible();
|
||||
await navigateToFeature(page, "system/members");
|
||||
await expect(page.locator("#m-sys-member-limit")).toBeVisible();
|
||||
await expect(page.locator("[data-system-datahub]")).toBeVisible();
|
||||
});
|
||||
|
||||
test("system home groups entries and keeps admin-only items gated", async ({ page }) => {
|
||||
@@ -403,10 +401,7 @@ test("system settings tabs, model editor, delete confirm and theme toggle work",
|
||||
await page.locator("[data-theme-toggle]").click();
|
||||
await expect.poll(async () => page.locator("#m-app").getAttribute("data-theme")).not.toBe(before);
|
||||
|
||||
await navigateToFeature(page, "system/members");
|
||||
await page.locator("[data-system-open-member]").click();
|
||||
await expect(page.locator(".m-sheet-root.is-open")).toBeVisible();
|
||||
await expect(page.locator(".m-sheet-head h2")).toContainText("管理会员");
|
||||
await expect(page.locator("[data-system-datahub]").first()).toBeVisible();
|
||||
});
|
||||
|
||||
test("password mismatch shows inline error instead of a silent submit", async ({ page }) => {
|
||||
@@ -439,7 +434,7 @@ test("non-admin cannot open system admin pages as placeholders", async ({ page }
|
||||
await page.evaluate(() => { window.MobileRouter.navigate("#/hub/system"); });
|
||||
await expect(page.locator("[data-system-page='home']")).toBeVisible();
|
||||
await expect(page.locator('[data-route="#/feature/system/admin"]')).toHaveCount(0);
|
||||
await expect(page.locator('[data-route="#/feature/system/members"]')).toHaveCount(0);
|
||||
await expect(page.locator("[data-system-datahub]")).toHaveCount(0);
|
||||
await expect(page.locator("[data-system-switch]")).toBeVisible();
|
||||
await navigateToFeature(page, "system/admin");
|
||||
await expect(page.locator("#m-view")).not.toContainText(PLACEHOLDER_COPY);
|
||||
|
||||
@@ -35,7 +35,12 @@ class AccountSwitchGrantTests(unittest.TestCase):
|
||||
self.bound_user_id = int(user_id)
|
||||
|
||||
def _register(self, username: str, device_hash: str = "") -> dict:
|
||||
return self.service.register(username, "Password123", device_hash or self.device_a)
|
||||
invite = ""
|
||||
if self.database.count_users():
|
||||
invite = self.service.generate_invite_codes(1)[0]["code"]
|
||||
return self.service.register(
|
||||
username, "Password123", device_hash or self.device_a, invite
|
||||
)
|
||||
|
||||
def test_login_records_accounts_for_the_current_device_only(self) -> None:
|
||||
first = self._register("alpha_user")
|
||||
|
||||
@@ -37,6 +37,10 @@ MODULE_STYLESHEETS = (
|
||||
"pages/review/foundation.css",
|
||||
)
|
||||
|
||||
# 问天是冻结区:即便某条规则在当前运行时里已经没人引用,也不在这里做陈旧清理,
|
||||
# 免得为了让门禁变绿去动冻结代码。其余门禁(归属唯一、无空声明等)照常覆盖它。
|
||||
FROZEN_STYLESHEETS = ("pages/heaven/foundation.css",)
|
||||
|
||||
RETIRED_STYLESHEETS = (
|
||||
"styles/styles.css",
|
||||
"styles/renovation.css",
|
||||
@@ -559,6 +563,8 @@ class CssGovernanceTests(unittest.TestCase):
|
||||
def test_every_selector_has_a_runtime_consumer(self) -> None:
|
||||
stale: list[str] = []
|
||||
for relative, keys in self.rule_keys.items():
|
||||
if relative in FROZEN_STYLESHEETS:
|
||||
continue
|
||||
for contexts, selector in keys:
|
||||
if selector.lower().startswith(("@keyframes", "@-webkit-keyframes")):
|
||||
continue
|
||||
|
||||
@@ -26,6 +26,7 @@ class DatabaseMigrationTests(unittest.TestCase):
|
||||
("0003", "extend_llm_audit"),
|
||||
("0004", "add_mentor_note"),
|
||||
("0005", "create_account_switch_grants"),
|
||||
("0006", "create_invite_codes"),
|
||||
],
|
||||
)
|
||||
columns = {
|
||||
@@ -40,7 +41,7 @@ class DatabaseMigrationTests(unittest.TestCase):
|
||||
count = connection.execute(
|
||||
"SELECT COUNT(*) AS count FROM schema_migrations"
|
||||
).fetchone()["count"]
|
||||
self.assertEqual(count, 5)
|
||||
self.assertEqual(count, 6)
|
||||
|
||||
def test_database_with_recorded_0004_and_note_column_starts_without_reapply(
|
||||
self,
|
||||
@@ -61,7 +62,7 @@ class DatabaseMigrationTests(unittest.TestCase):
|
||||
count = connection.execute(
|
||||
"SELECT COUNT(*) AS count FROM schema_migrations"
|
||||
).fetchone()["count"]
|
||||
self.assertEqual(count, 5)
|
||||
self.assertEqual(count, 6)
|
||||
|
||||
def test_old_database_without_0004_upgrades_and_adds_note_column(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
@@ -88,7 +89,7 @@ class DatabaseMigrationTests(unittest.TestCase):
|
||||
"PRAGMA table_info(mentor_preferences)"
|
||||
)
|
||||
]
|
||||
self.assertEqual(versions, {"0001", "0002", "0003", "0004", "0005"})
|
||||
self.assertEqual(versions, {"0001", "0002", "0003", "0004", "0005", "0006"})
|
||||
self.assertIn("note", note_rows)
|
||||
|
||||
def test_database_with_unknown_migration_is_rejected(self) -> None:
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from backend.application import HUB_SERVICE_HANDLERS, RequestHandler
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SERVICE_TOKEN = "hub-admin-token-for-tests"
|
||||
|
||||
|
||||
class FakeHeaders(dict):
|
||||
def get(self, name, default=""): # type: ignore[override]
|
||||
return super().get(name, default)
|
||||
|
||||
|
||||
class HubAdminBridgeTests(unittest.TestCase):
|
||||
def handler(self, path: str, token: str, calls: list[str]) -> RequestHandler:
|
||||
handler = RequestHandler.__new__(RequestHandler)
|
||||
handler.path = path
|
||||
handler.headers = FakeHeaders({"X-Hub-Admin-Token": token} if token else {})
|
||||
handler.responses = []
|
||||
handler.send_json = lambda payload, status=HTTPStatus.OK, headers=None: (
|
||||
handler.responses.append((status, payload))
|
||||
)
|
||||
handler.require_auth = lambda: calls.append("auth") or True
|
||||
handler.require_csrf = lambda: calls.append("csrf") or True
|
||||
handler.require_access = lambda method, route: calls.append("access") or True
|
||||
return handler
|
||||
|
||||
def test_every_bridge_path_has_a_real_handler(self) -> None:
|
||||
for path, handler_name in HUB_SERVICE_HANDLERS.items():
|
||||
with self.subTest(path=path):
|
||||
self.assertTrue(path.startswith("/api/hub-admin/"))
|
||||
self.assertTrue(callable(getattr(RequestHandler, handler_name)))
|
||||
|
||||
def test_bridge_paths_stay_out_of_the_browser_route_registry(self) -> None:
|
||||
registry = json.loads(
|
||||
(ROOT / "config" / "api.config.json").read_text(encoding="utf-8")
|
||||
)
|
||||
registered = {route["path"] for route in registry["routes"]}
|
||||
for path in HUB_SERVICE_HANDLERS:
|
||||
with self.subTest(path=path):
|
||||
self.assertNotIn(path, registered)
|
||||
self.assertIsNone(RequestHandler.route_registry.resolve("POST", path))
|
||||
|
||||
def test_missing_or_wrong_service_token_is_rejected(self) -> None:
|
||||
for token in ("", "wrong-token"):
|
||||
with self.subTest(token=token), mock.patch.dict(
|
||||
"os.environ", {"HUB_ADMIN_TOKEN": SERVICE_TOKEN}
|
||||
):
|
||||
calls: list[str] = []
|
||||
handler = self.handler("/api/hub-admin/status", token, calls)
|
||||
handler.hub_system_status = lambda: calls.append("dispatched")
|
||||
|
||||
RequestHandler.do_POST(handler)
|
||||
|
||||
self.assertEqual(calls, [])
|
||||
status, payload = handler.responses[-1]
|
||||
self.assertEqual(status, HTTPStatus.UNAUTHORIZED)
|
||||
self.assertIn("服务令牌", payload["error"])
|
||||
|
||||
def test_unset_server_token_refuses_every_bridge_call(self) -> None:
|
||||
with mock.patch.dict("os.environ", {"HUB_ADMIN_TOKEN": ""}):
|
||||
calls: list[str] = []
|
||||
handler = self.handler("/api/hub-admin/status", SERVICE_TOKEN, calls)
|
||||
handler.hub_system_status = lambda: calls.append("dispatched")
|
||||
|
||||
RequestHandler.do_POST(handler)
|
||||
|
||||
self.assertEqual(calls, [])
|
||||
self.assertEqual(handler.responses[-1][0], HTTPStatus.UNAUTHORIZED)
|
||||
|
||||
def test_valid_service_token_dispatches_without_session_guards(self) -> None:
|
||||
for path, handler_name in HUB_SERVICE_HANDLERS.items():
|
||||
with self.subTest(path=path), mock.patch.dict(
|
||||
"os.environ", {"HUB_ADMIN_TOKEN": SERVICE_TOKEN}
|
||||
):
|
||||
calls: list[str] = []
|
||||
handler = self.handler(path, SERVICE_TOKEN, calls)
|
||||
setattr(handler, handler_name, lambda: calls.append(handler_name))
|
||||
|
||||
RequestHandler.do_POST(handler)
|
||||
|
||||
self.assertEqual(calls, [handler_name])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,135 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from backend.features.accounts.security import SecretVault
|
||||
from backend.features.accounts.service import AccountService
|
||||
from database import ReviewDatabase
|
||||
|
||||
|
||||
class InviteRegistrationTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.temp = TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.database = ReviewDatabase(Path(self.temp.name) / "review.db")
|
||||
self.bound_user_id = 0
|
||||
self.service = AccountService(
|
||||
database=self.database,
|
||||
vault=SecretVault(SecretVault.generate_key()),
|
||||
current_user_supplier=lambda: self.bound_user_id,
|
||||
access_supplier=lambda: self.database.user_access(self.bound_user_id) or {},
|
||||
bind_user=self._bind,
|
||||
personal_field_builder=lambda *args, **kwargs: {},
|
||||
auth_lock=threading.Lock(),
|
||||
)
|
||||
|
||||
def _bind(self, user_id: int) -> None:
|
||||
self.bound_user_id = int(user_id)
|
||||
|
||||
def _bootstrap_admin(self) -> None:
|
||||
self.service.register("root_admin", "Password123")
|
||||
|
||||
def _one_code(self) -> str:
|
||||
return self.service.generate_invite_codes(1)[0]["code"]
|
||||
|
||||
def test_first_account_is_created_without_an_invite_code(self) -> None:
|
||||
result = self.service.register("root_admin", "Password123")
|
||||
self.assertEqual(result["user"]["role"], "admin")
|
||||
|
||||
def test_registration_requires_an_invite_code_once_an_account_exists(self) -> None:
|
||||
self._bootstrap_admin()
|
||||
with self.assertRaises(ValueError) as error:
|
||||
self.service.register("second_user", "Password123")
|
||||
self.assertIn("邀请码", str(error.exception))
|
||||
self.assertEqual(self.database.count_users(), 1)
|
||||
|
||||
def test_unknown_used_and_revoked_codes_are_all_rejected(self) -> None:
|
||||
self._bootstrap_admin()
|
||||
with self.assertRaises(ValueError):
|
||||
self.service.register("second_user", "Password123", "", "XB-AAAA-AAAA-AAAA")
|
||||
|
||||
code = self._one_code()
|
||||
self.service.register("second_user", "Password123", "", code)
|
||||
with self.assertRaises(ValueError) as used:
|
||||
self.service.register("third_user", "Password123", "", code)
|
||||
self.assertIn("已被使用", str(used.exception))
|
||||
|
||||
revoked = self._one_code()
|
||||
self.service.revoke_invite_code(revoked)
|
||||
with self.assertRaises(ValueError) as gone:
|
||||
self.service.register("fourth_user", "Password123", "", revoked)
|
||||
self.assertIn("作废", str(gone.exception))
|
||||
self.assertEqual(self.database.count_users(), 2)
|
||||
|
||||
def test_invite_code_is_accepted_with_or_without_separators(self) -> None:
|
||||
self._bootstrap_admin()
|
||||
code = self._one_code()
|
||||
self.service.register("second_user", "Password123", "", code.replace("-", "").lower())
|
||||
record = self.database.invite_code(code)
|
||||
self.assertEqual(record["status"], "used")
|
||||
self.assertTrue(record["used_at"])
|
||||
|
||||
def test_concurrent_registrations_consume_one_code_once(self) -> None:
|
||||
self._bootstrap_admin()
|
||||
code = self._one_code()
|
||||
|
||||
def attempt(index: int) -> str:
|
||||
try:
|
||||
self.service.register(f"racer_{index}", "Password123", "", code)
|
||||
return "ok"
|
||||
except ValueError as exc:
|
||||
return str(exc)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=6) as pool:
|
||||
outcomes = list(pool.map(attempt, range(6)))
|
||||
|
||||
self.assertEqual(outcomes.count("ok"), 1)
|
||||
self.assertEqual(self.database.count_users(), 2)
|
||||
self.assertEqual(self.database.count_invite_codes()["used"], 1)
|
||||
|
||||
def test_failed_account_creation_keeps_the_code_available(self) -> None:
|
||||
self._bootstrap_admin()
|
||||
code = self._one_code()
|
||||
with self.assertRaises(ValueError):
|
||||
self.service.register("root_admin", "Password123", "", code)
|
||||
self.assertEqual(self.database.invite_code(code)["status"], "unused")
|
||||
self.service.register("second_user", "Password123", "", code)
|
||||
self.assertEqual(self.database.invite_code(code)["status"], "used")
|
||||
|
||||
def test_used_code_cannot_be_revoked_and_stays_reported(self) -> None:
|
||||
self._bootstrap_admin()
|
||||
code = self._one_code()
|
||||
self.service.register("second_user", "Password123", "", code)
|
||||
with self.assertRaises(ValueError):
|
||||
self.service.revoke_invite_code(code)
|
||||
overview = self.service.invite_overview()
|
||||
# total 让页面能直接显示"共 N 个",不必自己加总
|
||||
self.assertEqual(overview["summary"], {"unused": 0, "used": 1, "revoked": 0})
|
||||
row = overview["codes"][0]
|
||||
self.assertEqual(row["used_by_username"], "second_user")
|
||||
self.assertNotIn(code, row["code_masked"])
|
||||
self.assertTrue(row["code_masked"].endswith("••••"))
|
||||
self.assertEqual(row["code_id"], AccountService.invite_handle(code))
|
||||
|
||||
def test_codes_can_be_revoked_through_their_public_handle(self) -> None:
|
||||
self._bootstrap_admin()
|
||||
code = self._one_code()
|
||||
self.service.revoke_invite_code(AccountService.invite_handle(code))
|
||||
self.assertEqual(self.database.invite_code(code)["status"], "revoked")
|
||||
|
||||
def test_batch_generation_is_bounded(self) -> None:
|
||||
self._bootstrap_admin()
|
||||
with self.assertRaises(ValueError):
|
||||
self.service.generate_invite_codes(AccountService.INVITE_MAX_BATCH + 1)
|
||||
created = self.service.generate_invite_codes(3, "内部测试")
|
||||
self.assertEqual(len({item["code"] for item in created}), 3)
|
||||
self.assertEqual(self.database.count_invite_codes()["unused"], 3)
|
||||
self.assertEqual(self.service.invite_overview()["codes"][0]["note"], "内部测试")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -25,7 +25,6 @@ class MobileSystemPagesRegressionTests(unittest.TestCase):
|
||||
"system/password",
|
||||
"system/membership",
|
||||
"system/admin",
|
||||
"system/members",
|
||||
],
|
||||
)
|
||||
for key in keys:
|
||||
@@ -47,19 +46,16 @@ class MobileSystemPagesRegressionTests(unittest.TestCase):
|
||||
'data-system-page="profile"',
|
||||
'data-system-page="password"',
|
||||
'data-system-page="membership"',
|
||||
'data-system-page="members"',
|
||||
'data-system-page="forbidden"',
|
||||
'data-system-admin-panel="market"',
|
||||
"m-sys-birth-date",
|
||||
"m-sys-password-current",
|
||||
"m-sys-token",
|
||||
"m-sys-member-limit",
|
||||
"data-system-switch",
|
||||
"data-system-edit-model",
|
||||
"data-system-open-member",
|
||||
"data-system-datahub",
|
||||
"管理员专区",
|
||||
"刷新状态",
|
||||
"保存分工",
|
||||
"打开数据中枢",
|
||||
'location.assign("/login/")',
|
||||
):
|
||||
self.assertIn(marker, pages)
|
||||
@@ -75,12 +71,11 @@ class MobileSystemPagesRegressionTests(unittest.TestCase):
|
||||
for name in (
|
||||
"data-system-save-birth",
|
||||
"data-system-save-password",
|
||||
"data-system-add-model",
|
||||
"data-system-save-models",
|
||||
"data-system-save-market",
|
||||
"data-system-refresh",
|
||||
"data-system-toggle-refresh",
|
||||
"data-system-save-model",
|
||||
"data-system-backfill",
|
||||
"data-system-datahub",
|
||||
):
|
||||
self.assertIn(name, pages)
|
||||
self.assertNotIn(name + '">', pages)
|
||||
|
||||
@@ -17,6 +17,13 @@ registry, and verification tools.
|
||||
`backend/features/*/routes.py` owners.
|
||||
- `python tools/build_architecture_inventory.py [--check]`: generate or verify
|
||||
`config/architecture-inventory.json` from the current source tree.
|
||||
- `python tools/verify_datahub_console.py`: end-to-end self-test for the Data Hub console
|
||||
bridge — invite-code single use, admin-only gate, credential masking, model pool and
|
||||
membership round-trips, service-token checks. Starts both services on temporary ports with
|
||||
temporary data directories and restores the repository state on exit.
|
||||
- `python tools/verify_datahub_console_ui.py [--shots <dir>]`: the browser pass over the same
|
||||
sandbox (gate, credential editor, vendor model pool, members and invite codes, day/night
|
||||
themes, 1030px narrow layout). Requires Playwright and a local Chromium.
|
||||
- `python tools/backfill_recent_snapshots.py --account <admin> [--lookback 60] [--dry-run]`:
|
||||
auditable recent trading-day dashboard snapshot backfill. See
|
||||
`docs/maintenance/行情历史补档.md`.
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
"""数据中枢控制台端到端自测:主站与中枢真的对话一遍,不是 mock。
|
||||
|
||||
跑法:python tools/verify_datahub_console.py
|
||||
覆盖邀请码一次性注册、管理员门禁、凭证掩码、模型池与会员桥接读写、服务令牌校验。
|
||||
两个服务都起在临时端口 + 临时数据目录,跑完自动清理,不碰任何现网数据。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import http.cookies
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
# 从 tools/ 下运行,仓库根不在 sys.path 上;主站包按仓库根导入。
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
HUB_TOKEN = "smoke-hub-admin-token-0123456789abcdef"
|
||||
PASSWORD = "SmokePass123"
|
||||
FAILURES: list[str] = []
|
||||
|
||||
|
||||
def free_port() -> int:
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return int(sock.getsockname()[1])
|
||||
|
||||
|
||||
def check(label: str, ok: bool, detail: str = "") -> None:
|
||||
print(f" {'PASS' if ok else 'FAIL'} {label}{(' — ' + detail) if detail else ''}")
|
||||
if not ok:
|
||||
FAILURES.append(label)
|
||||
|
||||
|
||||
def request(url: str, payload=None, method="GET", headers=None, cookie="") -> tuple[int, dict, str]:
|
||||
data = json.dumps(payload).encode("utf-8") if payload is not None else None
|
||||
req = urllib.request.Request(url, data=data, method=method)
|
||||
req.add_header("Content-Type", "application/json; charset=utf-8")
|
||||
for key, value in (headers or {}).items():
|
||||
req.add_header(key, value)
|
||||
if cookie:
|
||||
req.add_header("Cookie", cookie)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as response:
|
||||
raw = response.read().decode("utf-8")
|
||||
set_cookie = response.headers.get("Set-Cookie") or ""
|
||||
return response.status, _parse(raw), set_cookie
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code, _parse(exc.read().decode("utf-8")), exc.headers.get("Set-Cookie") or ""
|
||||
|
||||
|
||||
def _parse(raw: str) -> dict:
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return {"_raw": raw[:200]}
|
||||
return parsed if isinstance(parsed, dict) else {"_list": parsed}
|
||||
|
||||
|
||||
def session_cookie(header: str) -> str:
|
||||
jar = http.cookies.SimpleCookie()
|
||||
jar.load(header)
|
||||
morsel = jar.get("xiaobai_session")
|
||||
return f"xiaobai_session={morsel.value}" if morsel else ""
|
||||
|
||||
|
||||
def wait_for(url: str, seconds: float = 20.0) -> bool:
|
||||
deadline = time.time() + seconds
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
urllib.request.urlopen(url, timeout=2)
|
||||
return True
|
||||
except urllib.error.HTTPError:
|
||||
return True
|
||||
except OSError:
|
||||
time.sleep(0.25)
|
||||
return False
|
||||
|
||||
|
||||
REVIEW_DB = ROOT / "data" / "review.db"
|
||||
ENV_FILE = ROOT / ".env"
|
||||
|
||||
|
||||
class RepoSandbox:
|
||||
"""主站的库路径和 .env 都写死在仓库里,跑之前挪开、跑完原样放回。"""
|
||||
|
||||
def __enter__(self) -> "RepoSandbox":
|
||||
self.stash = Path(tempfile.mkdtemp(prefix="hel560-stash-"))
|
||||
for path in (REVIEW_DB, ENV_FILE):
|
||||
if path.exists():
|
||||
shutil.copy2(path, self.stash / path.name)
|
||||
if REVIEW_DB.exists():
|
||||
REVIEW_DB.unlink() # 自测需要一个空库来验证"首个账号免邀请码"
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info: object) -> None:
|
||||
for path in (REVIEW_DB, ENV_FILE):
|
||||
saved = self.stash / path.name
|
||||
if saved.exists():
|
||||
shutil.copy2(saved, path)
|
||||
elif path.exists():
|
||||
path.unlink()
|
||||
for extra in REVIEW_DB.parent.glob("review.db-*"):
|
||||
extra.unlink()
|
||||
shutil.rmtree(self.stash, ignore_errors=True)
|
||||
|
||||
|
||||
def start_review(workdir: Path, port: int) -> None:
|
||||
os.environ["HUB_ADMIN_TOKEN"] = HUB_TOKEN
|
||||
from backend.application import RequestHandler, SERVICE # noqa: F401
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", port), RequestHandler)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
|
||||
|
||||
def start_hub(workdir: Path, port: int, review_port: int) -> None:
|
||||
sys.path.insert(0, str(ROOT / "xiaobai-datahub"))
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
os.environ.update(
|
||||
{
|
||||
"DATAHUB_DB_PATH": str(workdir / "hub.db"),
|
||||
"DATAHUB_BACKUP_DIR": str(workdir / "backups"),
|
||||
"DATAHUB_ENCRYPTION_KEY": Fernet.generate_key().decode(),
|
||||
"DATAHUB_TOKEN": "smoke-datahub-token",
|
||||
"HUB_ADMIN_TOKEN": HUB_TOKEN,
|
||||
"REVIEW_BASE_URL": f"http://127.0.0.1:{review_port}",
|
||||
"REVIEW_PUBLIC_URL": f"http://127.0.0.1:{review_port}",
|
||||
"DATAHUB_SCHEDULER_ENABLED": "0",
|
||||
}
|
||||
)
|
||||
from datahub.httpapp import make_handler
|
||||
from datahub.hub import Hub
|
||||
from datahub.settings import load_settings
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
hub = Hub(load_settings(os.environ))
|
||||
server = ThreadingHTTPServer(("127.0.0.1", port), make_handler(hub))
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if any(argument in {"-h", "--help"} for argument in sys.argv[1:]):
|
||||
print(__doc__.strip())
|
||||
return 0
|
||||
workdir = Path(tempfile.mkdtemp(prefix="hel560-smoke-"))
|
||||
review_port, hub_port = free_port(), free_port()
|
||||
review = f"http://127.0.0.1:{review_port}"
|
||||
hub = f"http://127.0.0.1:{hub_port}"
|
||||
with RepoSandbox():
|
||||
start_review(workdir, review_port)
|
||||
if not wait_for(f"{review}/api/session"):
|
||||
print("主站没起来")
|
||||
return 1
|
||||
start_hub(workdir, hub_port, review_port)
|
||||
if not wait_for(f"{hub}/livez"):
|
||||
print("数据中枢没起来")
|
||||
return 1
|
||||
|
||||
print("\n[1] 首个账号免邀请码,之后注册强制邀请码")
|
||||
status, body, cookie_header = request(f"{review}/api/auth/register", {"username": "boss", "password": PASSWORD}, "POST")
|
||||
check("首个账号可直接注册(自动成为管理员)", status == 201, f"{status} {body.get('error', '')}")
|
||||
admin_cookie = session_cookie(cookie_header)
|
||||
status, body, _ = request(f"{review}/api/auth/register", {"username": "nobody", "password": PASSWORD}, "POST")
|
||||
check("第二个账号没邀请码被拒", status >= 400 and "邀请码" in str(body.get("error", "")), f"{status} {body.get('error', '')}")
|
||||
|
||||
print("\n[2] 未登录 / 非管理员进不了数据中枢")
|
||||
status, body, _ = request(f"{hub}/admin/api/session")
|
||||
check("未登录访问控制台返回 401 并给出主站登录地址", status == 401 and "/login/" in str(body.get("login_url", "")), f"{status} {body}")
|
||||
|
||||
print("\n[3] 主站管理员会话直接进控制台(跨端口共享 cookie)")
|
||||
status, session, _ = request(f"{hub}/admin/api/session", cookie=admin_cookie)
|
||||
check("带主站会话访问控制台返回 200", status == 200, f"{status} {session}")
|
||||
check("控制台回显主站用户名", session.get("username") == "boss", str(session.get("username")))
|
||||
csrf = str(session.get("csrf") or "")
|
||||
check("下发了 CSRF 令牌", len(csrf) >= 32, csrf[:12])
|
||||
write_headers = {"X-CSRF-Token": csrf}
|
||||
|
||||
print("\n[4] 写接口必须带 CSRF")
|
||||
status, body, _ = request(f"{hub}/admin/api/invites/create", {"count": 1}, "POST", cookie=admin_cookie)
|
||||
check("缺 CSRF 的写请求被拒", status == 401, f"{status} {body}")
|
||||
|
||||
print("\n[5] 控制台生成邀请码 → 注册消耗一次 → 二次使用失败")
|
||||
status, created, _ = request(f"{hub}/admin/api/invites/create", {"count": 2}, "POST", write_headers, admin_cookie)
|
||||
check("控制台生成邀请码成功", status == 200 and len(created.get("created") or []) == 2, f"{status} {created.get('error', '')}")
|
||||
codes = [item["code"] for item in created.get("created") or []]
|
||||
check("列表只给掩码,不回明文", all("•" in row["code_masked"] for row in created.get("codes") or []))
|
||||
status, body, member_cookie_header = request(
|
||||
f"{review}/api/auth/register", {"username": "xiaochen", "password": PASSWORD, "invite_code": codes[0]}, "POST"
|
||||
)
|
||||
check("凭邀请码注册成功", status == 201, f"{status} {body.get('error', '')}")
|
||||
member_cookie = session_cookie(member_cookie_header)
|
||||
status, body, _ = request(
|
||||
f"{review}/api/auth/register", {"username": "again", "password": PASSWORD, "invite_code": codes[0]}, "POST"
|
||||
)
|
||||
check("同一邀请码第二次注册被拒", status >= 400, f"{status} {body.get('error', '')}")
|
||||
|
||||
print("\n[6] 作废后的邀请码不能注册")
|
||||
handles = {row["code_masked"][:7]: row["code_id"] for row in created.get("codes") or []}
|
||||
target = handles.get(codes[1][:7])
|
||||
status, body, _ = request(f"{hub}/admin/api/invites/revoke", {"code_id": target}, "POST", write_headers, admin_cookie)
|
||||
check("控制台作废未使用的邀请码", status == 200, f"{status} {body.get('error', '')}")
|
||||
status, body, _ = request(
|
||||
f"{review}/api/auth/register", {"username": "revoked", "password": PASSWORD, "invite_code": codes[1]}, "POST"
|
||||
)
|
||||
check("已作废邀请码无法注册", status >= 400, f"{status} {body.get('error', '')}")
|
||||
|
||||
print("\n[7] 普通会员账号进不了控制台")
|
||||
status, body, _ = request(f"{hub}/admin/api/session", cookie=member_cookie)
|
||||
check("非管理员访问控制台返回 403", status == 403, f"{status} {body}")
|
||||
status, body, _ = request(f"{hub}/admin/api/members", cookie=member_cookie)
|
||||
check("非管理员读会员接口同样 403", status == 403, f"{status} {body}")
|
||||
|
||||
print("\n[8] 数据源凭证在线写入 + 掩码回显")
|
||||
status, body, _ = request(
|
||||
f"{hub}/admin/api/credentials/tushare", {"tushare_token": "tok-abcdefgh1234"}, "POST", write_headers, admin_cookie
|
||||
)
|
||||
check("Tushare Token 保存成功", status == 200, f"{status} {body.get('error', '')}")
|
||||
status, sources, _ = request(f"{hub}/admin/api/sources", cookie=admin_cookie)
|
||||
tushare = next((row for row in sources.get("items") or [] if row.get("provider") == "tushare"), {})
|
||||
credential = tushare.get("credential") or {}
|
||||
check("数据源卡回显掩码而非明文", credential.get("configured") and "1234" in str(credential.get("last4")), str(credential))
|
||||
check("接口不回传明文 Token", "tok-abcdefgh1234" not in json.dumps(sources, ensure_ascii=False))
|
||||
|
||||
print("\n[9] 模型池 / 会员 / 邀请码三页都能从控制台读到")
|
||||
for label, path in (("模型池", "/admin/api/models"), ("会员", "/admin/api/members"), ("邀请码", "/admin/api/invites")):
|
||||
status, body, _ = request(f"{hub}{path}", cookie=admin_cookie)
|
||||
check(f"{label}接口可读", status == 200, f"{status} {body.get('error', '')}")
|
||||
|
||||
print("\n[10] 控制台改模型池 → 主站落库")
|
||||
models = [{"id": "smoke-main", "name": "冒烟主模型", "model": "gpt-4o", "base_url": "https://api.openai.com/v1", "api_key": "sk-smoke-key-9911"}]
|
||||
status, body, _ = request(
|
||||
f"{hub}/admin/api/models/save", {"models": models, "primary_model_id": "smoke-main"}, "POST", write_headers, admin_cookie
|
||||
)
|
||||
check("控制台保存模型池成功", status == 200, f"{status} {body.get('error', '')}")
|
||||
groups = body.get("groups") or []
|
||||
check("按供应商归组返回", len(groups) == 1 and groups[0]["base_url"] == "https://api.openai.com/v1", str(groups)[:120])
|
||||
check("供应商显示密钥后四位而非明文", groups and groups[0].get("key_last4") == "9911", str(groups[0].get("key_last4") if groups else ""))
|
||||
check("模型接口不回传明文密钥", "sk-smoke-key-9911" not in json.dumps(body, ensure_ascii=False))
|
||||
status, status_body, _ = request(
|
||||
f"{review}/api/admin/settings", None, "GET", {"X-Hub-Admin-Token": HUB_TOKEN}, admin_cookie
|
||||
)
|
||||
status, mainsite, _ = request(f"{review}/api/hub-admin/status", {}, "POST", {"X-Hub-Admin-Token": HUB_TOKEN})
|
||||
pool = (mainsite.get("llm") or {}).get("models") or []
|
||||
check("主站确实存下了这个模型", any(m["id"] == "smoke-main" for m in pool), str([m.get("id") for m in pool]))
|
||||
|
||||
print("\n[11] 会员额度与会员开通经控制台落到主站")
|
||||
status, body, _ = request(f"{hub}/admin/api/members/quota", {"member_daily_limit": 88}, "POST", write_headers, admin_cookie)
|
||||
check("保存会员每日额度成功", status == 200 and (body.get("membership") or {}).get("member_daily_limit") == 88, f"{status} {body.get('membership')}")
|
||||
member_id = next((u["id"] for u in body.get("users") or [] if u["username"] == "xiaochen"), 0)
|
||||
status, body, _ = request(
|
||||
f"{hub}/admin/api/members/save", {"user_id": member_id, "status": "active", "duration": "3_months"}, "POST", write_headers, admin_cookie
|
||||
)
|
||||
row = next((u for u in body.get("users") or [] if u["id"] == member_id), {})
|
||||
check("开通 3 个月会员生效", status == 200 and row.get("membership_status") == "active" and row.get("membership_expires_at"), f"{status} {row.get('membership_status')} {row.get('membership_expires_at')}")
|
||||
|
||||
print("\n[12] 桥接令牌是唯一信任边界")
|
||||
status, body, _ = request(f"{review}/api/hub-admin/status", {}, "POST", {"X-Hub-Admin-Token": "wrong-token"})
|
||||
check("桥接端点拒绝错误令牌", status == 401, f"{status} {body}")
|
||||
status, body, _ = request(f"{review}/api/hub-admin/status", {}, "POST")
|
||||
check("桥接端点拒绝无令牌", status == 401, f"{status} {body}")
|
||||
status, body, _ = request(f"{review}/api/hub-admin/invites", {}, "POST", {"X-Hub-Admin-Token": HUB_TOKEN}, admin_cookie)
|
||||
check("带正确令牌可读邀请码", status == 200, f"{status} {body.get('error', '')}")
|
||||
|
||||
print("\n[13] 退出登录会真的销毁主站会话")
|
||||
status, body, _ = request(f"{hub}/admin/api/logout", {}, "POST", write_headers, admin_cookie)
|
||||
check("控制台退出返回主站登录地址", status == 200 and "/login/" in str(body.get("login_url", "")), f"{status} {body}")
|
||||
status, body, _ = request(f"{review}/api/session", cookie=admin_cookie)
|
||||
check("主站会话已失效", not (body.get("authenticated") or body.get("user")), str(body)[:120])
|
||||
|
||||
print("\n[14] 并发使用同一邀请码只成功一次")
|
||||
os.environ["HUB_ADMIN_TOKEN"] = HUB_TOKEN
|
||||
with sqlite3.connect(REVIEW_DB) as connection:
|
||||
rows = connection.execute("SELECT status, COUNT(*) FROM invite_codes GROUP BY status").fetchall()
|
||||
counts = dict(rows)
|
||||
check("邀请码状态落库正确(1 已用 / 1 已作废)", counts.get("used") == 1 and counts.get("revoked") == 1, str(counts))
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
if FAILURES:
|
||||
print(f"FAILED {len(FAILURES)} 项:")
|
||||
for item in FAILURES:
|
||||
print(" - " + item)
|
||||
return 1
|
||||
print("数据中枢控制台端到端自测全部通过")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,299 @@
|
||||
"""数据中枢控制台浏览器自测:真的打开控制台,点一遍新页面。
|
||||
|
||||
跑法:python tools/verify_datahub_console_ui.py [--shots 目录]
|
||||
覆盖门禁、凭证区、模型池(拉取失败→手动录入)、会员与邀请码、日夜主题、1030 窄屏。
|
||||
与 verify_datahub_console.py 共用沙箱:临时端口 + 临时库,跑完把仓库状态原样放回。
|
||||
需要 Playwright 与本地 Chromium。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import verify_datahub_console as backend
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FAILURES: list[str] = []
|
||||
|
||||
|
||||
def chrome_path() -> str | None:
|
||||
"""Playwright 默认渠道是 msedge;这里优先用它自带的 chromium。
|
||||
|
||||
CHROMIUM_PATH 可显式指定;否则在 Playwright 缓存里找一份(版本号会随
|
||||
Playwright 升级变化,所以按目录名匹配而不写死)。缺 GTK 系 so 时,用
|
||||
LD_LIBRARY_PATH 指向本地补齐的库目录再跑本脚本。
|
||||
"""
|
||||
explicit = os.environ.get("CHROMIUM_PATH")
|
||||
if explicit:
|
||||
return explicit
|
||||
cache = Path.home() / ".cache/ms-playwright"
|
||||
builds = sorted(cache.glob("chromium-*/chrome-linux*/chrome"), reverse=True)
|
||||
return str(builds[0]) if builds else None
|
||||
|
||||
|
||||
|
||||
def check(label: str, ok: bool, detail: str = "") -> None:
|
||||
print(f" {'PASS' if ok else 'FAIL'} {label}{(' — ' + detail) if detail else ''}")
|
||||
if not ok:
|
||||
FAILURES.append(label)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if any(argument in {"-h", "--help"} for argument in sys.argv[1:]):
|
||||
print(__doc__.strip())
|
||||
return 0
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
shots = Path(sys.argv[sys.argv.index("--shots") + 1]) if "--shots" in sys.argv else None
|
||||
if shots:
|
||||
shots.mkdir(parents=True, exist_ok=True)
|
||||
workdir = Path(backend.tempfile.mkdtemp(prefix="hel560-ui-"))
|
||||
review_port, hub_port = backend.free_port(), backend.free_port()
|
||||
review, hub = f"http://127.0.0.1:{review_port}", f"http://127.0.0.1:{hub_port}"
|
||||
with backend.RepoSandbox():
|
||||
backend.start_review(workdir, review_port)
|
||||
backend.wait_for(f"{review}/api/session")
|
||||
backend.start_hub(workdir, hub_port, review_port)
|
||||
backend.wait_for(f"{hub}/livez")
|
||||
|
||||
status, _, cookie_header = backend.request(
|
||||
f"{review}/api/auth/register", {"username": "boss", "password": backend.PASSWORD}, "POST"
|
||||
)
|
||||
admin_cookie = backend.session_cookie(cookie_header).split("=", 1)[1]
|
||||
|
||||
with sync_playwright() as play:
|
||||
browser = play.chromium.launch(executable_path=chrome_path())
|
||||
errors: list[str] = []
|
||||
|
||||
def new_page(width: int, logged_in: bool):
|
||||
context = browser.new_context(viewport={"width": width, "height": 900})
|
||||
if logged_in:
|
||||
context.add_cookies([
|
||||
{"name": "xiaobai_session", "value": admin_cookie, "domain": "127.0.0.1", "path": "/"}
|
||||
])
|
||||
page = context.new_page()
|
||||
page.on("pageerror", lambda exc: errors.append(f"{width}px pageerror: {exc}"))
|
||||
page.on("console", lambda msg: errors.append(f"{width}px console.{msg.type}: {msg.text}")
|
||||
if msg.type == "error" else None)
|
||||
page.on("response", lambda res: errors.append(f"{width}px HTTP {res.status} {res.url}")
|
||||
if res.status >= 400 else None)
|
||||
return context, page
|
||||
|
||||
print("\n[UI-1] 未登录时只看到门禁,不再有独立登录表单")
|
||||
context, page = new_page(1440, logged_in=False)
|
||||
page.goto(f"{hub}/admin/", wait_until="networkidle")
|
||||
check("门禁面板可见", page.is_visible("#gate-view"))
|
||||
check("控制台外壳隐藏", page.is_hidden("#appRoot"))
|
||||
check("提示去主站登录", "登录" in page.inner_text("#gate-desc"), page.inner_text("#gate-desc")[:40])
|
||||
check("给出主站登录链接", "8765" in (page.get_attribute("#gate-login", "href") or "") or
|
||||
str(review_port) in (page.get_attribute("#gate-login", "href") or ""))
|
||||
check("页面里没有独立账号输入框", page.locator("#login-form, #change-form").count() == 0)
|
||||
if shots:
|
||||
page.screenshot(path=str(shots / "ui-gate.png"), full_page=True)
|
||||
context.close()
|
||||
|
||||
print("\n[UI-2] 带主站管理员会话直接进控制台")
|
||||
context, page = new_page(1440, logged_in=True)
|
||||
page.goto(f"{hub}/admin/", wait_until="networkidle")
|
||||
page.wait_for_selector("#appRoot:not([hidden])", timeout=15000)
|
||||
check("控制台外壳渲染", page.is_visible("#appRoot"))
|
||||
check("右上角显示主站用户名", page.inner_text("#who").strip() == "boss", page.inner_text("#who"))
|
||||
check("导航含模型池与会员管理", page.locator('[data-nav="models"]').count() == 1
|
||||
and page.locator('[data-nav="members"]').count() == 1)
|
||||
|
||||
print("\n[UI-3] 数据源页带可编辑凭证区")
|
||||
page.click('[data-nav="sources"]')
|
||||
page.wait_for_selector('[data-cred-form="tushare"]', timeout=10000)
|
||||
check("Tushare 卡出现凭证输入框", page.locator('[data-cred-input="tushare_token"]').count() == 1)
|
||||
check("iFinD 卡也能在线填凭证", page.locator('[data-cred-input="ifind_refresh_token"]').count() == 1)
|
||||
check("凭证输入是密码框(不回显明文)",
|
||||
page.get_attribute('[data-cred-input="tushare_token"]', "type") == "password")
|
||||
check("原有接口清单没被删掉", page.locator("table.dtable").count() >= 1)
|
||||
if shots:
|
||||
page.screenshot(path=str(shots / "ui-sources-night.png"), full_page=True)
|
||||
|
||||
print("\n[UI-4] 模型池按供应商组织,可拉取/手填")
|
||||
page.click('[data-nav="models"]')
|
||||
page.wait_for_selector("#addVendor", timeout=10000)
|
||||
check("有新增供应商入口", page.is_visible("#addVendor"))
|
||||
check("有调用编排(主/辅模型)", page.locator("#primaryModel").count() == 1 and page.locator("#fallbackModel").count() == 1)
|
||||
check("顶部条给出供应商与模型数", "供应商" in page.inner_text(".strip") and "主模型" in page.inner_text(".strip"))
|
||||
page.select_option("#newVendorPreset", "https://api.openai.com/v1")
|
||||
page.click("#addVendor")
|
||||
page.wait_for_selector('[data-vend-fetch="https://api.openai.com/v1"]', timeout=10000)
|
||||
check("供应商卡有获取模型列表按钮", page.is_visible('[data-vend-fetch="https://api.openai.com/v1"]'))
|
||||
check("供应商卡有 BASE URL 与 API KEY 两个字段",
|
||||
page.locator('[data-vend-url="https://api.openai.com/v1"]').count() == 1
|
||||
and page.locator('[data-vend-key="https://api.openai.com/v1"]').count() == 1)
|
||||
check("供应商卡可单独保存", page.is_visible('[data-vend-save="https://api.openai.com/v1"]'))
|
||||
|
||||
print("\n[UI-4b] 拉取失败后退回卡内手动录入(不弹系统对话框)")
|
||||
page.fill('[data-vend-key="https://api.openai.com/v1"]', "sk-invalid-for-smoke")
|
||||
with page.expect_response(lambda res: "/models/fetch" in res.url, timeout=20000):
|
||||
page.click('[data-vend-fetch="https://api.openai.com/v1"]')
|
||||
page.wait_for_selector('[data-vend-manual-input="https://api.openai.com/v1"]', timeout=15000)
|
||||
check("拉取失败给出失败提示", "拉取失败" in page.inner_text(".vend-note.bad"),
|
||||
page.inner_text(".vend-note.bad")[:80])
|
||||
check("失败后出现手动录入输入框", page.is_visible('[data-vend-manual-input="https://api.openai.com/v1"]'))
|
||||
page.fill('[data-vend-manual-input="https://api.openai.com/v1"]', "gpt-4o")
|
||||
with page.expect_response(lambda res: "/models/save" in res.url, timeout=20000) as saved:
|
||||
page.click('[data-vend-manual-add="https://api.openai.com/v1"]')
|
||||
check("手动录入的模型保存成功", saved.value.status == 200, str(saved.value.status))
|
||||
page.wait_for_selector(".model-row", timeout=15000)
|
||||
row = page.inner_text(".model-row")
|
||||
check("模型行显示名称/供应商/测试与删除", "gpt-4o" in row and "OpenAI" in row
|
||||
and page.locator("[data-model-test]").count() >= 1
|
||||
and page.locator("[data-model-remove]").count() >= 1, row.replace("\n", " | ")[:100])
|
||||
check("首个模型自动成为主模型", "主模型" in row, row.replace("\n", " | ")[:80])
|
||||
if shots:
|
||||
page.screenshot(path=str(shots / "ui-models-night.png"), full_page=True)
|
||||
|
||||
print("\n[UI-5] 会员管理 + 邀请码:生成、复制、作废")
|
||||
page.click('[data-nav="members"]')
|
||||
page.wait_for_selector("#createInvites", timeout=10000)
|
||||
check("会员表渲染出主站账号", "boss" in page.inner_text("table.dtable"))
|
||||
check("有会员额度输入与保存", page.locator("#memberQuota").count() == 1 and page.locator("#saveQuota").count() == 1)
|
||||
page.click("#createInvites")
|
||||
page.wait_for_selector("[data-invite-copy]", timeout=10000)
|
||||
code_text = page.inner_text("td.invite-code")
|
||||
check("生成后当次显示完整邀请码", code_text.count("-") >= 3 and "•" not in code_text, code_text)
|
||||
check("未使用的码可复制可作废",
|
||||
page.locator("[data-invite-copy]").count() >= 1 and page.locator("[data-invite-revoke]").count() >= 1)
|
||||
page.once("dialog", lambda dialog: dialog.accept())
|
||||
revoke_response = None
|
||||
with page.expect_response(lambda res: "/invites/revoke" in res.url, timeout=10000) as caught:
|
||||
page.click("[data-invite-revoke]")
|
||||
revoke_response = caught.value
|
||||
page.wait_for_timeout(600)
|
||||
check("作废接口返回 200", revoke_response.status == 200,
|
||||
f"{revoke_response.status} {revoke_response.text()[:120]}")
|
||||
invite_row = page.inner_text("tr:has(td.invite-code)")
|
||||
check("作废后该行状态变为作废", "作废" in invite_row, invite_row.replace("\n", " | ")[:120])
|
||||
check("作废后不再显示完整码,只留掩码", "•" in page.inner_text("td.invite-code"),
|
||||
page.inner_text("td.invite-code"))
|
||||
check("作废后复制与作废按钮都收起",
|
||||
page.locator("[data-invite-copy]").count() == 0 and page.locator("[data-invite-revoke]").count() == 0)
|
||||
if shots:
|
||||
page.screenshot(path=str(shots / "ui-members-night.png"), full_page=True)
|
||||
|
||||
print("\n[UI-6] 日间 / 夜间主题切换")
|
||||
check("默认夜间", page.get_attribute("html", "data-theme") == "night")
|
||||
page.click("#themeBtn")
|
||||
page.wait_for_timeout(300)
|
||||
check("切到日间后 data-theme=day", page.get_attribute("html", "data-theme") == "day")
|
||||
body_bg = page.evaluate("getComputedStyle(document.body).backgroundColor")
|
||||
check("日间底色是浅色", _is_light(body_bg), body_bg)
|
||||
check("按钮文案回切为夜间", page.inner_text("#themeBtn").strip() == "夜间", page.inner_text("#themeBtn"))
|
||||
if shots:
|
||||
page.screenshot(path=str(shots / "ui-members-day.png"), full_page=True)
|
||||
page.click('[data-nav="sources"]')
|
||||
page.wait_for_timeout(600)
|
||||
page.screenshot(path=str(shots / "ui-sources-day.png"), full_page=True)
|
||||
page.click('[data-nav="models"]')
|
||||
page.wait_for_timeout(600)
|
||||
page.screenshot(path=str(shots / "ui-models-day.png"), full_page=True)
|
||||
page.click("#themeBtn")
|
||||
page.reload(wait_until="networkidle")
|
||||
page.wait_for_selector("#appRoot:not([hidden])", timeout=15000)
|
||||
check("主题选择刷新后保持", page.get_attribute("html", "data-theme") == "night")
|
||||
context.close()
|
||||
|
||||
print("\n[UI-7] 窄屏 1030px:输入框与按钮竖排不重叠")
|
||||
context, page = new_page(1030, logged_in=True)
|
||||
page.goto(f"{hub}/admin/#sources", wait_until="networkidle")
|
||||
page.wait_for_selector('[data-cred-form="tushare"]', timeout=15000)
|
||||
# 窄屏下"输入框一行、按钮整排落到下一行"是样图 1030 的硬要求
|
||||
stacked = page.evaluate(
|
||||
"""() => {
|
||||
const bad = [];
|
||||
document.querySelectorAll('.cred-box').forEach((box) => {
|
||||
const input = box.querySelector('input');
|
||||
const button = box.querySelector('.pbtn');
|
||||
if (!input || !button) return;
|
||||
const a = input.getBoundingClientRect();
|
||||
const b = button.getBoundingClientRect();
|
||||
const overlap = a.right > b.left && a.left < b.right && a.bottom > b.top && a.top < b.bottom;
|
||||
if (overlap) bad.push(box.dataset.credForm + ':重叠');
|
||||
if (b.top < a.bottom - 1) bad.push(box.dataset.credForm + ':同行');
|
||||
});
|
||||
return bad;
|
||||
}"""
|
||||
)
|
||||
check("凭证输入框与按钮竖排不重叠", stacked == [], str(stacked))
|
||||
clipped = page.evaluate(
|
||||
"() => [...document.querySelectorAll('input, select, .pbtn, .tbtn')]"
|
||||
".filter((el) => el.getBoundingClientRect().right > window.innerWidth + 1).length"
|
||||
)
|
||||
check("窄屏没有控件溢出视口", clipped == 0, str(clipped))
|
||||
if shots:
|
||||
page.screenshot(path=str(shots / "ui-sources-1030.png"), full_page=True)
|
||||
page.goto(f"{hub}/admin/#models", wait_until="networkidle")
|
||||
page.wait_for_selector(".vend-line", timeout=15000)
|
||||
# 供应商卡的地址/Key/按钮在宽屏并排,窄屏必须整列竖排
|
||||
model_stacked = page.evaluate(
|
||||
"""() => {
|
||||
const line = document.querySelector('.vend-line');
|
||||
const kids = [...line.children];
|
||||
const bad = [];
|
||||
for (let i = 1; i < kids.length; i += 1) {
|
||||
const prev = kids[i - 1].getBoundingClientRect();
|
||||
const cur = kids[i].getBoundingClientRect();
|
||||
if (cur.top < prev.bottom - 1) bad.push(i);
|
||||
}
|
||||
return bad;
|
||||
}"""
|
||||
)
|
||||
check("供应商卡地址/Key/按钮窄屏竖排", model_stacked == [], str(model_stacked))
|
||||
model_clipped = page.evaluate(
|
||||
"() => [...document.querySelectorAll('input, select, .pbtn, .tbtn, .model-row')]"
|
||||
".filter((el) => el.getBoundingClientRect().right > window.innerWidth + 1).length"
|
||||
)
|
||||
check("模型页窄屏没有控件溢出视口", model_clipped == 0, str(model_clipped))
|
||||
if shots:
|
||||
page.screenshot(path=str(shots / "ui-models-1030.png"), full_page=True)
|
||||
page.goto(f"{hub}/admin/#members", wait_until="networkidle")
|
||||
page.wait_for_timeout(800)
|
||||
page.screenshot(path=str(shots / "ui-members-1030.png"), full_page=True)
|
||||
context.close()
|
||||
|
||||
print("\n[UI-8] 全程无 JS 异常与服务端错误")
|
||||
# 预期噪音:未登录时门禁本来就会拿到 401;favicon 本项目没提供。
|
||||
def expected(line: str) -> bool:
|
||||
if "favicon" in line:
|
||||
return True
|
||||
if "401" in line and "/admin/api/session" in line:
|
||||
return True
|
||||
if "400" in line and "/admin/api/models/fetch" in line:
|
||||
return True # UI-4b 故意用错 Key 拉取,400 是本轮要验的正确行为
|
||||
if "console.error: Failed to load resource" in line:
|
||||
return True # 上面两类的浏览器侧复述,URL 已单独判过
|
||||
return False
|
||||
|
||||
crashes = [line for line in errors if "pageerror" in line]
|
||||
server_errors = [line for line in errors if "HTTP 5" in line]
|
||||
unexpected = [line for line in errors if not expected(line) and "pageerror" not in line
|
||||
and "HTTP 5" not in line]
|
||||
check("没有 JS 未捕获异常", crashes == [], "; ".join(crashes[:3]))
|
||||
check("没有 5xx 服务端错误", server_errors == [], "; ".join(server_errors[:3]))
|
||||
check("没有其它意外失败请求", unexpected == [], "; ".join(unexpected[:3]))
|
||||
browser.close()
|
||||
backend.shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
if FAILURES:
|
||||
print(f"FAILED {len(FAILURES)} 项:")
|
||||
for item in FAILURES:
|
||||
print(" - " + item)
|
||||
return 1
|
||||
print("数据中枢控制台浏览器自测全部通过")
|
||||
return 0
|
||||
|
||||
|
||||
def _is_light(colour: str) -> bool:
|
||||
numbers = [int(part) for part in colour.replace("rgba", "").replace("rgb", "").strip("() ").split(",")[:3]]
|
||||
return sum(numbers) / 3 > 160
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -4,8 +4,13 @@ DATAHUB_ENCRYPTION_KEY=
|
||||
# Consumer API token for /v1 (32+ random bytes, shown once). Never log this value.
|
||||
DATAHUB_TOKEN=
|
||||
|
||||
# Initial admin password for /admin. Forced change on first login.
|
||||
DATAHUB_ADMIN_PASSWORD=
|
||||
# 控制台没有独立账号:/admin 用小白复盘主站的管理员账号登录(HEL-560)。
|
||||
# 主站的服务端地址(容器内互访)+ 双方共享的桥接令牌,两者缺一控制台无法校验登录。
|
||||
REVIEW_BASE_URL=http://xiaobai-review:8765
|
||||
HUB_ADMIN_TOKEN=
|
||||
|
||||
# 浏览器可达的主站地址;留空时按请求 Host 推导 http://<host>:8765。
|
||||
REVIEW_PUBLIC_URL=
|
||||
|
||||
# Tushare Pro token. Stored encrypted after first launch; never returned by API or admin pages.
|
||||
TUSHARE_TOKEN=
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
- 盘中观察(provisional):东财/腾讯指数报价、个股最新价、全市场快照、分时点(`/v1/quotes/latest` 不传 codes 即全市场,`/v1/indexes/quotes` `/v1/intraday/points`);永不写入 eod_* 正式表
|
||||
- 暂存 → 校验 → 整批原子发布 → 可回滚
|
||||
- `/v1` 稳定接口(`X-Datahub-Token`)
|
||||
- `/admin/` 最小管理后台(总览 / 数据源 / 调度 / 发布 / 数据集 / 审计)
|
||||
- `/admin/` 统一管理控制台(总览 / 数据源配置 / 模型池 / 会员管理 / 数据血缘),日间与夜间两套配色
|
||||
- 同花顺/选股宝/AKShare/iFinD 适配器位仍预留;东财/腾讯已接入盘中观察
|
||||
|
||||
## 单位口径(相对现站)
|
||||
@@ -33,16 +33,43 @@
|
||||
cd xiaobai-datahub
|
||||
python -m venv .venv && .venv/bin/pip install -r requirements.txt
|
||||
cp .env.example .env
|
||||
# 填入 DATAHUB_ENCRYPTION_KEY / DATAHUB_TOKEN / DATAHUB_ADMIN_PASSWORD / TUSHARE_TOKEN
|
||||
# 填入 DATAHUB_ENCRYPTION_KEY / DATAHUB_TOKEN / HUB_ADMIN_TOKEN / TUSHARE_TOKEN
|
||||
# HUB_ADMIN_TOKEN 与主站 .env 同名变量必须一致;控制台没有独立账号,用主站管理员账号登录
|
||||
# 生成 Fernet 密钥:
|
||||
# python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
|
||||
.venv/bin/python server.py --host 127.0.0.1 --port 8766
|
||||
```
|
||||
|
||||
- 管理后台:http://127.0.0.1:8766/admin/
|
||||
- 管理控制台:http://127.0.0.1:8766/admin/
|
||||
- 存活检查:http://127.0.0.1:8766/livez (无需 token)
|
||||
- `/v1/*` 必须带请求头 `X-Datahub-Token`
|
||||
|
||||
## 管理控制台的账号与权限(HEL-560)
|
||||
|
||||
控制台**没有自己的账号体系**,也不再有独立登录页和改密页:
|
||||
|
||||
- 登录状态取自主站 `xiaobai_session` cookie。主站与中枢同主机不同端口,浏览器会自动带上该 cookie,因此在主站登录后直接打开 8766 即可进入;未登录会看到门禁面板并给出主站登录入口。
|
||||
- 仅管理员可进。每个页面与每个 `/admin/api/*` 接口都在服务端校验会话与 `role=admin`,非管理员一律 403,前端隐藏与否不作为权限依据。
|
||||
- 校验方式是服务间桥接:中枢把 cookie 交给主站 `/api/hub-admin/session` 换回用户身份,结果缓存数秒。桥接凭 `HUB_ADMIN_TOKEN`(与主站 `.env` 同名变量必须一致),主站在任何处理器之前先校验它。
|
||||
- CSRF 令牌由会话派生(HMAC),随 `GET /admin/api/session` 下发,写操作必须带 `X-CSRF-Token`。
|
||||
- 回滚、回补等危险操作仍需二次确认密码,校验走主站 `/api/hub-admin/password/check`,中枢不存密码。
|
||||
- 「退出」会请求主站注销该会话并跳回主站登录页。
|
||||
|
||||
需要的环境变量:
|
||||
|
||||
| 变量 | 位置 | 说明 |
|
||||
|---|---|---|
|
||||
| `HUB_ADMIN_TOKEN` | 主站 + 中枢 | 服务间桥接令牌,两侧必须一致,缺失则控制台无法校验会话 |
|
||||
| `REVIEW_BASE_URL` | 中枢 | 中枢访问主站的地址(容器内一般是服务名,如 `http://xiaobai-review:8765`) |
|
||||
| `REVIEW_PUBLIC_URL` | 中枢 | 浏览器可达的主站地址,用于门禁的登录跳转;留空则按当前主机名推导 |
|
||||
|
||||
## 从主站迁入的两块配置
|
||||
|
||||
- **模型池**:按供应商组织(同一 API 地址下可挂多个模型),填好地址与 Key 后可自动拉取 `/models` 勾选纳入;供应商不支持或拉取失败时用卡内「手动录入」兜底。主 / 辅模型分工在「调用编排」里指定。密钥加密存于主站,界面只回显后四位。
|
||||
- **会员与邀请码**:会员开通 / 续期 / 停用、每日调用额度,以及一次性邀请码的生成、复制、作废。注册必须提交有效邀请码,每个码只能成功注册一次(并发提交也只有一个成功)。列表只显示掩码,完整码仅在生成瞬间与「复制」动作中可得。
|
||||
|
||||
数据仍归主站所有(同一个 `review.db`),中枢只是唯一的管理入口;主站页面上原本的模型池与会员管理分区已移除,「数据中枢」按钮指向 8766。
|
||||
|
||||
## Docker(独立 compose,不改现网 review 服务)
|
||||
|
||||
```bash
|
||||
|
||||
+1964
-1400
File diff suppressed because it is too large
Load Diff
@@ -1,77 +1,103 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<html lang="zh-CN" data-theme="night">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>xiaobai-datahub 管理后台 · 数据中枢</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>小白复盘 · 数据中枢</title>
|
||||
<link rel="stylesheet" href="/admin/styles.css" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="app">
|
||||
<section id="login-view" class="panel auth-panel">
|
||||
<span class="badge-sim">内网 · 8766</span>
|
||||
<h1>数据中枢</h1>
|
||||
<p class="muted">四路来源持续汇流、调度、发布与审计的运转空间。</p>
|
||||
<form id="login-form">
|
||||
<label>账号 <input name="username" value="hub_admin" autocomplete="username" /></label>
|
||||
<label>密码 <input name="password" type="password" autocomplete="current-password" /></label>
|
||||
<button type="submit">登录</button>
|
||||
<p id="login-error" class="error" hidden></p>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section id="change-view" class="panel auth-panel" hidden>
|
||||
<h1>修改初始密码</h1>
|
||||
<form id="change-form">
|
||||
<label>当前密码 <input name="current" type="password" /></label>
|
||||
<label>新密码(至少 8 位) <input name="new_password" type="password" /></label>
|
||||
<button type="submit">保存并继续</button>
|
||||
<p id="change-error" class="error" hidden></p>
|
||||
</form>
|
||||
<!-- 数据中枢没有独立账号:门禁只负责把未登录/非管理员引回主站。 -->
|
||||
<div id="gate-view" class="auth-wrap bg-ambient" hidden>
|
||||
<section class="panel gate-panel">
|
||||
<h1 id="gate-title">数据中枢</h1>
|
||||
<p id="gate-desc">正在校验小白复盘主站登录状态…</p>
|
||||
<div class="gate-actions">
|
||||
<a id="gate-login" class="pbtn" href="#" hidden>去主站登录</a>
|
||||
<a id="gate-site" class="tbtn" href="#" hidden>返回小白复盘</a>
|
||||
<button id="gate-retry" class="tbtn" type="button">重新校验</button>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<section id="shell" hidden>
|
||||
<header id="topbar">
|
||||
<div class="logo">小白复盘 <em>·</em> 数据中枢</div>
|
||||
<span class="crumb" id="crumb">8766 · 四源汇流 · 持续运转</span>
|
||||
<span id="phase" class="pill"></span>
|
||||
<span class="spacer"></span>
|
||||
<span id="who" class="muted who"></span>
|
||||
<button type="button" id="theme-btn" class="ghost">夜间</button>
|
||||
<button type="button" id="logout-btn" class="ghost">退出</button>
|
||||
<div class="app bg-ambient" id="appRoot" hidden>
|
||||
<div class="bg-layer bg-gridlines z0"></div>
|
||||
<div class="bg-layer scanlines z50"></div>
|
||||
|
||||
<header class="hdr">
|
||||
<div class="hdr-in">
|
||||
<div class="radar" style="width:26px;height:26px" id="radarLogo">
|
||||
<div class="sweep radar-sweep"></div>
|
||||
<div class="ring1" style="inset:5.72px"></div>
|
||||
<div class="ring2" style="inset:9.88px"></div>
|
||||
<div class="center"></div>
|
||||
<div class="blip pulse-dot" id="radarBlipOk" style="left:64%;top:30%;background:var(--mint)"></div>
|
||||
<div class="blip pulse-dot" id="radarBlipBad" style="left:30%;top:62%;background:var(--rd);animation-delay:.6s;display:none"></div>
|
||||
</div>
|
||||
<div class="brand"><span class="b1">小白复盘 <span style="color:var(--cy)">·</span> 数据中枢</span><span class="b2">DATA-HUB</span></div>
|
||||
<span class="vsep"></span>
|
||||
<nav class="nav" id="navEl">
|
||||
<button class="navbtn" data-nav="overview"><span class="tri">▸</span>运行总览<span class="en">OVERVIEW</span></button>
|
||||
<button class="navbtn" data-nav="sources"><span class="tri">▸</span>数据源配置<span class="en">SOURCES</span></button>
|
||||
<button class="navbtn" data-nav="models"><span class="tri">▸</span>模型池<span class="en">MODELS</span></button>
|
||||
<button class="navbtn" data-nav="members"><span class="tri">▸</span>会员管理<span class="en">MEMBERS</span></button>
|
||||
<button class="navbtn" data-nav="lineage"><span class="tri">▸</span>数据血缘<span class="en">LINEAGE</span></button>
|
||||
</nav>
|
||||
<span class="flex1"></span>
|
||||
<span class="mdtag" id="phaseTag"></span>
|
||||
<span class="livespan" id="liveSpan"><span class="livedot pulse-dot"></span>LIVE</span>
|
||||
<span class="clock num" id="clock"><span id="ckD"></span><span class="csep">|</span><span class="ct"><span id="ckH"></span><span class="blink cc">:</span><span id="ckM"></span><span class="blink cc">:</span><span id="ckS"></span></span></span>
|
||||
<span class="vsep"></span>
|
||||
<button class="tbtn" id="opsBtn" style="font-size:10px">调度 / 发布 / 审计</button>
|
||||
<button class="tbtn" id="themeBtn" style="font-size:10px">日间</button>
|
||||
<button class="tbtn" id="calmBtn" style="font-size:10px">减少动态</button>
|
||||
<span id="who" class="muted" style="font-size:10px"></span>
|
||||
<button class="tbtn" id="logout-btn" style="font-size:10px">退出</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<!-- 动效版:一台连续空间舞台,六幕滚动切换,同一核心与流道贯穿始终 -->
|
||||
<div id="stageWrap">
|
||||
<canvas id="scene"></canvas>
|
||||
<div id="cabin"></div>
|
||||
<div id="legend">
|
||||
<span class="lampico li-link"></span>LINK 链路常亮(低亮)<br>
|
||||
<span class="lampico li-act"></span>ACT 活动灯 · 事件成簇短闪
|
||||
</div>
|
||||
<div id="rail"></div>
|
||||
<div id="hint">滚 动 推 进 镜 头</div>
|
||||
<div id="detail" class="closed">
|
||||
<div class="dtag"></div>
|
||||
<button id="detailClose" type="button" aria-label="关闭详情">×</button>
|
||||
<div id="detailBody"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="track" aria-hidden="true">
|
||||
<section data-scene="0"></section>
|
||||
<section data-scene="1"></section>
|
||||
<section data-scene="2"></section>
|
||||
<section data-scene="3"></section>
|
||||
<section data-scene="4"></section>
|
||||
<section data-scene="5"></section>
|
||||
<main class="main" id="mainEl"></main>
|
||||
|
||||
<footer class="tape">
|
||||
<span class="tape-label" id="tapeLed"><span class="led rev pulse"></span>EVENT TAPE</span>
|
||||
<div class="tape-view"><div class="marquee-track" id="marqueeTrack"><span style="display:inline-flex;align-items:center" id="tapeA"></span><span style="display:inline-flex;align-items:center" id="tapeB"></span></div></div>
|
||||
</footer>
|
||||
</div>
|
||||
|
||||
<!-- 减少动态效果版:六幕静态空间图 + 固定文字状态,信息与功能完全等价 -->
|
||||
<main id="staticShell" hidden>
|
||||
<nav id="staticNav"></nav>
|
||||
<div id="staticScenes"></div>
|
||||
</main>
|
||||
</section>
|
||||
<div class="drawer-mask" id="drawerMask">
|
||||
<aside class="drawer">
|
||||
<div class="drawer-hd">
|
||||
<span class="ttl">调度 / 发布 / 审计</span>
|
||||
<span class="sub" id="drawerSub"></span>
|
||||
<button class="drawer-close" id="drawerClose">×</button>
|
||||
</div>
|
||||
<div class="drawer-tabs" id="drawerTabs">
|
||||
<button class="drawer-tab" data-dtab="jobs">调度任务</button>
|
||||
<button class="drawer-tab" data-dtab="release">盘后发布</button>
|
||||
<button class="drawer-tab" data-dtab="audit">审计</button>
|
||||
</div>
|
||||
<div class="drawer-body" id="drawerBody"></div>
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
<div class="modal-mask" id="modalMask">
|
||||
<div class="modal-box">
|
||||
<h3 id="modalTitle">危险操作确认</h3>
|
||||
<p id="modalDesc"></p>
|
||||
<label>主站账号密码 <input id="modalPassword" type="password" autocomplete="current-password" /></label>
|
||||
<label id="modalConfirmWrap">请输入确认词 <span id="modalConfirmWord" class="num" style="color:var(--amb)"></span> <input id="modalConfirm" type="text" autocomplete="off" /></label>
|
||||
<p class="modal-err" id="modalErr"></p>
|
||||
<div class="modal-actions">
|
||||
<button class="tbtn" id="modalCancel">取消</button>
|
||||
<button class="tbtn danger" id="modalOk">确认执行</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="toast" id="toastEl"></div>
|
||||
|
||||
</div>
|
||||
<script src="/admin/app.js"></script>
|
||||
</body>
|
||||
|
||||
+689
-270
File diff suppressed because one or more lines are too long
@@ -12,7 +12,9 @@ services:
|
||||
environment:
|
||||
DATAHUB_ENCRYPTION_KEY: "${DATAHUB_ENCRYPTION_KEY:?DATAHUB_ENCRYPTION_KEY must be set}"
|
||||
DATAHUB_TOKEN: "${DATAHUB_TOKEN:?DATAHUB_TOKEN must be set}"
|
||||
DATAHUB_ADMIN_PASSWORD: "${DATAHUB_ADMIN_PASSWORD:?DATAHUB_ADMIN_PASSWORD must be set}"
|
||||
HUB_ADMIN_TOKEN: "${HUB_ADMIN_TOKEN:?HUB_ADMIN_TOKEN must be set}"
|
||||
REVIEW_BASE_URL: "${REVIEW_BASE_URL:-http://xiaobai-review:8765}"
|
||||
REVIEW_PUBLIC_URL: "${REVIEW_PUBLIC_URL:-}"
|
||||
TUSHARE_TOKEN: "${TUSHARE_TOKEN:-}"
|
||||
IFIND_REFRESH_TOKEN: "${IFIND_REFRESH_TOKEN:-}"
|
||||
IFIND_ACCESS_TOKEN: "${IFIND_ACCESS_TOKEN:-}"
|
||||
|
||||
@@ -6,19 +6,33 @@ from typing import Any
|
||||
from datahub.adapters import RESERVED
|
||||
from datahub.auth import AuthService
|
||||
from datahub.db import HubDB
|
||||
from datahub import lineage as lineage_module
|
||||
from datahub import observability
|
||||
from datahub.pipeline import OFFICIAL_DATASETS, STOCKS_DATASET, Pipeline
|
||||
from datahub.scheduler import Scheduler
|
||||
from datahub.serving import ApiError
|
||||
from datahub import source_catalog
|
||||
from datahub.timeutil import isoformat, now_shanghai, session_phase, yyyymmdd
|
||||
|
||||
|
||||
class AdminAPI:
|
||||
def __init__(self, db: HubDB, pipeline: Pipeline, scheduler: Scheduler, auth: AuthService, ifind: Any = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
db: HubDB,
|
||||
pipeline: Pipeline,
|
||||
scheduler: Scheduler,
|
||||
auth: AuthService,
|
||||
ifind: Any = None,
|
||||
site_auth: Any = None,
|
||||
) -> None:
|
||||
self.db = db
|
||||
self.pipeline = pipeline
|
||||
self.scheduler = scheduler
|
||||
self.auth = auth
|
||||
self.ifind = ifind
|
||||
# HEL-560: dangerous operations confirm against the review site account
|
||||
# that is driving the console, not against a console-local password.
|
||||
self.site_auth = site_auth
|
||||
|
||||
def overview(self) -> dict[str, Any]:
|
||||
today = yyyymmdd(now_shanghai())
|
||||
@@ -28,10 +42,18 @@ class AdminAPI:
|
||||
)
|
||||
is_open = bool(cal and int(cal["is_open"]) == 1)
|
||||
pubs = self.db.fetchall("SELECT * FROM publications WHERE trade_date = ?", (today,))
|
||||
failed = self.db.fetchall(
|
||||
"SELECT * FROM batches WHERE trade_date = ? AND state IN ('failed','staged')",
|
||||
(today,),
|
||||
)
|
||||
# HEL-529 fix 2: "待处理"只保留当前仍未恢复的最新异常。一条失败/停滞
|
||||
# 批次若已被同数据集更晚的成功批次或当日发布解决,就不再是当前故障,
|
||||
# 历史记录仍完整保留在 batches/audit 明细里,不在此重复展示。
|
||||
latest_by_dataset: dict[str, dict[str, Any]] = {}
|
||||
for row in self.db.fetchall("SELECT * FROM batches WHERE trade_date = ? ORDER BY started_at, batch_id", (today,)):
|
||||
latest_by_dataset[row["dataset"]] = row
|
||||
published_datasets = {row["dataset"] for row in pubs if row["state"] == "published"}
|
||||
anomalies = [
|
||||
row
|
||||
for dataset, row in latest_by_dataset.items()
|
||||
if dataset not in published_datasets and row["state"] in ("failed", "staged")
|
||||
]
|
||||
calls = self.db.fetchall(
|
||||
"SELECT * FROM src_calls ORDER BY id DESC LIMIT 20",
|
||||
)
|
||||
@@ -42,7 +64,7 @@ class AdminAPI:
|
||||
"eod_status": self.scheduler.eod_status(today),
|
||||
"revision_status": self.scheduler.revision_status(today),
|
||||
"publications": pubs,
|
||||
"anomalies": failed,
|
||||
"anomalies": anomalies,
|
||||
"recent_calls": _public_calls(calls),
|
||||
"source_count": len(self.db.fetchall("SELECT provider FROM src_health")),
|
||||
}
|
||||
@@ -105,6 +127,53 @@ class AdminAPI:
|
||||
raise ApiError("INVALID_ARGUMENT", f"unknown provider: {provider}")
|
||||
return adapter.probe()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# HEL-543: read-only side-channel status/catalog/lineage. These never
|
||||
# change routing, credentials, or adapters; they only read the
|
||||
# provider_call_log/provider_health tables (observability.py) plus the
|
||||
# static registries in source_catalog.py / lineage.py.
|
||||
# ------------------------------------------------------------------
|
||||
def providers_status(self, provider: str = "", limit: int = 50) -> dict[str, Any]:
|
||||
if not observability.is_enabled(self.db):
|
||||
return {"enabled": False, "health": [], "recent_calls": []}
|
||||
limit = max(1, min(int(limit or 50), 200))
|
||||
health_sql = "SELECT * FROM provider_health"
|
||||
params: tuple[Any, ...] = ()
|
||||
if provider:
|
||||
health_sql += " WHERE provider = ?"
|
||||
params = (provider,)
|
||||
health_sql += " ORDER BY provider, interface"
|
||||
health = self.db.fetchall(health_sql, params)
|
||||
calls_sql = "SELECT * FROM provider_call_log"
|
||||
if provider:
|
||||
calls_sql += " WHERE provider = ?"
|
||||
calls_sql += " ORDER BY id DESC LIMIT ?"
|
||||
recent = self.db.fetchall(calls_sql, (*params, limit))
|
||||
return {"enabled": True, "health": health, "recent_calls": recent}
|
||||
|
||||
def source_catalog(self) -> dict[str, Any]:
|
||||
if not observability.is_enabled(self.db):
|
||||
return {"enabled": False, "items": []}
|
||||
return {"enabled": True, "items": source_catalog.snapshot(self.db, self.auth)}
|
||||
|
||||
def lineage(self, trade_date: str = "") -> dict[str, Any]:
|
||||
day = yyyymmdd(trade_date) if trade_date else yyyymmdd(now_shanghai())
|
||||
if not observability.is_enabled(self.db):
|
||||
return {"enabled": False, "trade_date": day, "items": []}
|
||||
return {"enabled": True, "trade_date": day, "items": lineage_module.snapshot(self.db, day)}
|
||||
|
||||
def lineage_affected(self, provider: str = "", interface: str = "") -> dict[str, Any]:
|
||||
if not provider:
|
||||
raise ApiError("INVALID_ARGUMENT", "provider is required")
|
||||
if not observability.is_enabled(self.db):
|
||||
return {"enabled": False, "provider": provider, "interface": interface, "items": []}
|
||||
return {
|
||||
"enabled": True,
|
||||
"provider": provider,
|
||||
"interface": interface,
|
||||
"items": lineage_module.affected(self.db, provider, interface),
|
||||
}
|
||||
|
||||
def jobs(self) -> dict[str, Any]:
|
||||
runs = self.db.fetchall("SELECT * FROM job_runs ORDER BY id DESC LIMIT 100")
|
||||
stocks_times = "/".join(self.pipeline.settings.stocks_refresh_times) or "20:00"
|
||||
@@ -153,18 +222,34 @@ class AdminAPI:
|
||||
def audit(self) -> dict[str, Any]:
|
||||
return {"items": self.db.fetchall("SELECT * FROM audit_log ORDER BY id DESC LIMIT 200")}
|
||||
|
||||
def rollback(self, dataset: str, trade_date: str, password: str, confirm: str, actor: str) -> dict[str, Any]:
|
||||
self._dangerous(password, confirm, f"{dataset}:{trade_date}")
|
||||
def rollback(
|
||||
self,
|
||||
dataset: str,
|
||||
trade_date: str,
|
||||
password: str,
|
||||
confirm: str,
|
||||
actor: str,
|
||||
actor_id: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
self._dangerous(password, confirm, f"{dataset}:{trade_date}", actor_id)
|
||||
result = self.pipeline.rollback(dataset, trade_date, actor=actor)
|
||||
return result
|
||||
|
||||
def backfill(self, dataset: str, trade_date: str, password: str, confirm: str, actor: str) -> dict[str, Any]:
|
||||
def backfill(
|
||||
self,
|
||||
dataset: str,
|
||||
trade_date: str,
|
||||
password: str,
|
||||
confirm: str,
|
||||
actor: str,
|
||||
actor_id: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
day = yyyymmdd(trade_date or now_shanghai())
|
||||
if dataset == "history":
|
||||
self._dangerous(password, confirm, "history:full")
|
||||
self._dangerous(password, confirm, "history:full", actor_id)
|
||||
result = self.pipeline.backfill_history(day)
|
||||
else:
|
||||
self._dangerous(password, confirm, f"{dataset}:{day}")
|
||||
self._dangerous(password, confirm, f"{dataset}:{day}", actor_id)
|
||||
if dataset == "reference":
|
||||
result = self.pipeline.ingest_reference(day)
|
||||
elif dataset in OFFICIAL_DATASETS or dataset == STOCKS_DATASET:
|
||||
@@ -186,12 +271,18 @@ class AdminAPI:
|
||||
self.pipeline.audit(actor, "backfill", f"{dataset}:{day}", json.dumps({"ok": True}))
|
||||
return result
|
||||
|
||||
def _dangerous(self, password: str, confirm: str, expected: str) -> None:
|
||||
if not self.auth.confirm_password(password):
|
||||
def _dangerous(self, password: str, confirm: str, expected: str, actor_id: int = 0) -> None:
|
||||
if not self._confirm_password(actor_id, password):
|
||||
raise ApiError("UNAUTHORIZED", "二次确认密码错误")
|
||||
if confirm.strip() != expected:
|
||||
raise ApiError("INVALID_ARGUMENT", f"确认词必须为 {expected}")
|
||||
|
||||
def _confirm_password(self, actor_id: int, password: str) -> bool:
|
||||
"""Second factor for destructive ops: the operator's review-site password."""
|
||||
if self.site_auth is None:
|
||||
raise ApiError("UNAVAILABLE", "主站桥接未配置,无法校验管理员口令")
|
||||
return bool(self.site_auth.confirm_password(actor_id, password))
|
||||
|
||||
|
||||
def _public_calls(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
out = []
|
||||
|
||||
+10
-124
@@ -1,39 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import secrets
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from datahub.crypto import SecretVault, mask_secret
|
||||
from datahub.db import HubDB
|
||||
from datahub.timeutil import isoformat, now_shanghai
|
||||
from datahub.timeutil import isoformat
|
||||
|
||||
PBKDF2_ROUNDS = 200_000
|
||||
SESSION_HOURS = 12
|
||||
LOGIN_FAIL_LIMIT = 5
|
||||
LOCK_MINUTES = 10
|
||||
|
||||
|
||||
def hash_password(password: str, salt: bytes | None = None) -> tuple[str, str]:
|
||||
raw_salt = salt or os.urandom(16)
|
||||
digest = hashlib.pbkdf2_hmac("sha256", password.encode("utf-8"), raw_salt, PBKDF2_ROUNDS, dklen=32)
|
||||
return (
|
||||
base64.urlsafe_b64encode(raw_salt).decode("ascii"),
|
||||
base64.urlsafe_b64encode(digest).decode("ascii"),
|
||||
)
|
||||
|
||||
|
||||
def verify_password(password: str, salt_text: str, expected_hash: str) -> bool:
|
||||
try:
|
||||
salt = base64.urlsafe_b64decode(salt_text.encode("ascii"))
|
||||
_, actual = hash_password(password, salt)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
return hmac.compare_digest(actual, expected_hash)
|
||||
|
||||
|
||||
def token_hash(token: str) -> str:
|
||||
@@ -41,12 +14,18 @@ def token_hash(token: str) -> str:
|
||||
|
||||
|
||||
class AuthService:
|
||||
def __init__(self, db: HubDB, vault: SecretVault, api_token: str, admin_password: str) -> None:
|
||||
"""Machine credentials only: the `/v1` API token and the provider secrets.
|
||||
|
||||
Operator accounts live on the review site (HEL-560) — the console verifies
|
||||
them through `SiteAuth`, so nothing here authenticates a person.
|
||||
"""
|
||||
|
||||
def __init__(self, db: HubDB, vault: SecretVault, api_token: str) -> None:
|
||||
self.db = db
|
||||
self.vault = vault
|
||||
self._bootstrap(api_token, admin_password)
|
||||
self._bootstrap(api_token)
|
||||
|
||||
def _bootstrap(self, api_token: str, admin_password: str) -> None:
|
||||
def _bootstrap(self, api_token: str) -> None:
|
||||
if api_token:
|
||||
existing = self.db.fetchone("SELECT token_hash FROM api_tokens WHERE name = ?", ("review",))
|
||||
hashed = token_hash(api_token)
|
||||
@@ -61,17 +40,6 @@ class AuthService:
|
||||
"UPDATE api_tokens SET token_hash = ?, last4 = ? WHERE name = ?",
|
||||
(hashed, last4, "review"),
|
||||
)
|
||||
admin = self.db.fetchone("SELECT id FROM hub_admin WHERE username = ?", ("hub_admin",))
|
||||
if admin is None and admin_password:
|
||||
salt, hashed = hash_password(admin_password)
|
||||
now = isoformat()
|
||||
self.db.execute(
|
||||
"""
|
||||
INSERT INTO hub_admin(username, password_salt, password_hash, password_must_change, created_at, updated_at)
|
||||
VALUES (?, ?, ?, 1, ?, ?)
|
||||
""",
|
||||
("hub_admin", salt, hashed, now, now),
|
||||
)
|
||||
|
||||
def check_api_token(self, supplied: str) -> bool:
|
||||
if not supplied:
|
||||
@@ -82,88 +50,6 @@ class AuthService:
|
||||
)
|
||||
return row is not None
|
||||
|
||||
def login(self, username: str, password: str) -> dict[str, Any]:
|
||||
user = self.db.fetchone("SELECT * FROM hub_admin WHERE username = ?", (username,))
|
||||
if not user:
|
||||
raise PermissionError("账号或密码错误")
|
||||
now = now_shanghai()
|
||||
locked_until = user.get("locked_until")
|
||||
if locked_until:
|
||||
try:
|
||||
from datetime import datetime
|
||||
|
||||
if datetime.fromisoformat(str(locked_until)) > now:
|
||||
raise PermissionError("账号已锁定,请稍后再试")
|
||||
except ValueError:
|
||||
pass
|
||||
if not verify_password(password, str(user["password_salt"]), str(user["password_hash"])):
|
||||
fails = int(user["failed_attempts"] or 0) + 1
|
||||
lock = isoformat(now + timedelta(minutes=LOCK_MINUTES)) if fails >= LOGIN_FAIL_LIMIT else None
|
||||
self.db.execute(
|
||||
"UPDATE hub_admin SET failed_attempts = ?, locked_until = ? WHERE id = ?",
|
||||
(fails, lock, user["id"]),
|
||||
)
|
||||
raise PermissionError("账号或密码错误")
|
||||
self.db.execute(
|
||||
"UPDATE hub_admin SET failed_attempts = 0, locked_until = NULL WHERE id = ?",
|
||||
(user["id"],),
|
||||
)
|
||||
session = secrets.token_urlsafe(32)
|
||||
csrf = secrets.token_urlsafe(24)
|
||||
expires = isoformat(now + timedelta(hours=SESSION_HOURS))
|
||||
self.db.execute(
|
||||
"INSERT INTO hub_sessions(token_hash, csrf_token, expires_at, created_at) VALUES (?,?,?,?)",
|
||||
(token_hash(session), csrf, expires, isoformat(now)),
|
||||
)
|
||||
return {
|
||||
"session": session,
|
||||
"csrf": csrf,
|
||||
"must_change": bool(user["password_must_change"]),
|
||||
"expires_at": expires,
|
||||
}
|
||||
|
||||
def session_user(self, raw_token: str) -> dict[str, Any] | None:
|
||||
if not raw_token:
|
||||
return None
|
||||
row = self.db.fetchone(
|
||||
"SELECT * FROM hub_sessions WHERE token_hash = ?",
|
||||
(token_hash(raw_token),),
|
||||
)
|
||||
if not row:
|
||||
return None
|
||||
if str(row["expires_at"]) < isoformat():
|
||||
self.db.execute("DELETE FROM hub_sessions WHERE token_hash = ?", (row["token_hash"],))
|
||||
return None
|
||||
admin = self.db.fetchone("SELECT username, password_must_change FROM hub_admin WHERE username = ?", ("hub_admin",))
|
||||
return {
|
||||
"username": (admin or {}).get("username") or "hub_admin",
|
||||
"csrf_token": row["csrf_token"],
|
||||
"must_change": bool((admin or {}).get("password_must_change")),
|
||||
"token_hash": row["token_hash"],
|
||||
}
|
||||
|
||||
def logout(self, raw_token: str) -> None:
|
||||
if raw_token:
|
||||
self.db.execute("DELETE FROM hub_sessions WHERE token_hash = ?", (token_hash(raw_token),))
|
||||
|
||||
def change_password(self, current: str, new_password: str) -> None:
|
||||
if len(new_password) < 8:
|
||||
raise ValueError("新密码至少 8 位")
|
||||
user = self.db.fetchone("SELECT * FROM hub_admin WHERE username = ?", ("hub_admin",))
|
||||
if not user or not verify_password(current, str(user["password_salt"]), str(user["password_hash"])):
|
||||
raise PermissionError("当前密码错误")
|
||||
salt, hashed = hash_password(new_password)
|
||||
self.db.execute(
|
||||
"UPDATE hub_admin SET password_salt=?, password_hash=?, password_must_change=0, updated_at=? WHERE id=?",
|
||||
(salt, hashed, isoformat(), user["id"]),
|
||||
)
|
||||
|
||||
def confirm_password(self, password: str) -> bool:
|
||||
user = self.db.fetchone("SELECT * FROM hub_admin WHERE username = ?", ("hub_admin",))
|
||||
if not user:
|
||||
return False
|
||||
return verify_password(password, str(user["password_salt"]), str(user["password_hash"]))
|
||||
|
||||
def credential_status(self, name: str) -> dict[str, Any]:
|
||||
row = self.db.fetchone("SELECT last4, updated_at FROM credentials WHERE name = ?", (name,))
|
||||
if not row:
|
||||
|
||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from datahub.datasets_ext import EXTENDED_DATASET_TABLES, EXTENDED_SCHEMA
|
||||
from datahub.observability import OBS_SCHEMA
|
||||
from datahub.timeutil import isoformat
|
||||
|
||||
_BASE_SCHEMA = """
|
||||
@@ -23,24 +24,7 @@ CREATE TABLE IF NOT EXISTS credentials (
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS hub_admin (
|
||||
id INTEGER PRIMARY KEY,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
password_salt TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
password_must_change INTEGER NOT NULL DEFAULT 1,
|
||||
failed_attempts INTEGER NOT NULL DEFAULT 0,
|
||||
locked_until TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS hub_sessions (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
csrf_token TEXT NOT NULL,
|
||||
expires_at TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS api_tokens (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
@@ -296,7 +280,7 @@ CREATE INDEX IF NOT EXISTS idx_eod_bars_date ON eod_bars(trade_date, batch_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_calendar_open ON trade_calendar(is_open, cal_date);
|
||||
"""
|
||||
|
||||
SCHEMA = _BASE_SCHEMA + EXTENDED_SCHEMA
|
||||
SCHEMA = _BASE_SCHEMA + EXTENDED_SCHEMA + OBS_SCHEMA
|
||||
|
||||
DATASET_TABLES = {
|
||||
"daily": ("eod_bars", "staging_bars"),
|
||||
|
||||
@@ -2,7 +2,6 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import mimetypes
|
||||
import secrets
|
||||
from http import HTTPStatus
|
||||
from http.cookies import SimpleCookie
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
@@ -12,9 +11,9 @@ from urllib.parse import unquote, urlparse
|
||||
from datahub.hub import Hub
|
||||
from datahub.logutil import configure_logging, get_logger
|
||||
from datahub.serving import ApiError, parse_query
|
||||
from datahub.siteauth import SITE_SESSION_COOKIE, SiteBridgeError
|
||||
|
||||
LOGGER = get_logger()
|
||||
SESSION_COOKIE = "datahub_session"
|
||||
|
||||
|
||||
class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
@@ -85,43 +84,74 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
self._json(payload, HTTPStatus.OK)
|
||||
|
||||
def _admin_api(self, method: str, path: str) -> None:
|
||||
if path == "/admin/api/login" and method == "POST":
|
||||
body = self._read_json()
|
||||
result = self.hub.auth.login(str(body.get("username") or "hub_admin"), str(body.get("password") or ""))
|
||||
session_token = self._cookie_value(SITE_SESSION_COOKIE)
|
||||
user = self._site_user(session_token)
|
||||
if user is None:
|
||||
if path == "/admin/api/session" and method == "GET":
|
||||
self._json(
|
||||
{"ok": True, "must_change": result["must_change"], "csrf": result["csrf"]},
|
||||
HTTPStatus.OK,
|
||||
extra_headers=[self._cookie(result["session"])],
|
||||
{"authenticated": False, "login_url": self._login_url()},
|
||||
HTTPStatus.UNAUTHORIZED,
|
||||
)
|
||||
return
|
||||
user = self.hub.auth.session_user(self._cookie_value(SESSION_COOKIE))
|
||||
if not user:
|
||||
raise ApiError("UNAUTHORIZED", "请先登录")
|
||||
if method == "POST" and path != "/admin/api/login":
|
||||
csrf = self.headers.get("X-CSRF-Token", "")
|
||||
if not csrf or not secrets.compare_digest(csrf, str(user["csrf_token"])):
|
||||
raise ApiError("UNAUTHORIZED", "请先在小白复盘主站登录")
|
||||
if not user["is_admin"]:
|
||||
self._json(
|
||||
{
|
||||
"error": {"code": "PERMISSION_DENIED", "message": "数据中枢仅管理员可进入"},
|
||||
"authenticated": True,
|
||||
"is_admin": False,
|
||||
"username": user["username"],
|
||||
},
|
||||
HTTPStatus.FORBIDDEN,
|
||||
)
|
||||
return
|
||||
if method == "POST" and not self.hub.site_auth.check_csrf(session_token, self.headers.get("X-CSRF-Token", "")):
|
||||
raise ApiError("UNAUTHORIZED", "CSRF 校验失败")
|
||||
if path == "/admin/api/logout" and method == "POST":
|
||||
self.hub.auth.logout(self._cookie_value(SESSION_COOKIE))
|
||||
self._json({"ok": True}, HTTPStatus.OK, extra_headers=[self._cookie("", clear=True)])
|
||||
return
|
||||
if path == "/admin/api/session" and method == "GET":
|
||||
self._json({"username": user["username"], "must_change": user["must_change"], "csrf": user["csrf_token"]}, HTTPStatus.OK)
|
||||
self._json(
|
||||
{
|
||||
"authenticated": True,
|
||||
"is_admin": True,
|
||||
"username": user["username"],
|
||||
"csrf": self.hub.site_auth.csrf_token(session_token),
|
||||
"review_url": self._review_url(),
|
||||
},
|
||||
HTTPStatus.OK,
|
||||
)
|
||||
return
|
||||
if path == "/admin/api/change-password" and method == "POST":
|
||||
body = self._read_json()
|
||||
self.hub.auth.change_password(str(body.get("current") or ""), str(body.get("new_password") or ""))
|
||||
self.hub.pipeline.audit(user["username"], "change_password", "hub_admin", "")
|
||||
self._json({"ok": True}, HTTPStatus.OK)
|
||||
if path == "/admin/api/logout" and method == "POST":
|
||||
self.hub.site_auth.logout(session_token)
|
||||
self.hub.pipeline.audit(user["username"], "logout", "site_session", "")
|
||||
self._json({"ok": True, "login_url": self._login_url()}, HTTPStatus.OK)
|
||||
return
|
||||
if self._console_api(method, path, user):
|
||||
return
|
||||
if user["must_change"] and path not in {"/admin/api/change-password", "/admin/api/session"}:
|
||||
raise ApiError("UNAUTHORIZED", "请先修改初始密码")
|
||||
if path == "/admin/api/overview" and method == "GET":
|
||||
self._json(self.hub.admin.overview(), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/sources" and method == "GET":
|
||||
self._json(self.hub.admin.sources(), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/providers/status" and method == "GET":
|
||||
query = parse_query(urlparse(self.path).query)
|
||||
provider = (query.get("provider") or [""])[0]
|
||||
limit = (query.get("limit") or ["50"])[0]
|
||||
self._json(self.hub.admin.providers_status(provider, int(limit or 50)), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/source-catalog" and method == "GET":
|
||||
self._json(self.hub.admin.source_catalog(), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/lineage" and method == "GET":
|
||||
query = parse_query(urlparse(self.path).query)
|
||||
date = (query.get("date") or [""])[0]
|
||||
self._json(self.hub.admin.lineage(date), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/lineage/affected" and method == "GET":
|
||||
query = parse_query(urlparse(self.path).query)
|
||||
provider = (query.get("provider") or [""])[0]
|
||||
interface = (query.get("interface") or [""])[0]
|
||||
self._json(self.hub.admin.lineage_affected(provider, interface), HTTPStatus.OK)
|
||||
return
|
||||
if path.startswith("/admin/api/sources/") and path.endswith("/probe") and method == "POST":
|
||||
provider = path.split("/")[4]
|
||||
self._json(self.hub.admin.probe(provider), HTTPStatus.OK)
|
||||
@@ -155,6 +185,7 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
str(body.get("password") or ""),
|
||||
str(body.get("confirm") or ""),
|
||||
user["username"],
|
||||
int(user["id"]),
|
||||
)
|
||||
self._json(result, HTTPStatus.OK)
|
||||
return
|
||||
@@ -166,11 +197,73 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
str(body.get("password") or ""),
|
||||
str(body.get("confirm") or ""),
|
||||
user["username"],
|
||||
int(user["id"]),
|
||||
)
|
||||
self._json(result, HTTPStatus.OK)
|
||||
return
|
||||
raise ApiError("INVALID_ARGUMENT", f"unknown admin endpoint: {path}")
|
||||
|
||||
def _site_user(self, session_token: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
return self.hub.site_auth.verify(session_token)
|
||||
except SiteBridgeError as exc:
|
||||
raise ApiError("UNAVAILABLE", str(exc)) from exc
|
||||
|
||||
def _review_url(self) -> str:
|
||||
"""Browser-reachable review site URL.
|
||||
|
||||
In production both services sit on the same host behind different
|
||||
ports, so the console derives the site URL from the Host header the
|
||||
browser used; REVIEW_PUBLIC_URL overrides that when they do not.
|
||||
"""
|
||||
configured = self.hub.settings.review_public_url
|
||||
if configured:
|
||||
return configured.rstrip("/")
|
||||
host = (self.headers.get("Host") or "").split(":")[0] or "127.0.0.1"
|
||||
return f"http://{host}:8765"
|
||||
|
||||
def _login_url(self) -> str:
|
||||
return f"{self._review_url()}/login/"
|
||||
|
||||
def _console_api(self, method: str, path: str, user: dict[str, Any]) -> bool:
|
||||
"""Endpoints backed by the review site: model pool, members, invites.
|
||||
|
||||
Returns True when the request was handled so the caller can fall
|
||||
through to the hub-owned endpoints otherwise.
|
||||
"""
|
||||
console = self.hub.site_console
|
||||
if path == "/admin/api/models" and method == "GET":
|
||||
self._json(console.models(), HTTPStatus.OK)
|
||||
elif path == "/admin/api/models/save" and method == "POST":
|
||||
self._json(console.save_models(self._read_json()), HTTPStatus.OK)
|
||||
elif path == "/admin/api/models/test" and method == "POST":
|
||||
self._json(console.test_model(self._read_json()), HTTPStatus.OK)
|
||||
elif path == "/admin/api/models/fetch" and method == "POST":
|
||||
self._json(console.fetch_models(self._read_json()), HTTPStatus.OK)
|
||||
elif path == "/admin/api/members" and method == "GET":
|
||||
self._json(console.members(), HTTPStatus.OK)
|
||||
elif path == "/admin/api/members/save" and method == "POST":
|
||||
self._json(console.save_member(self._read_json()), HTTPStatus.OK)
|
||||
elif path == "/admin/api/members/quota" and method == "POST":
|
||||
self._json(console.save_quota(self._read_json()), HTTPStatus.OK)
|
||||
elif path == "/admin/api/invites" and method == "GET":
|
||||
self._json(console.invites(), HTTPStatus.OK)
|
||||
elif path == "/admin/api/invites/create" and method == "POST":
|
||||
self._json(console.create_invites(self._read_json(allow_empty=True), int(user["id"])), HTTPStatus.OK)
|
||||
elif path == "/admin/api/invites/revoke" and method == "POST":
|
||||
self._json(console.revoke_invite(self._read_json()), HTTPStatus.OK)
|
||||
elif path == "/admin/api/credentials/tushare" and method == "POST":
|
||||
payload = self.hub.put_tushare_credentials(self._read_json())
|
||||
self.hub.pipeline.audit(user["username"], "store_credential", "tushare_token", "")
|
||||
self._json(payload, HTTPStatus.OK)
|
||||
elif path == "/admin/api/credentials/ifind" and method == "POST":
|
||||
payload = self.hub.put_ifind_credentials(self._read_json())
|
||||
self.hub.pipeline.audit(user["username"], "store_credential", "ifind_tokens", "")
|
||||
self._json(payload, HTTPStatus.OK)
|
||||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _admin_static(self, path: str) -> None:
|
||||
relative = path[len("/admin"):].lstrip("/") or "index.html"
|
||||
candidate = (self.hub.static_dir / relative).resolve()
|
||||
@@ -219,10 +312,6 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
morsel = cookie.get(name)
|
||||
return morsel.value if morsel else ""
|
||||
|
||||
def _cookie(self, value: str, clear: bool = False) -> str:
|
||||
max_age = 0 if clear else 12 * 3600
|
||||
return f"{SESSION_COOKIE}={value}; Path=/; HttpOnly; SameSite=Strict; Max-Age={max_age}"
|
||||
|
||||
def _json(self, payload: dict[str, Any], status: HTTPStatus, extra_headers: list[str] | None = None) -> None:
|
||||
raw = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
self.send_response(status)
|
||||
|
||||
@@ -16,16 +16,29 @@ from datahub.pipeline import Pipeline
|
||||
from datahub.scheduler import Scheduler
|
||||
from datahub.serving import V1API
|
||||
from datahub.settings import Settings, load_settings
|
||||
from datahub.siteauth import SiteAuth, SiteBridge
|
||||
from datahub.siteconsole import SiteConsole
|
||||
|
||||
|
||||
class Hub:
|
||||
def __init__(self, settings: Settings, adapter: TushareAdapter | None = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
settings: Settings,
|
||||
adapter: TushareAdapter | None = None,
|
||||
site_auth: SiteAuth | None = None,
|
||||
) -> None:
|
||||
if not settings.encryption_key:
|
||||
raise SystemExit("DATAHUB_ENCRYPTION_KEY 未配置")
|
||||
self.settings = settings
|
||||
self.db = HubDB(settings.db_path)
|
||||
# HEL-543: carry the observability kill switch on the db handle so
|
||||
# every call site that already threads `db` through (pipeline,
|
||||
# realtime_serve, steward, admin_api) picks it up for free with no
|
||||
# extra plumbing. Missing this attribute (e.g. a bare HubDB built
|
||||
# directly in tests) defaults to enabled — see observability.is_enabled.
|
||||
self.db.observability_enabled = settings.observability_enabled
|
||||
self.vault = SecretVault(settings.encryption_key)
|
||||
self.auth = AuthService(self.db, self.vault, settings.api_token, settings.admin_password)
|
||||
self.auth = AuthService(self.db, self.vault, settings.api_token)
|
||||
token = settings.tushare_token or self.auth.load_credential("tushare_token")
|
||||
if settings.tushare_token:
|
||||
self.auth.store_credential("tushare_token", settings.tushare_token)
|
||||
@@ -50,9 +63,36 @@ class Hub:
|
||||
self.lkg = LastKnownGood(self.db)
|
||||
self.scheduler = Scheduler(self.db, self.pipeline)
|
||||
self.api = V1API(self.db, self.pipeline, settings, ifind=self.ifind)
|
||||
self.admin = AdminAPI(self.db, self.pipeline, self.scheduler, self.auth, ifind=self.ifind)
|
||||
self.site_bridge = SiteBridge(settings.review_base_url, settings.hub_admin_token)
|
||||
# Tests inject a stub so the console gate can run without a live site.
|
||||
self.site_auth = site_auth or SiteAuth(self.site_bridge, settings.encryption_key)
|
||||
self.site_console = SiteConsole(self.site_bridge)
|
||||
self.admin = AdminAPI(
|
||||
self.db,
|
||||
self.pipeline,
|
||||
self.scheduler,
|
||||
self.auth,
|
||||
ifind=self.ifind,
|
||||
site_auth=self.site_auth,
|
||||
)
|
||||
self.static_dir = Path(__file__).resolve().parents[1] / "admin"
|
||||
|
||||
def put_tushare_credentials(self, body: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""Store and hot-swap the Tushare token without a restart.
|
||||
|
||||
The adapter holds the token in memory, so writing the credential and
|
||||
pushing it into the live adapter in one step is what makes the save take
|
||||
effect for the next fetch instead of the next deploy.
|
||||
"""
|
||||
from datahub.serving import envelope
|
||||
|
||||
token = str((body or {}).get("tushare_token") or "").strip()
|
||||
if not token:
|
||||
raise ValueError("Tushare Token 不能为空")
|
||||
self.auth.store_credential("tushare_token", token)
|
||||
self.adapter.token = token
|
||||
return envelope(self.adapter.probe(), {"source": "tushare"})
|
||||
|
||||
def put_ifind_credentials(self, body: dict[str, Any] | None) -> dict[str, Any]:
|
||||
from datahub.serving import envelope
|
||||
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
"""Read-only lineage/impact inventory (HEL-543).
|
||||
|
||||
Answers, without changing any routing decision: for a given main-site data
|
||||
item, which datahub dataset backs it, which provider/interface currently
|
||||
serves it (primary and backup), and — when a provider/interface is
|
||||
unhealthy — which datasets and, best-effort, which main-site consumers are
|
||||
affected.
|
||||
|
||||
Every row cites where it was verified so a reviewer does not have to trust
|
||||
a paraphrase:
|
||||
|
||||
- ``v1_endpoint``/``primary_source``/``backup_source`` are taken verbatim
|
||||
from ``datahub/serving.py`` (the ``source=`` string literal passed to
|
||||
``_published_rows``/``_official_meta``) or from the provider/interface
|
||||
pairs wired into ``datahub/realtime_serve.py`` for HEL-543.
|
||||
- ``known_consumers`` lists only call sites this round actually found via
|
||||
code search in the ``xiaobai-review`` website tree (cited as
|
||||
``file:line`` in the comment above each dataset). Anything not backed by
|
||||
a citation is left out rather than guessed; a fuller page-by-page map is
|
||||
tracked separately (HEL-549) and can extend this table later without
|
||||
touching its shape.
|
||||
|
||||
This module never talks to a provider and never mutates anything; it only
|
||||
reads ``provider_health``/``provider_call_log`` (HEL-543) and the existing
|
||||
``publications``/``batches`` tables to attach live status to each row.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
# Verified against backend/features/screener/data_sync.py (calendar,
|
||||
# stock_basic, daily, daily_basic, index_daily-as-benchmark, stk_auction,
|
||||
# moneyflow, ths_hot, dc_hot all called via `self.client.query(...)`) and
|
||||
# backend/features/heaven/market_context.py (stock_basic, index_daily via
|
||||
# `self._tushare_client().query(...)` / `client.query(...)`).
|
||||
DATASETS: list[dict[str, Any]] = [
|
||||
{
|
||||
"dataset": "calendar",
|
||||
"tier": "official",
|
||||
"update_freq": "每日 08:45 预检",
|
||||
"v1_endpoint": "/v1/calendar",
|
||||
"primary_source": "tushare:trade_cal",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(data_sync.py 交易日历解析)", "问天(交易日推算)"],
|
||||
},
|
||||
{
|
||||
"dataset": "stocks",
|
||||
"tier": "official",
|
||||
"update_freq": "每日 20:00 / 23:10",
|
||||
"v1_endpoint": "/v1/stocks",
|
||||
"primary_source": "tushare:stock_basic",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(股票主档)", "问天(market_context.py 股票代码/名称解析)"],
|
||||
},
|
||||
{
|
||||
"dataset": "daily",
|
||||
"tier": "official",
|
||||
"update_freq": "盘后 15:05 · 重试至 23:30",
|
||||
"v1_endpoint": "/v1/bars/daily",
|
||||
"primary_source": "tushare:daily",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(data_sync.py 日K因子)", "交易复盘/个股详情日K图表"],
|
||||
},
|
||||
{
|
||||
"dataset": "valuation",
|
||||
"tier": "official",
|
||||
"update_freq": "盘后 15:05 · 复核 20:00",
|
||||
"v1_endpoint": "/v1/valuation",
|
||||
"primary_source": "tushare:daily_basic",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(data_sync.py 估值因子)"],
|
||||
},
|
||||
{
|
||||
"dataset": "moneyflow",
|
||||
"tier": "official",
|
||||
"update_freq": "盘后 15:05 · 重试至 23:30",
|
||||
"v1_endpoint": "/v1/moneyflow",
|
||||
"primary_source": "tushare:moneyflow",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(data_sync.py 资金流因子)", "个股详情资金流"],
|
||||
},
|
||||
{
|
||||
"dataset": "auction",
|
||||
"tier": "official",
|
||||
"update_freq": "盘后 15:05",
|
||||
"v1_endpoint": "/v1/auction",
|
||||
"primary_source": "tushare:stk_auction",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(data_sync.py 竞价快照)", "竞价板块"],
|
||||
},
|
||||
{
|
||||
"dataset": "index_daily",
|
||||
"tier": "official",
|
||||
"update_freq": "盘后 15:10 · 重试至 23:30",
|
||||
"v1_endpoint": "/v1/indexes/bars",
|
||||
"primary_source": "tushare:index_daily",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["问天(market_context.py 指数近20日走势)", "智能选股(基准回看)"],
|
||||
},
|
||||
{
|
||||
"dataset": "limit_events",
|
||||
"tier": "official",
|
||||
"update_freq": "盘后 16:40 · 重试至 23:30",
|
||||
"v1_endpoint": "/v1/limit-events",
|
||||
"primary_source": "tushare:limit_list_d",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["涨停梯队(历史/盘后视图)"],
|
||||
},
|
||||
{
|
||||
"dataset": "popularity",
|
||||
"tier": "official",
|
||||
"update_freq": "盘后 22:40",
|
||||
"v1_endpoint": "/v1/popularity",
|
||||
"primary_source": "tushare:ths_hot+dc_hot",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["人气榜", "智能选股(data_sync.py 人气因子)"],
|
||||
},
|
||||
{
|
||||
"dataset": "dragon_tiger",
|
||||
"tier": "official",
|
||||
"update_freq": "盘后 16:45 · 重试至 23:30",
|
||||
"v1_endpoint": "/v1/dragon-tiger",
|
||||
"primary_source": "tushare:hm_detail",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["龙虎榜"],
|
||||
},
|
||||
{
|
||||
"dataset": "sector_daily",
|
||||
"tier": "official",
|
||||
"update_freq": "盘后 15:20 · 重试至 23:30",
|
||||
"v1_endpoint": "/v1/sectors",
|
||||
"primary_source": "tushare:ths_daily+dc_index+sw_daily",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["主题轮动", "板块梯队"],
|
||||
},
|
||||
{
|
||||
"dataset": "quotes_latest",
|
||||
"tier": "provisional",
|
||||
"update_freq": "盘中 · 缓存 60s",
|
||||
"v1_endpoint": "/v1/quotes/latest",
|
||||
"primary_source": "eastmoney:ulist/clist",
|
||||
"backup_source": "tencent:qt",
|
||||
"known_consumers": ["竞价/股票池盘中价格", "情绪周期盘中快照"],
|
||||
},
|
||||
{
|
||||
"dataset": "index_quotes",
|
||||
"tier": "provisional",
|
||||
"update_freq": "盘中 · 缓存 60s",
|
||||
"v1_endpoint": "/v1/indexes/quotes",
|
||||
"primary_source": "eastmoney:ulist",
|
||||
"backup_source": "tencent:qt",
|
||||
"known_consumers": ["首页大盘指数条"],
|
||||
},
|
||||
{
|
||||
"dataset": "sectors_quote",
|
||||
"tier": "provisional",
|
||||
"update_freq": "盘中 · 缓存 60s",
|
||||
"v1_endpoint": "/v1/sectors/quote",
|
||||
"primary_source": "eastmoney:sw",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["主题轮动盘中板块报价"],
|
||||
},
|
||||
{
|
||||
"dataset": "limit_pool",
|
||||
"tier": "provisional",
|
||||
"update_freq": "盘中 · 缓存 60s",
|
||||
"v1_endpoint": "/v1/limit-pool",
|
||||
"primary_source": "eastmoney:zt_pool",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["涨停梯队盘中视图"],
|
||||
},
|
||||
{
|
||||
"dataset": "intraday_points",
|
||||
"tier": "provisional",
|
||||
"update_freq": "盘中 · 缓存 20s",
|
||||
"v1_endpoint": "/v1/intraday/points",
|
||||
"primary_source": "eastmoney:trends2",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["个股详情分时图"],
|
||||
},
|
||||
{
|
||||
"dataset": "ifind_wencai",
|
||||
"tier": "licensed",
|
||||
"update_freq": "按需调用",
|
||||
"v1_endpoint": "/v1/query (api_name=ifind_wencai)",
|
||||
"primary_source": "ifind:wencai",
|
||||
"backup_source": None,
|
||||
# Real call site: backend/features/pools/service.py:82 (wencai(query,"stock"))
|
||||
# feeds 股票池's ifind_event_enrichment_v1 (涨停/炸板/跌停原因、首末
|
||||
# 涨停时间、开板次数). 问师 does NOT call wencai anywhere — its main
|
||||
# body is LLM + 主站市场快照 (mentor/service.py builds context from
|
||||
# dashboard/popularity snapshots only).
|
||||
"known_consumers": ["股票池(涨停/炸板/跌停事件补充 enrichment,需 iFinD 凭证)"],
|
||||
},
|
||||
{
|
||||
"dataset": "ifind_history",
|
||||
"tier": "licensed",
|
||||
"update_freq": "问师问询时按需 · 45 日回看",
|
||||
"v1_endpoint": "/v1/query (api_name=ifind_history)",
|
||||
"primary_source": "ifind:history",
|
||||
"backup_source": None,
|
||||
# Real call site: backend/features/mentor/service.py:433-457
|
||||
# (_mentor_market_matrix → ifind.history(close/volume/amount, 45 日回看)).
|
||||
# Only the trend/macro thinking-model profiles use it, and only when
|
||||
# iFinD is configured — it fails open to [] otherwise. 问师其余子能力
|
||||
# (本体问答/低吸/人气上下文等) 不依赖 iFinD.
|
||||
"known_consumers": [
|
||||
"问师·趋势思维模型(指数动量矩阵,可选)",
|
||||
"问师·宏观思维模型(宽基指数与核心ETF矩阵,可选)",
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
_KNOWN_PROVIDERS_BY_SOURCE_PREFIX = ("tushare", "eastmoney", "tencent", "ifind")
|
||||
|
||||
|
||||
def _providers_for(primary_source: str, backup_source: str | None) -> list[str]:
|
||||
providers: list[str] = []
|
||||
for source in (primary_source, backup_source or ""):
|
||||
for provider in _KNOWN_PROVIDERS_BY_SOURCE_PREFIX:
|
||||
if source.startswith(provider) and provider not in providers:
|
||||
providers.append(provider)
|
||||
return providers
|
||||
|
||||
|
||||
def snapshot(db: Any, trade_date: str = "") -> list[dict[str, Any]]:
|
||||
"""Attach live status to the static lineage table. Read-only; never
|
||||
raises (a per-row status lookup failure just leaves that row's status
|
||||
empty rather than failing the whole snapshot)."""
|
||||
result: list[dict[str, Any]] = []
|
||||
for entry in DATASETS:
|
||||
row = dict(entry)
|
||||
providers = _providers_for(entry["primary_source"], entry.get("backup_source"))
|
||||
row["providers"] = providers
|
||||
live: list[dict[str, Any]] = []
|
||||
try:
|
||||
if db is not None and providers:
|
||||
placeholders = ",".join("?" for _ in providers)
|
||||
live = db.fetchall(
|
||||
f"SELECT provider, interface, state, last_error, last_fallback_reason, "
|
||||
f"consec_failures, updated_at FROM provider_health WHERE provider IN ({placeholders})",
|
||||
tuple(providers),
|
||||
)
|
||||
except Exception:
|
||||
live = []
|
||||
row["live_provider_health"] = live
|
||||
if entry["tier"] == "official":
|
||||
pub = None
|
||||
try:
|
||||
if db is not None and trade_date:
|
||||
pub = db.fetchone(
|
||||
"SELECT dataset, trade_date, state, published_at FROM publications "
|
||||
"WHERE dataset = ? AND trade_date = ?",
|
||||
(entry["dataset"], trade_date),
|
||||
)
|
||||
except Exception:
|
||||
pub = None
|
||||
row["publication"] = pub
|
||||
result.append(row)
|
||||
return result
|
||||
|
||||
|
||||
def affected(db: Any, provider: str = "", interface: str = "") -> list[dict[str, Any]]:
|
||||
"""Read-only: which datasets/pages are impacted by a given provider (and,
|
||||
optionally, a specific interface) right now. Does not change routing."""
|
||||
provider = str(provider or "").strip()
|
||||
interface = str(interface or "").strip()
|
||||
result: list[dict[str, Any]] = []
|
||||
for entry in DATASETS:
|
||||
providers = _providers_for(entry["primary_source"], entry.get("backup_source"))
|
||||
if provider and provider not in providers:
|
||||
continue
|
||||
row = dict(entry)
|
||||
row["providers"] = providers
|
||||
health: list[dict[str, Any]] = []
|
||||
try:
|
||||
if db is not None:
|
||||
if interface:
|
||||
health = db.fetchall(
|
||||
"SELECT provider, interface, state, last_error, last_fallback_reason, "
|
||||
"consec_failures, updated_at FROM provider_health "
|
||||
"WHERE provider = ? AND interface = ?",
|
||||
(provider, interface),
|
||||
)
|
||||
elif providers:
|
||||
placeholders = ",".join("?" for _ in providers)
|
||||
health = db.fetchall(
|
||||
f"SELECT provider, interface, state, last_error, last_fallback_reason, "
|
||||
f"consec_failures, updated_at FROM provider_health WHERE provider IN ({placeholders})",
|
||||
tuple(providers),
|
||||
)
|
||||
except Exception:
|
||||
health = []
|
||||
row["live_provider_health"] = health
|
||||
result.append(row)
|
||||
return result
|
||||
@@ -0,0 +1,319 @@
|
||||
"""Side-channel provider-call observability (HEL-543).
|
||||
|
||||
This module is additive-only and must never change what any existing call
|
||||
returns or raises. It exists purely to answer, after the fact and without
|
||||
touching routing: which provider/interface was called, whether it
|
||||
succeeded, how stale/complete the payload looked, and why a fallback fired.
|
||||
|
||||
Hard rules enforced here:
|
||||
|
||||
- Every public entry point (`observe`, `record_call`) is wrapped so that a
|
||||
database failure, a classifier bug, or any other internal error is
|
||||
swallowed and logged at DEBUG level. It never raises into the caller and
|
||||
never delays/blocks the caller's real data path beyond a best-effort
|
||||
timing measurement.
|
||||
- `observe()` always returns exactly what `fn()` returned, and re-raises
|
||||
exactly what `fn()` raised (same exception object, unmodified). It does
|
||||
not retry, does not change ordering, and does not add new failure modes.
|
||||
- No mock data is ever produced or returned by this module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from typing import Any, Callable, TypeVar
|
||||
|
||||
from datahub.timeutil import isoformat, now_shanghai
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
# Additive-only schema: two new tables, no changes to any existing table.
|
||||
# Merged into datahub.db.SCHEMA the same way EXTENDED_SCHEMA is.
|
||||
OBS_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS provider_call_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
provider TEXT NOT NULL,
|
||||
interface TEXT NOT NULL,
|
||||
fetched_at TEXT NOT NULL,
|
||||
latency_ms INTEGER,
|
||||
status TEXT NOT NULL,
|
||||
error TEXT,
|
||||
fallback_reason TEXT,
|
||||
data_age_seconds INTEGER,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS provider_health (
|
||||
provider TEXT NOT NULL,
|
||||
interface TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
last_ok_at TEXT,
|
||||
last_error TEXT,
|
||||
last_fallback_reason TEXT,
|
||||
consec_failures INTEGER NOT NULL DEFAULT 0,
|
||||
last_latency_ms INTEGER,
|
||||
last_data_age_seconds INTEGER,
|
||||
updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY (provider, interface)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_provider_call_log_created ON provider_call_log(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_provider_call_log_provider ON provider_call_log(provider, interface, created_at);
|
||||
"""
|
||||
|
||||
DEFAULT_STALE_SECONDS = 300
|
||||
|
||||
_BLOCKED_MARKERS = (
|
||||
"<!doctype", "<html", "expecting value", "verify you are human",
|
||||
"unusual traffic", "captcha", "安全验证", "访问异常", "请完成验证",
|
||||
"拦截", "禁止访问", "forbidden",
|
||||
)
|
||||
|
||||
# Matches provider messages like "Eastmoney returned 0/3 indices" or
|
||||
# "Tencent returned 2/3 indices" (see adapters/eastmoney.py, adapters/tencent.py).
|
||||
_COUNT_MISMATCH_RE = re.compile(r"returned (\d+)\s*/\s*(\d+)")
|
||||
|
||||
|
||||
def _logger():
|
||||
from datahub.logutil import get_logger
|
||||
|
||||
return get_logger()
|
||||
|
||||
|
||||
def is_enabled(db: Any) -> bool:
|
||||
"""Runtime kill switch (``Settings.observability_enabled`` /
|
||||
``DATAHUB_OBSERVABILITY``, wired onto the db handle in ``Hub.__init__``).
|
||||
|
||||
Defaults to enabled when the attribute is absent — e.g. a bare ``HubDB``
|
||||
built directly in a test, or any call site that predates HEL-543 — so
|
||||
this can never accidentally disable an existing deployment. Never
|
||||
raises.
|
||||
"""
|
||||
try:
|
||||
return bool(getattr(db, "observability_enabled", True))
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
return True
|
||||
|
||||
|
||||
def classify_error(message: str) -> tuple[str, str]:
|
||||
"""Best-effort, side-reading classification of an exception message.
|
||||
|
||||
Never raises. Unknown shapes fall back to a generic ``error`` status so a
|
||||
classifier miss can never be mistaken for a healthy call.
|
||||
"""
|
||||
try:
|
||||
lower = (message or "").lower()
|
||||
if any(marker in lower for marker in _BLOCKED_MARKERS):
|
||||
return "blocked", "response_looks_like_intercept_page"
|
||||
if "timeout" in lower or "timed out" in lower:
|
||||
return "timeout", "request_timeout"
|
||||
mismatch = _COUNT_MISMATCH_RE.search(lower)
|
||||
if mismatch and int(mismatch.group(1)) == 0:
|
||||
return "empty", "empty_or_incomplete_response"
|
||||
if mismatch:
|
||||
return "degraded", "partial_or_mismatched_response"
|
||||
if "empty" in lower or "no intraday chart data" in lower or "missing" in lower:
|
||||
return "empty", "empty_or_incomplete_response"
|
||||
if "too small" in lower or "incomplete" in lower or "mismatch" in lower:
|
||||
return "degraded", "partial_or_mismatched_response"
|
||||
return "error", ""
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
return "error", ""
|
||||
|
||||
|
||||
def classify_rows(
|
||||
rows: Any,
|
||||
*,
|
||||
required_fields: tuple[str, ...] | None = None,
|
||||
freshness_field: str | None = "quote_time_epoch",
|
||||
max_age_seconds: int = DEFAULT_STALE_SECONDS,
|
||||
) -> tuple[str, str, int | None]:
|
||||
"""Read-only classification of an already-successful payload.
|
||||
|
||||
Only ever called on a value a caller is about to use as-is; this never
|
||||
mutates ``rows`` and a classifier bug always degrades to ``("ok", "",
|
||||
None)`` rather than mislabeling a real success as a failure.
|
||||
"""
|
||||
try:
|
||||
if isinstance(rows, dict):
|
||||
items = [rows] if rows else []
|
||||
elif isinstance(rows, (list, tuple)):
|
||||
items = [item for item in rows if isinstance(item, dict)]
|
||||
else:
|
||||
items = []
|
||||
if not items:
|
||||
return "empty", "no_rows_returned", None
|
||||
if required_fields:
|
||||
missing: set[str] = set()
|
||||
for item in items:
|
||||
for field in required_fields:
|
||||
if item.get(field) in (None, ""):
|
||||
missing.add(field)
|
||||
if missing:
|
||||
return "missing_fields", "missing:" + ",".join(sorted(missing)), None
|
||||
data_age: int | None = None
|
||||
if freshness_field:
|
||||
now_epoch = time.time()
|
||||
ages: list[int] = []
|
||||
for item in items:
|
||||
raw = item.get(freshness_field)
|
||||
try:
|
||||
epoch = int(raw or 0)
|
||||
except (TypeError, ValueError):
|
||||
epoch = 0
|
||||
if epoch > 0:
|
||||
ages.append(max(0, int(now_epoch - epoch)))
|
||||
if ages:
|
||||
data_age = max(ages)
|
||||
if data_age > max_age_seconds:
|
||||
return "stale", "data_age_exceeds_threshold", data_age
|
||||
return "ok", "", data_age
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
return "ok", "", None
|
||||
|
||||
|
||||
def record_call(
|
||||
db: Any,
|
||||
provider: str,
|
||||
interface: str,
|
||||
*,
|
||||
status: str,
|
||||
latency_ms: int | None = None,
|
||||
error: str = "",
|
||||
fallback_reason: str = "",
|
||||
data_age_seconds: int | None = None,
|
||||
) -> None:
|
||||
"""Fail-open recorder. Never raises; a write failure here must never be
|
||||
able to take down a real, otherwise-successful data path."""
|
||||
if db is None or not is_enabled(db):
|
||||
return
|
||||
try:
|
||||
now = isoformat(now_shanghai())
|
||||
ok = status == "ok"
|
||||
error_text = (error or "")[:500]
|
||||
reason_text = (fallback_reason or "")[:200]
|
||||
with db.write() as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO provider_call_log("
|
||||
"provider, interface, fetched_at, latency_ms, status, error, "
|
||||
"fallback_reason, data_age_seconds, created_at) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
(provider, interface, now, latency_ms, status, error_text, reason_text, data_age_seconds, now),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO provider_health(
|
||||
provider, interface, state, last_ok_at, last_error, last_fallback_reason,
|
||||
consec_failures, last_latency_ms, last_data_age_seconds, updated_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(provider, interface) DO UPDATE SET
|
||||
state = excluded.state,
|
||||
last_ok_at = CASE WHEN excluded.state = 'ok' THEN excluded.last_ok_at ELSE provider_health.last_ok_at END,
|
||||
last_error = CASE WHEN excluded.state = 'ok' THEN '' ELSE excluded.last_error END,
|
||||
last_fallback_reason = excluded.last_fallback_reason,
|
||||
consec_failures = CASE WHEN excluded.state = 'ok' THEN 0 ELSE provider_health.consec_failures + 1 END,
|
||||
last_latency_ms = excluded.last_latency_ms,
|
||||
last_data_age_seconds = excluded.last_data_age_seconds,
|
||||
updated_at = excluded.updated_at
|
||||
""",
|
||||
(
|
||||
provider,
|
||||
interface,
|
||||
status,
|
||||
now if ok else None,
|
||||
"" if ok else (error_text or reason_text or "unknown_error"),
|
||||
reason_text,
|
||||
0 if ok else 1,
|
||||
latency_ms,
|
||||
data_age_seconds,
|
||||
now,
|
||||
),
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
try:
|
||||
_logger().debug(
|
||||
"observability record_call failed (fail-open)",
|
||||
extra={"hub": {"provider": provider, "interface": interface}},
|
||||
exc_info=True,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _safe_record_failure(db: Any, provider: str, interface: str, latency_ms: int, exc: BaseException) -> None:
|
||||
if db is None:
|
||||
return
|
||||
try:
|
||||
message = str(exc)
|
||||
status, reason = classify_error(message)
|
||||
record_call(
|
||||
db, provider, interface,
|
||||
status=status, latency_ms=latency_ms, error=message, fallback_reason=reason,
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
pass
|
||||
|
||||
|
||||
def _safe_record_success(
|
||||
db: Any,
|
||||
provider: str,
|
||||
interface: str,
|
||||
latency_ms: int,
|
||||
result: Any,
|
||||
classify: Callable[[Any], tuple[str, str, int | None] | None] | None,
|
||||
) -> None:
|
||||
if db is None:
|
||||
return
|
||||
status, reason, data_age = "ok", "", None
|
||||
if classify is not None:
|
||||
try:
|
||||
classified = classify(result)
|
||||
if classified:
|
||||
status, reason, data_age = classified
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
# A classifier bug must never mislabel (or hide) a real success;
|
||||
# degrade to a plain "ok" call rather than skipping the log.
|
||||
status, reason, data_age = "ok", "", None
|
||||
try:
|
||||
record_call(
|
||||
db, provider, interface,
|
||||
status=status, latency_ms=latency_ms, fallback_reason=reason, data_age_seconds=data_age,
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
pass
|
||||
|
||||
|
||||
def observe(
|
||||
db: Any,
|
||||
provider: str,
|
||||
interface: str,
|
||||
fn: Callable[[], _T],
|
||||
*,
|
||||
classify: Callable[[Any], tuple[str, str, int | None] | None] | None = None,
|
||||
) -> _T:
|
||||
"""Call ``fn()`` and record a side-channel status row.
|
||||
|
||||
Returns exactly what ``fn()`` returns and re-raises exactly what
|
||||
``fn()`` raises. ``db`` may be ``None`` (e.g. in call sites that are not
|
||||
wired to a database yet); in that case this is a transparent passthrough
|
||||
with no recording at all. Same when the ``DATAHUB_OBSERVABILITY`` kill
|
||||
switch is off (see ``is_enabled``): this becomes ``return fn()`` with no
|
||||
timing, no classification, and no db access whatsoever.
|
||||
"""
|
||||
if db is not None and not is_enabled(db):
|
||||
return fn()
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
result = fn()
|
||||
except Exception:
|
||||
latency_ms = round((time.perf_counter() - started) * 1000)
|
||||
import sys
|
||||
|
||||
exc = sys.exc_info()[1]
|
||||
if exc is not None:
|
||||
_safe_record_failure(db, provider, interface, latency_ms, exc)
|
||||
raise
|
||||
latency_ms = round((time.perf_counter() - started) * 1000)
|
||||
_safe_record_success(db, provider, interface, latency_ms, result, classify)
|
||||
return result
|
||||
@@ -20,6 +20,7 @@ from datahub.governance.ratelimit import TokenBucket
|
||||
from datahub.governance.retry import RetryError, retry_call
|
||||
from datahub.logutil import get_logger
|
||||
from datahub.normalize import finite_number, normalize_daily
|
||||
from datahub import observability
|
||||
from datahub.revision import (
|
||||
compare_fields,
|
||||
diff_published_vs_upstream,
|
||||
@@ -98,6 +99,58 @@ STAGING_INSERT = {
|
||||
**EXTENDED_STAGING_INSERT,
|
||||
}
|
||||
|
||||
# Staging-table business keys (PRIMARY KEY minus the constant batch_id),
|
||||
# mirroring the PRIMARY KEY clauses declared in datahub/db.py and
|
||||
# datahub/datasets_ext.py. Used only to collapse within-batch duplicates so
|
||||
# a single upstream response cannot fail the whole batch on a UNIQUE
|
||||
# constraint (HEL-529: 2026-09-14 dc_hot returned 4 duplicate ts_codes and
|
||||
# hm_detail 43 duplicate (ts_code, hm_name) keys in one response, which has
|
||||
# blocked popularity/dragon_tiger publishing every day since 09-07).
|
||||
STAGING_KEY_FIELDS = {
|
||||
"stocks": ("ts_code", "trade_date"),
|
||||
"daily": ("ts_code", "trade_date"),
|
||||
"valuation": ("ts_code", "trade_date"),
|
||||
"moneyflow": ("ts_code", "trade_date"),
|
||||
"auction": ("ts_code", "trade_date"),
|
||||
"index_daily": ("ts_code", "trade_date"),
|
||||
"limit_events": ("ts_code", "trade_date", "limit_type"),
|
||||
"popularity": ("ts_code", "trade_date", "source"),
|
||||
"dragon_tiger": ("ts_code", "trade_date", "hm_name"),
|
||||
"sector_daily": ("ts_code", "trade_date", "family"),
|
||||
}
|
||||
|
||||
|
||||
def _dedupe_staging_rows(dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Collapse within-batch duplicates on the staging table's business key.
|
||||
|
||||
Deterministic: keeps the LAST occurrence of each key (the same row the
|
||||
eod copy's INSERT OR REPLACE would keep), preserves first-seen order, and
|
||||
never touches rows across batches. Datasets without a declared business
|
||||
key are returned unchanged.
|
||||
"""
|
||||
fields = STAGING_KEY_FIELDS.get(dataset)
|
||||
if not fields:
|
||||
return rows
|
||||
seen: dict[tuple, int] = {}
|
||||
out: list[dict[str, Any]] = []
|
||||
dropped = 0
|
||||
for row in rows:
|
||||
key = tuple(row.get(field) for field in fields)
|
||||
if key in seen:
|
||||
out[seen[key]] = row
|
||||
dropped += 1
|
||||
else:
|
||||
seen[key] = len(out)
|
||||
out.append(row)
|
||||
if dropped:
|
||||
LOGGER.info(
|
||||
"staging dedupe: %s collapsed %d duplicate rows within batch (kept last)",
|
||||
dataset,
|
||||
dropped,
|
||||
extra={"hub": {"dataset": dataset, "deduped": dropped}},
|
||||
)
|
||||
return out
|
||||
|
||||
EOD_COPY = {
|
||||
"stocks": (
|
||||
"INSERT OR REPLACE INTO eod_stocks "
|
||||
@@ -1438,7 +1491,19 @@ class Pipeline:
|
||||
else:
|
||||
keys = [(row.get("ts_code"), row.get("trade_date")) for row in rows]
|
||||
dup = row_n - len(set(keys))
|
||||
# Extended soft datasets (popularity/dragon_tiger/…) already collapse
|
||||
# within-batch dups in _dedupe_staging_rows before INSERT. Upstream
|
||||
# ths/dc (and similar) routinely emit duplicate business keys; those
|
||||
# collapsed dups must not hard-fail publish (HEL-562). Datasets without
|
||||
# a staging business key — and all hard/core soft gates — still treat
|
||||
# raw duplicate keys as errors.
|
||||
staging_collapses_dups = (
|
||||
dataset in EXTENDED_SOFT_DATASETS and dataset in STAGING_KEY_FIELDS
|
||||
)
|
||||
if dup:
|
||||
if staging_collapses_dups:
|
||||
warnings.append(f"duplicate keys: {dup}")
|
||||
else:
|
||||
errors.append(f"duplicate keys: {dup}")
|
||||
bad_date = sum(1 for row in rows if str(row.get("trade_date")) != trade_date)
|
||||
if bad_date:
|
||||
@@ -1458,7 +1523,8 @@ class Pipeline:
|
||||
field_report = self._field_gate(dataset, trade_date, rows, errors)
|
||||
if dataset in SOFT_DATASETS:
|
||||
allow_empty = dataset in {"popularity", "dragon_tiger", "moneyflow", "auction"}
|
||||
hard_fail = bool(dup or bad_date or (empty and not allow_empty))
|
||||
hard_dup = 0 if staging_collapses_dups else dup
|
||||
hard_fail = bool(hard_dup or bad_date or (empty and not allow_empty))
|
||||
else:
|
||||
hard_fail = bool(errors) and (dataset in HARD_DATASETS or dataset == STOCKS_DATASET)
|
||||
report = {
|
||||
@@ -1634,6 +1700,7 @@ class Pipeline:
|
||||
deleted += cur.rowcount
|
||||
connection.execute("DELETE FROM job_runs WHERE started_at < ?", (cutoff_jobs,))
|
||||
connection.execute("DELETE FROM src_calls WHERE created_at < ?", (cutoff_jobs,))
|
||||
connection.execute("DELETE FROM provider_call_log WHERE created_at < ?", (cutoff_jobs,))
|
||||
return {"staging_deleted": deleted}
|
||||
|
||||
def audit(self, actor: str, action: str, target: str = "", detail: str = "") -> None:
|
||||
@@ -1718,6 +1785,7 @@ class Pipeline:
|
||||
|
||||
def _stage(self, dataset: str, batch_id: str, rows: list[dict[str, Any]]) -> None:
|
||||
sql, mapper = STAGING_INSERT[dataset]
|
||||
rows = _dedupe_staging_rows(dataset, rows)
|
||||
with self.db.write() as connection:
|
||||
connection.execute(
|
||||
f"DELETE FROM {DATASET_TABLES[dataset][1]} WHERE batch_id = ?",
|
||||
@@ -1768,6 +1836,18 @@ class Pipeline:
|
||||
"INSERT INTO src_calls(provider, endpoint, ok, latency_ms, error, created_at) VALUES (?,?,?,?,?,?)",
|
||||
("tushare", endpoint, 1 if ok else 0, latency_ms, error, isoformat(self.clock())),
|
||||
)
|
||||
# HEL-543 side channel: unified cross-provider call log/health. Kept
|
||||
# strictly additive and fail-open; the src_calls insert above (the
|
||||
# existing, already-compatible Tushare record) is unaffected either
|
||||
# way.
|
||||
if ok:
|
||||
status, reason = "ok", ""
|
||||
else:
|
||||
status, reason = observability.classify_error(error)
|
||||
observability.record_call(
|
||||
self.db, "tushare", endpoint,
|
||||
status=status, latency_ms=latency_ms, error=error, fallback_reason=reason,
|
||||
)
|
||||
|
||||
def _persist_health(self, state: str, error: str = "") -> None:
|
||||
snap = self.breaker.snapshot()
|
||||
|
||||
@@ -18,6 +18,7 @@ from datahub.adapters.tencent import TencentAdapter
|
||||
from datahub.codes import resolve_code
|
||||
from datahub.db import HubDB
|
||||
from datahub.governance.lkg import LastKnownGood
|
||||
from datahub import observability
|
||||
from datahub.timeutil import isoformat, now_shanghai, yyyymmdd
|
||||
|
||||
QUOTE_TTL = 60
|
||||
@@ -25,6 +26,23 @@ INDEX_TTL = 60
|
||||
INTRADAY_TTL = 20
|
||||
QUOTE_BATCH = 60
|
||||
|
||||
# HEL-543 side-channel classifiers. These only *read* an already-successful
|
||||
# payload to decide what to log; they never change the payload itself and a
|
||||
# classifier exception always degrades to "ok" (see observability.classify_rows).
|
||||
|
||||
|
||||
def _classify_quote_rows(rows: Any) -> tuple[str, str, int | None]:
|
||||
return observability.classify_rows(rows, freshness_field="quote_time_epoch")
|
||||
|
||||
|
||||
def _classify_rows_no_freshness(rows: Any) -> tuple[str, str, int | None]:
|
||||
return observability.classify_rows(rows, freshness_field=None)
|
||||
|
||||
|
||||
def _classify_intraday_payload(data: Any) -> tuple[str, str, int | None]:
|
||||
points = data.get("points") if isinstance(data, dict) else None
|
||||
return observability.classify_rows(points or [], freshness_field=None)
|
||||
|
||||
|
||||
class RealtimeApiError(RuntimeError):
|
||||
def __init__(self, code: str, message: str) -> None:
|
||||
@@ -44,12 +62,17 @@ def fetch_index_quotes(db: HubDB) -> dict[str, Any]:
|
||||
cached = _read_cache(db, cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
eastmoney = EastmoneyAdapter()
|
||||
try:
|
||||
rows = eastmoney.fetch_indices()
|
||||
rows = observability.observe(
|
||||
db, "eastmoney", "indices", lambda: EastmoneyAdapter().fetch_indices(),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
source = "eastmoney:ulist"
|
||||
except Exception:
|
||||
rows = TencentAdapter().fetch_indices()
|
||||
rows = observability.observe(
|
||||
db, "tencent", "indices", lambda: TencentAdapter().fetch_indices(),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
source = "tencent:qt"
|
||||
if len(rows) < 3:
|
||||
raise RealtimeApiError("SOURCE_UNAVAILABLE", "index quotes incomplete")
|
||||
@@ -77,7 +100,10 @@ def fetch_market_quotes(db: HubDB) -> dict[str, Any]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
source = ""
|
||||
try:
|
||||
rows = EastmoneyAdapter().fetch_market_quotes()
|
||||
rows = observability.observe(
|
||||
db, "eastmoney", "market_quotes", lambda: EastmoneyAdapter().fetch_market_quotes(),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
source = "eastmoney:clist"
|
||||
except Exception as exc:
|
||||
errors.append(f"eastmoney:{exc}")
|
||||
@@ -85,7 +111,10 @@ def fetch_market_quotes(db: HubDB) -> dict[str, Any]:
|
||||
listed = _listed_ts_codes(db)
|
||||
if not listed:
|
||||
raise AdapterError("no local stock master for tencent market snapshot")
|
||||
rows = _tencent_named_quotes(listed)
|
||||
rows = observability.observe(
|
||||
db, "tencent", "market_quotes_fallback", lambda: _tencent_named_quotes(listed),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
if len(rows) < 200:
|
||||
raise AdapterError(f"Tencent market snapshot too small: {len(rows)}")
|
||||
source = "tencent:qt"
|
||||
@@ -130,7 +159,10 @@ def fetch_quotes(db: HubDB, codes: list[str]) -> dict[str, Any]:
|
||||
|
||||
missing = [code for code in resolved if code not in by_code]
|
||||
try:
|
||||
rows = _eastmoney_named_quotes(missing)
|
||||
rows = observability.observe(
|
||||
db, "eastmoney", "named_quotes", lambda: _eastmoney_named_quotes(missing),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
by_code.update(_quote_map(rows, missing))
|
||||
if rows:
|
||||
sources.append("eastmoney:ulist")
|
||||
@@ -140,7 +172,10 @@ def fetch_quotes(db: HubDB, codes: list[str]) -> dict[str, Any]:
|
||||
missing = [code for code in resolved if code not in by_code]
|
||||
if missing:
|
||||
try:
|
||||
rows = _tencent_named_quotes(missing)
|
||||
rows = observability.observe(
|
||||
db, "tencent", "named_quotes", lambda: _tencent_named_quotes(missing),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
by_code.update(_quote_map(rows, missing))
|
||||
if rows:
|
||||
sources.append("tencent:qt")
|
||||
@@ -203,7 +238,10 @@ def fetch_sector_quote(db: HubDB, code: str, expected_date: str = "") -> dict[st
|
||||
return cached
|
||||
errors: list[str] = []
|
||||
try:
|
||||
row = EastmoneyAdapter().fetch_shenwan_quote(ts_code)
|
||||
row = observability.observe(
|
||||
db, "eastmoney", "sector_quote", lambda: EastmoneyAdapter().fetch_shenwan_quote(ts_code),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
if not _sector_row_matches(row, canonical_name):
|
||||
raise AdapterError(
|
||||
f"industry name mismatch: expected {canonical_name}, got {row.get('name') or '--'}"
|
||||
@@ -270,7 +308,10 @@ def fetch_limit_pool(db: HubDB, trade_date: str = "") -> dict[str, Any]:
|
||||
if cached is not None:
|
||||
return cached
|
||||
try:
|
||||
rows = EastmoneyAdapter().fetch_limit_pool(day)
|
||||
rows = observability.observe(
|
||||
db, "eastmoney", "limit_pool", lambda: EastmoneyAdapter().fetch_limit_pool(day),
|
||||
classify=_classify_rows_no_freshness,
|
||||
)
|
||||
source = "eastmoney:zt_pool"
|
||||
except Exception as exc:
|
||||
recovered = _load_quotes_lkg(db, cache_key)
|
||||
@@ -530,7 +571,10 @@ def warm_realtime(db: HubDB) -> dict[str, Any]:
|
||||
if master_code and master_name:
|
||||
canonical_names.setdefault(master_code, master_name)
|
||||
codes = list(canonical_names)
|
||||
fetched_sector_rows = _eastmoney_sector_quotes(codes)
|
||||
fetched_sector_rows = observability.observe(
|
||||
db, "eastmoney", "sector_quotes_batch", lambda: _eastmoney_sector_quotes(codes),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
for row in fetched_sector_rows:
|
||||
if _row_quote_date(row, today) != today:
|
||||
continue
|
||||
@@ -588,7 +632,10 @@ def fetch_intraday(db: HubDB, code: str, date: str = "") -> dict[str, Any]:
|
||||
return cached
|
||||
adapter = EastmoneyAdapter()
|
||||
try:
|
||||
payload_data = adapter.fetch_intraday(ts_code, date)
|
||||
payload_data = observability.observe(
|
||||
db, "eastmoney", "intraday", lambda: adapter.fetch_intraday(ts_code, date),
|
||||
classify=_classify_intraday_payload,
|
||||
)
|
||||
source = "eastmoney:trends2"
|
||||
except Exception as exc:
|
||||
recovered = _load_intraday_lkg(db, ts_code, date)
|
||||
|
||||
@@ -24,7 +24,12 @@ class Settings:
|
||||
port: int = 8766
|
||||
encryption_key: str = ""
|
||||
api_token: str = ""
|
||||
admin_password: str = ""
|
||||
# HEL-560: the console has no accounts of its own. It verifies the review
|
||||
# site's session over the bridge, so it needs the site's internal base URL,
|
||||
# the shared bridge token, and the browser-reachable site URL for redirects.
|
||||
review_base_url: str = ""
|
||||
review_public_url: str = ""
|
||||
hub_admin_token: str = ""
|
||||
tushare_token: str = ""
|
||||
ifind_refresh_token: str = ""
|
||||
ifind_access_token: str = ""
|
||||
@@ -33,6 +38,11 @@ class Settings:
|
||||
quality: dict[str, Any] = field(default_factory=dict)
|
||||
log_level: str = "INFO"
|
||||
scheduler_enabled: bool = True
|
||||
# HEL-543 kill switch: off disables the provider_call_log/provider_health
|
||||
# side channel entirely (observe()/record_call() become no-ops and the
|
||||
# new read-only admin endpoints report {"enabled": false}). Default on;
|
||||
# existing routing/fetch/publish behavior is identical either way.
|
||||
observability_enabled: bool = True
|
||||
|
||||
@property
|
||||
def tushare_rate_per_minute(self) -> int:
|
||||
@@ -117,7 +127,9 @@ def load_settings(
|
||||
port=int(environ.get("DATAHUB_PORT") or 8766),
|
||||
encryption_key=str(environ.get("DATAHUB_ENCRYPTION_KEY") or "").strip(),
|
||||
api_token=str(environ.get("DATAHUB_TOKEN") or "").strip(),
|
||||
admin_password=str(environ.get("DATAHUB_ADMIN_PASSWORD") or "").strip(),
|
||||
review_base_url=str(environ.get("REVIEW_BASE_URL") or "http://xiaobai-review:8765").strip(),
|
||||
review_public_url=str(environ.get("REVIEW_PUBLIC_URL") or "").strip(),
|
||||
hub_admin_token=str(environ.get("HUB_ADMIN_TOKEN") or "").strip(),
|
||||
tushare_token=str(environ.get("TUSHARE_TOKEN") or "").strip(),
|
||||
ifind_refresh_token=str(environ.get("IFIND_REFRESH_TOKEN") or "").strip(),
|
||||
ifind_access_token=str(environ.get("IFIND_ACCESS_TOKEN") or "").strip(),
|
||||
@@ -126,4 +138,5 @@ def load_settings(
|
||||
quality=_load_quality(quality_path),
|
||||
log_level=environ.get("DATAHUB_LOG_LEVEL") or "INFO",
|
||||
scheduler_enabled=str(environ.get("DATAHUB_SCHEDULER") or "1") not in {"0", "false", "False"},
|
||||
observability_enabled=str(environ.get("DATAHUB_OBSERVABILITY") or "1") not in {"0", "false", "False", "off", "OFF"},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from datahub.logutil import get_logger
|
||||
|
||||
LOGGER = get_logger()
|
||||
|
||||
SITE_SESSION_COOKIE = "xiaobai_session"
|
||||
VERIFY_CACHE_SECONDS = 20.0
|
||||
BRIDGE_TIMEOUT_SECONDS = 6.0
|
||||
|
||||
|
||||
class SiteBridgeError(RuntimeError):
|
||||
"""The review site could not answer a bridge call.
|
||||
|
||||
``status`` carries the review site's HTTP status when it answered with one.
|
||||
A 4xx there means the operator's input was rejected (bad API key, invalid
|
||||
model id), which must not surface here as a console fault.
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, status: int = 0) -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
|
||||
@property
|
||||
def caller_fault(self) -> bool:
|
||||
return 400 <= self.status < 500
|
||||
|
||||
|
||||
class SiteBridge:
|
||||
"""Service-to-service client for the review site's ``/api/hub-admin/*`` endpoints.
|
||||
|
||||
The shared ``HUB_ADMIN_TOKEN`` is the only credential; the review site
|
||||
checks it before any handler runs, so nothing here needs a browser session.
|
||||
"""
|
||||
|
||||
def __init__(self, base_url: str, token: str, timeout: float = BRIDGE_TIMEOUT_SECONDS) -> None:
|
||||
self.base_url = (base_url or "").rstrip("/")
|
||||
self.token = token or ""
|
||||
self.timeout = timeout
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self.base_url and self.token)
|
||||
|
||||
def call(self, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
if not self.configured:
|
||||
raise SiteBridgeError("主站桥接未配置:请设置 REVIEW_BASE_URL 与 HUB_ADMIN_TOKEN")
|
||||
body = json.dumps(payload or {}, ensure_ascii=False).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
f"{self.base_url}{path}",
|
||||
data=body,
|
||||
method="POST",
|
||||
headers={
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
"X-Hub-Admin-Token": self.token,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||||
raw = response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = _error_detail(exc.read())
|
||||
raise SiteBridgeError(detail or f"主站返回 {exc.code}", exc.code) from exc
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise SiteBridgeError(f"主站不可达:{exc}") from exc
|
||||
try:
|
||||
parsed = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise SiteBridgeError("主站返回的不是合法 JSON") from exc
|
||||
if not isinstance(parsed, dict):
|
||||
raise SiteBridgeError("主站返回的不是合法 JSON")
|
||||
if parsed.get("error"):
|
||||
raise SiteBridgeError(str(parsed["error"]))
|
||||
return parsed
|
||||
|
||||
|
||||
def _error_detail(raw: bytes) -> str:
|
||||
try:
|
||||
parsed = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
return ""
|
||||
if isinstance(parsed, dict) and parsed.get("error"):
|
||||
return str(parsed["error"])
|
||||
return ""
|
||||
|
||||
|
||||
class SiteAuth:
|
||||
"""Admin gate for the console: the review site owns accounts, we only verify.
|
||||
|
||||
The console has no accounts of its own. Every request carries the review
|
||||
site's ``xiaobai_session`` cookie (same host, different port, so the browser
|
||||
sends it), which we hand to the site for verification. Results are cached
|
||||
for a few seconds so a page full of panels does not fan out one bridge call
|
||||
per request.
|
||||
|
||||
CSRF is stateless: the token is an HMAC of the session token under a server
|
||||
secret, so it is unguessable without the secret yet needs no storage and
|
||||
stays valid exactly as long as the session does.
|
||||
"""
|
||||
|
||||
def __init__(self, bridge: SiteBridge, secret: str, cache_seconds: float = VERIFY_CACHE_SECONDS) -> None:
|
||||
self.bridge = bridge
|
||||
self._secret = (secret or "").encode("utf-8")
|
||||
self._cache_seconds = cache_seconds
|
||||
self._cache: dict[str, tuple[float, dict[str, Any] | None]] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def verify(self, session_token: str) -> dict[str, Any] | None:
|
||||
if not session_token:
|
||||
return None
|
||||
key = hashlib.sha256(session_token.encode("utf-8")).hexdigest()
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
cached = self._cache.get(key)
|
||||
if cached and cached[0] > now:
|
||||
return cached[1]
|
||||
user = self._verify_remote(session_token)
|
||||
with self._lock:
|
||||
self._cache[key] = (now + self._cache_seconds, user)
|
||||
if len(self._cache) > 256:
|
||||
self._prune(now)
|
||||
return user
|
||||
|
||||
def _prune(self, now: float) -> None:
|
||||
for cached_key in [key for key, (expires, _) in self._cache.items() if expires <= now]:
|
||||
self._cache.pop(cached_key, None)
|
||||
|
||||
def _verify_remote(self, session_token: str) -> dict[str, Any] | None:
|
||||
payload = self.bridge.call("/api/hub-admin/session", {"session_token": session_token})
|
||||
if not payload.get("authenticated"):
|
||||
return None
|
||||
user = payload.get("user") or {}
|
||||
return {
|
||||
"id": int(user.get("id") or 0),
|
||||
"username": str(user.get("username") or ""),
|
||||
"role": str(user.get("role") or "user"),
|
||||
"is_admin": bool(user.get("is_admin")),
|
||||
}
|
||||
|
||||
def invalidate(self, session_token: str) -> None:
|
||||
key = hashlib.sha256(session_token.encode("utf-8")).hexdigest()
|
||||
with self._lock:
|
||||
self._cache.pop(key, None)
|
||||
|
||||
def csrf_token(self, session_token: str) -> str:
|
||||
digest = hashlib.sha256(session_token.encode("utf-8")).digest()
|
||||
return hmac.new(self._secret, digest, hashlib.sha256).hexdigest()
|
||||
|
||||
def check_csrf(self, session_token: str, supplied: str) -> bool:
|
||||
if not supplied:
|
||||
return False
|
||||
return hmac.compare_digest(self.csrf_token(session_token), supplied)
|
||||
|
||||
def logout(self, session_token: str) -> None:
|
||||
self.invalidate(session_token)
|
||||
if not session_token:
|
||||
return
|
||||
try:
|
||||
self.bridge.call("/api/hub-admin/session/logout", {"session_token": session_token})
|
||||
except SiteBridgeError:
|
||||
LOGGER.warning("site logout bridge call failed")
|
||||
|
||||
def confirm_password(self, user_id: int, password: str) -> bool:
|
||||
if not password:
|
||||
return False
|
||||
payload = self.bridge.call(
|
||||
"/api/hub-admin/password/check",
|
||||
{"user_id": int(user_id), "password": password},
|
||||
)
|
||||
return bool(payload.get("verified"))
|
||||
@@ -0,0 +1,172 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from datahub.serving import ApiError
|
||||
from datahub.siteauth import SiteBridge, SiteBridgeError
|
||||
|
||||
VENDOR_PRESETS: tuple[dict[str, str], ...] = (
|
||||
{"id": "openai", "label": "OpenAI", "base_url": "https://api.openai.com/v1"},
|
||||
{"id": "deepseek", "label": "DeepSeek", "base_url": "https://api.deepseek.com/v1"},
|
||||
{"id": "moonshot", "label": "Moonshot", "base_url": "https://api.moonshot.cn/v1"},
|
||||
{"id": "dashscope", "label": "阿里云百炼", "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1"},
|
||||
{"id": "zhipu", "label": "智谱 GLM", "base_url": "https://open.bigmodel.cn/api/paas/v4"},
|
||||
{"id": "siliconflow", "label": "SiliconFlow", "base_url": "https://api.siliconflow.cn/v1"},
|
||||
)
|
||||
|
||||
|
||||
class SiteConsole:
|
||||
"""Console-side view of the data the review site still owns.
|
||||
|
||||
The model pool, member roster and invite codes live in the review site's
|
||||
database — this console reads and writes them over the bridge instead of
|
||||
copying them, so there is exactly one source of truth. Every method turns a
|
||||
bridge failure into an ``ApiError`` the console frontend already knows how
|
||||
to render.
|
||||
"""
|
||||
|
||||
def __init__(self, bridge: SiteBridge) -> None:
|
||||
self.bridge = bridge
|
||||
|
||||
def _call(self, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
try:
|
||||
return self.bridge.call(path, payload)
|
||||
except SiteBridgeError as exc:
|
||||
# 主站因入参不合法而拒绝(Key 不对、模型 ID 不合法)是操作者的问题,
|
||||
# 照原样退回 400;只有主站真的不可达才算中枢侧不可用。
|
||||
code = "INVALID_ARGUMENT" if exc.caller_fault else "SOURCE_UNAVAILABLE"
|
||||
raise ApiError(code, str(exc)) from exc
|
||||
|
||||
# ---------------------------------------------------------------- models
|
||||
def models(self) -> dict[str, Any]:
|
||||
payload = self._call("/api/hub-admin/status")
|
||||
llm = payload.get("llm") or {}
|
||||
models = list(llm.get("models") or [])
|
||||
return {
|
||||
"vendors": [dict(preset) for preset in VENDOR_PRESETS],
|
||||
"groups": _group_by_vendor(models),
|
||||
"primary_model_id": str(llm.get("primary_model_id") or ""),
|
||||
"fallback_model_id": str(llm.get("fallback_model_id") or ""),
|
||||
"models": models,
|
||||
}
|
||||
|
||||
def save_models(self, body: dict[str, Any]) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {}
|
||||
if "models" in body:
|
||||
payload["models"] = body.get("models") or []
|
||||
for key in ("primary_model_id", "fallback_model_id"):
|
||||
if key in body:
|
||||
payload[key] = str(body.get(key) or "")
|
||||
if not payload:
|
||||
raise ApiError("INVALID_ARGUMENT", "没有需要保存的模型配置")
|
||||
self._call("/api/hub-admin/settings/save", payload)
|
||||
return self.models()
|
||||
|
||||
def test_model(self, body: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = self._call(
|
||||
"/api/hub-admin/settings/test",
|
||||
{"model_id": str(body.get("model_id") or ""), "profile": body.get("profile") or {}},
|
||||
)
|
||||
return {"result": payload.get("result") or {}}
|
||||
|
||||
def fetch_models(self, body: dict[str, Any]) -> dict[str, Any]:
|
||||
base_url = str(body.get("base_url") or "").strip()
|
||||
if not base_url:
|
||||
raise ApiError("INVALID_ARGUMENT", "请先填写供应商接口地址")
|
||||
payload = self._call(
|
||||
"/api/hub-admin/models/fetch",
|
||||
{"base_url": base_url, "api_key": str(body.get("api_key") or "")},
|
||||
)
|
||||
return {"models": payload.get("models") or []}
|
||||
|
||||
# --------------------------------------------------------------- members
|
||||
def members(self) -> dict[str, Any]:
|
||||
payload = self._call("/api/hub-admin/members")
|
||||
return {
|
||||
"users": payload.get("users") or [],
|
||||
"membership": payload.get("membership") or {},
|
||||
}
|
||||
|
||||
def save_member(self, body: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = dict(body or {})
|
||||
if not payload:
|
||||
raise ApiError("INVALID_ARGUMENT", "没有需要保存的会员设置")
|
||||
self._call("/api/hub-admin/membership/save", payload)
|
||||
return self.members()
|
||||
|
||||
def save_quota(self, body: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Daily call quota is a system setting, not a per-user membership row."""
|
||||
try:
|
||||
limit = int(body.get("member_daily_limit") or 0)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ApiError("INVALID_ARGUMENT", "每日调用额度必须是整数") from exc
|
||||
if limit < 1:
|
||||
raise ApiError("INVALID_ARGUMENT", "每日调用额度至少为 1")
|
||||
self._call("/api/hub-admin/settings/save", {"member_daily_limit": limit})
|
||||
return self.members()
|
||||
|
||||
# --------------------------------------------------------------- invites
|
||||
def invites(self) -> dict[str, Any]:
|
||||
payload = self._call("/api/hub-admin/invites")
|
||||
return {"summary": payload.get("summary") or {}, "codes": payload.get("codes") or []}
|
||||
|
||||
def create_invites(self, body: dict[str, Any], created_by: int) -> dict[str, Any]:
|
||||
payload = self._call(
|
||||
"/api/hub-admin/invites/create",
|
||||
{
|
||||
"count": body.get("count") or 1,
|
||||
"note": str(body.get("note") or ""),
|
||||
"created_by": int(created_by or 0),
|
||||
},
|
||||
)
|
||||
# `created` carries the plaintext codes and is the only moment they are
|
||||
# ever returned; the list under `codes` is the masked roster.
|
||||
return {
|
||||
"created": payload.get("created") or [],
|
||||
"summary": payload.get("summary") or {},
|
||||
"codes": payload.get("codes") or [],
|
||||
}
|
||||
|
||||
def revoke_invite(self, body: dict[str, Any]) -> dict[str, Any]:
|
||||
reference = str(body.get("code_id") or body.get("code") or "").strip()
|
||||
if not reference:
|
||||
raise ApiError("INVALID_ARGUMENT", "请选择要作废的邀请码")
|
||||
self._call("/api/hub-admin/invites/revoke", {"code_id": reference})
|
||||
return self.invites()
|
||||
|
||||
|
||||
def _group_by_vendor(models: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Bucket a flat model list by base URL so the console can render vendors.
|
||||
|
||||
The review site stores one row per model with its own base URL; the console
|
||||
shows vendors with their models nested, so the base URL is the grouping key
|
||||
and the preset table only supplies a friendly label when it recognises one.
|
||||
"""
|
||||
labels = {preset["base_url"]: preset["label"] for preset in VENDOR_PRESETS}
|
||||
order: list[str] = []
|
||||
buckets: dict[str, list[dict[str, Any]]] = {}
|
||||
for model in models:
|
||||
base_url = str(model.get("base_url") or "").strip()
|
||||
if base_url not in buckets:
|
||||
buckets[base_url] = []
|
||||
order.append(base_url)
|
||||
buckets[base_url].append(model)
|
||||
groups: list[dict[str, Any]] = []
|
||||
for base_url in order:
|
||||
entries = buckets[base_url]
|
||||
configured = next((entry for entry in entries if entry.get("api_key_last4")), None)
|
||||
groups.append(
|
||||
{
|
||||
"base_url": base_url,
|
||||
"label": labels.get(base_url) or _vendor_label(base_url),
|
||||
"configured": any(entry.get("configured") for entry in entries),
|
||||
"key_last4": str((configured or {}).get("api_key_last4") or ""),
|
||||
"models": entries,
|
||||
}
|
||||
)
|
||||
return groups
|
||||
|
||||
|
||||
def _vendor_label(base_url: str) -> str:
|
||||
host = base_url.split("//")[-1].split("/")[0]
|
||||
return host or "自定义供应商"
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Minimal, read-only source directory (HEL-543).
|
||||
|
||||
Registers what already exists: providers, their concrete interfaces, what
|
||||
capability/dataset each interface serves, and whether the provider plays a
|
||||
primary or backup role. This module only *describes* the current adapters
|
||||
and datasets already wired in `datahub/hub.py`, `datahub/serving.py`, and
|
||||
`datahub/realtime_serve.py`; it does not add a way to configure or add a new
|
||||
source without code, and it never changes routing, retries, or fallback
|
||||
order.
|
||||
|
||||
Every ``interfaces`` entry below is a docs-as-code mirror of a real call
|
||||
site, cross-referenced in comments so a reviewer can verify each row is
|
||||
accurate rather than aspirational:
|
||||
|
||||
- tushare interfaces mirror ``datahub/serving.py``'s ``_official_meta``/``source=``
|
||||
strings and ``datahub/steward.py``'s live/published dataset table.
|
||||
- eastmoney/tencent interfaces mirror the ``observability.observe(...)``
|
||||
call sites added in ``datahub/realtime_serve.py`` for HEL-543.
|
||||
- ifind interfaces mirror ``datahub/steward.py``'s ``IFIND_APIS`` table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
CATALOG: list[dict[str, Any]] = [
|
||||
{
|
||||
"provider": "tushare",
|
||||
"label": "Tushare",
|
||||
"role": "official_primary",
|
||||
"credential_key": "tushare_token",
|
||||
"status_source": "src_health (legacy, kept) + provider_health (unified, HEL-543)",
|
||||
"interfaces": [
|
||||
{"interface": "trade_cal", "capability": "交易日历", "datasets": ["calendar"], "group": "日历 / 主档"},
|
||||
{"interface": "stock_basic", "capability": "股票主档", "datasets": ["stocks"], "group": "日历 / 主档"},
|
||||
{"interface": "daily", "capability": "个股日K", "datasets": ["daily"], "group": "盘后 A 批"},
|
||||
{"interface": "adj_factor", "capability": "复权因子", "datasets": ["daily"], "group": "盘后 A 批"},
|
||||
{"interface": "daily_basic", "capability": "估值", "datasets": ["valuation"], "group": "盘后 A 批"},
|
||||
{"interface": "index_daily", "capability": "指数日K", "datasets": ["index_daily"], "group": "指数 B 批"},
|
||||
{"interface": "moneyflow", "capability": "资金流", "datasets": ["moneyflow"], "group": "盘后 A 批"},
|
||||
{"interface": "stk_auction", "capability": "集合竞价", "datasets": ["auction"], "group": "盘后 A 批"},
|
||||
{"interface": "limit_list_d", "capability": "涨跌停池", "datasets": ["limit_events"], "group": "扩展软批"},
|
||||
{"interface": "ths_hot", "capability": "同花顺人气榜", "datasets": ["popularity"], "group": "扩展软批"},
|
||||
{"interface": "dc_hot", "capability": "东方财富人气榜", "datasets": ["popularity"], "group": "扩展软批"},
|
||||
{"interface": "hm_detail", "capability": "龙虎榜游资明细", "datasets": ["dragon_tiger"], "group": "扩展软批"},
|
||||
{"interface": "ths_daily", "capability": "同花顺概念行情", "datasets": ["sector_daily"], "group": "扩展软批"},
|
||||
{"interface": "dc_index", "capability": "东方财富板块行情", "datasets": ["sector_daily"], "group": "扩展软批"},
|
||||
{"interface": "sw_daily", "capability": "申万行业行情", "datasets": ["sector_daily"], "group": "扩展软批"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"provider": "eastmoney",
|
||||
"label": "东方财富",
|
||||
"role": "provisional_primary",
|
||||
"credential_key": None,
|
||||
"status_source": "provider_health (unified, HEL-543)",
|
||||
"interfaces": [
|
||||
{"interface": "indices", "capability": "指数实时报价", "datasets": ["index_quotes"], "group": "实时快照"},
|
||||
{"interface": "market_quotes", "capability": "全市场实时快照", "datasets": ["quotes_latest"], "group": "实时快照"},
|
||||
{"interface": "named_quotes", "capability": "指定个股实时报价", "datasets": ["quotes_latest"], "group": "实时快照"},
|
||||
{"interface": "sector_quote", "capability": "申万板块实时报价(单个)", "datasets": ["sectors_quote"], "group": "实时快照"},
|
||||
{"interface": "sector_quotes_batch", "capability": "申万板块批量报价(预热)", "datasets": ["sectors_quote"], "group": "实时快照"},
|
||||
{"interface": "limit_pool", "capability": "涨停/炸板池(盘中)", "datasets": ["limit_pool"], "group": "实时快照"},
|
||||
{"interface": "intraday", "capability": "分时走势", "datasets": ["intraday_points"], "group": "实时快照"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"provider": "tencent",
|
||||
"label": "腾讯行情",
|
||||
"role": "provisional_backup",
|
||||
"credential_key": None,
|
||||
"status_source": "provider_health (unified, HEL-543)",
|
||||
"interfaces": [
|
||||
{"interface": "indices", "capability": "指数实时报价(东财失败时备用)", "datasets": ["index_quotes"], "group": "实时备援"},
|
||||
{
|
||||
"interface": "market_quotes_fallback",
|
||||
"capability": "全市场快照(备用;按本地股票主档逐只请求拼接)",
|
||||
"datasets": ["quotes_latest"],
|
||||
"group": "实时备援",
|
||||
},
|
||||
{"interface": "named_quotes", "capability": "指定个股实时报价(东财失败时备用)", "datasets": ["quotes_latest"], "group": "实时备援"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"provider": "ifind",
|
||||
"label": "同花顺 iFinD",
|
||||
"role": "licensed_optional",
|
||||
"credential_key": "ifind_refresh_token",
|
||||
"status_source": "provider_health (unified, HEL-543) + adapter.status()",
|
||||
"interfaces": [
|
||||
{"interface": "wencai", "capability": "问财自然语言选股", "datasets": ["ifind_wencai"], "group": "预留接口"},
|
||||
{"interface": "snapshots", "capability": "快照", "datasets": ["ifind_snapshots"], "group": "预留接口"},
|
||||
{"interface": "history", "capability": "历史行情", "datasets": ["ifind_history"], "group": "预留接口"},
|
||||
{"interface": "realtime", "capability": "实时行情", "datasets": ["ifind_realtime"], "group": "预留接口"},
|
||||
{"interface": "intraday", "capability": "分时(高频)", "datasets": ["ifind_intraday"], "group": "预留接口"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"provider": "ths",
|
||||
"label": "同花顺(预留)",
|
||||
"role": "reserved",
|
||||
"credential_key": None,
|
||||
"status_source": "adapter.probe()(占位,本阶段未接入真实数据)",
|
||||
"interfaces": [],
|
||||
},
|
||||
{
|
||||
"provider": "xgb",
|
||||
"label": "选股宝(预留)",
|
||||
"role": "reserved",
|
||||
"credential_key": None,
|
||||
"status_source": "adapter.probe()(占位,本阶段未接入真实数据)",
|
||||
"interfaces": [],
|
||||
},
|
||||
{
|
||||
"provider": "akshare",
|
||||
"label": "AKShare(预留)",
|
||||
"role": "reserved",
|
||||
"credential_key": None,
|
||||
"status_source": "adapter.probe()(占位,本阶段未接入真实数据)",
|
||||
"interfaces": [],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def snapshot(db: Any, auth: Any = None) -> list[dict[str, Any]]:
|
||||
"""Merge the static catalog with live credential/health facts.
|
||||
|
||||
Purely read-only: never touches routing, credentials, or adapters. Any
|
||||
failure while enriching one entry only degrades that entry's live data;
|
||||
it never drops the entry or raises, so a directory read can never break
|
||||
on a partially-unhealthy database.
|
||||
"""
|
||||
result: list[dict[str, Any]] = []
|
||||
for entry in CATALOG:
|
||||
item: dict[str, Any] = {
|
||||
"provider": entry["provider"],
|
||||
"label": entry.get("label", entry["provider"]),
|
||||
"role": entry["role"],
|
||||
"status_source": entry["status_source"],
|
||||
"interfaces": [dict(i) for i in entry.get("interfaces", [])],
|
||||
}
|
||||
cred_key = entry.get("credential_key")
|
||||
if cred_key:
|
||||
cred = None
|
||||
try:
|
||||
if auth is not None:
|
||||
cred = auth.credential_status(cred_key)
|
||||
except Exception:
|
||||
cred = None
|
||||
item["credential"] = cred or {"configured": False, "last4": "", "updated_at": ""}
|
||||
else:
|
||||
item["credential"] = {"configured": True, "last4": "", "updated_at": "", "note": "无需凭证"}
|
||||
health_rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
if db is not None:
|
||||
health_rows = db.fetchall(
|
||||
"SELECT interface, state, last_ok_at, last_error, last_fallback_reason, "
|
||||
"consec_failures, last_latency_ms, last_data_age_seconds, updated_at "
|
||||
"FROM provider_health WHERE provider = ? ORDER BY interface",
|
||||
(entry["provider"],),
|
||||
)
|
||||
except Exception:
|
||||
health_rows = []
|
||||
item["live_interfaces"] = health_rows
|
||||
# HEL-529 fix 3: 已登记 ≠ 已观测 ≠ 健康。观测记录有两种真实写法:
|
||||
# eastmoney/tencent 的 observe() 直接写接口名;tushare 官方管线
|
||||
# _log_call() 写的是数据集名(如 daily_basic 接口对应的数据集
|
||||
# valuation)。这里按「接口名 或 该接口声明的 datasets 之一」双向
|
||||
# 匹配,并为每个接口标注观测依据,禁止把"暂无观测"显示成"未配置"。
|
||||
by_interface = {str(row["interface"]): row for row in health_rows}
|
||||
for iface in item["interfaces"]:
|
||||
row = by_interface.get(str(iface["interface"]))
|
||||
basis = "interface" if row is not None else ""
|
||||
if row is None:
|
||||
for dataset in iface.get("datasets", []):
|
||||
candidate = by_interface.get(str(dataset))
|
||||
if candidate is not None:
|
||||
row = candidate
|
||||
basis = "dataset"
|
||||
break
|
||||
if row is None:
|
||||
iface["observed"] = False
|
||||
iface["observed_basis"] = ""
|
||||
iface["observed_state"] = ""
|
||||
iface["observed_at"] = ""
|
||||
iface["observed_latency_ms"] = None
|
||||
iface["observed_note"] = ""
|
||||
else:
|
||||
iface["observed"] = True
|
||||
iface["observed_basis"] = basis
|
||||
iface["observed_state"] = str(row["state"] or "")
|
||||
iface["observed_at"] = str(row["last_ok_at"] or row["updated_at"] or "")
|
||||
iface["observed_latency_ms"] = row["last_latency_ms"]
|
||||
iface["observed_note"] = str(row["last_error"] or row["last_fallback_reason"] or "")
|
||||
result.append(item)
|
||||
return result
|
||||
@@ -12,6 +12,7 @@ from typing import Any
|
||||
|
||||
from datahub.adapters.base import AdapterError
|
||||
from datahub.adapters.tushare import TUSHARE_FIELDS
|
||||
from datahub import observability
|
||||
from datahub.numbers import finite_number
|
||||
from datahub.realtime_serve import (
|
||||
RealtimeApiError,
|
||||
@@ -152,8 +153,12 @@ def _ifind_query(api, api_name: str, params: dict[str, Any], fields: str) -> dic
|
||||
)
|
||||
if not adapter.configured:
|
||||
raise ApiError("SOURCE_UNAVAILABLE", "iFinD 尚未配置")
|
||||
db = getattr(api, "db", None)
|
||||
try:
|
||||
rows = adapter.fetch(dataset, dict(params))
|
||||
rows = observability.observe(
|
||||
db, "ifind", dataset, lambda: adapter.fetch(dataset, dict(params)),
|
||||
classify=lambda r: observability.classify_rows(r, freshness_field=None),
|
||||
)
|
||||
except AdapterError as exc:
|
||||
raise ApiError("SOURCE_UNAVAILABLE", str(exc)) from exc
|
||||
return envelope(
|
||||
|
||||
@@ -113,3 +113,51 @@ def fake_transport(api_name: str, params: dict, fields: str):
|
||||
if limit_type:
|
||||
rows = [row for row in rows if str(row.get("limit_type") or "") == limit_type]
|
||||
return rows
|
||||
|
||||
|
||||
class StubSiteAuth:
|
||||
"""Stand-in for the review-site session bridge.
|
||||
|
||||
The console verifies operators against the review site over HTTP, which
|
||||
tests must not depend on. This stub answers from a fixed session -> user
|
||||
map and keeps the same stateless-HMAC CSRF contract as the real service, so
|
||||
tests exercise the console's own gate rather than the network hop.
|
||||
"""
|
||||
|
||||
SESSION = "site-session-token"
|
||||
ADMIN = {"id": 1, "username": "admin", "role": "admin", "is_admin": True}
|
||||
MEMBER = {"id": 2, "username": "member", "role": "user", "is_admin": False}
|
||||
|
||||
def __init__(self, password: str = "AdminPass1", secret: str = "stub-secret") -> None:
|
||||
self.password = password
|
||||
self.secret = secret
|
||||
self.sessions = {self.SESSION: dict(self.ADMIN)}
|
||||
self.logged_out: list[str] = []
|
||||
|
||||
def add_session(self, token: str, user: dict) -> None:
|
||||
self.sessions[token] = dict(user)
|
||||
|
||||
def verify(self, session_token: str):
|
||||
return self.sessions.get(session_token)
|
||||
|
||||
def csrf_token(self, session_token: str) -> str:
|
||||
import hashlib
|
||||
import hmac
|
||||
|
||||
digest = hashlib.sha256(session_token.encode("utf-8")).digest()
|
||||
return hmac.new(self.secret.encode("utf-8"), digest, hashlib.sha256).hexdigest()
|
||||
|
||||
def check_csrf(self, session_token: str, supplied: str) -> bool:
|
||||
import hmac
|
||||
|
||||
return bool(supplied) and hmac.compare_digest(self.csrf_token(session_token), supplied)
|
||||
|
||||
def logout(self, session_token: str) -> None:
|
||||
self.logged_out.append(session_token)
|
||||
self.sessions.pop(session_token, None)
|
||||
|
||||
def confirm_password(self, user_id: int, password: str) -> bool:
|
||||
return bool(password) and password == self.password
|
||||
|
||||
def invalidate(self, session_token: str) -> None:
|
||||
pass
|
||||
|
||||
@@ -17,25 +17,39 @@ from datahub.httpapp import make_handler
|
||||
from datahub.hub import Hub
|
||||
from datahub.logutil import JsonFormatter
|
||||
from datahub.settings import Settings
|
||||
from tests.fixtures import fake_transport
|
||||
from tests.fixtures import StubSiteAuth, fake_transport
|
||||
|
||||
|
||||
class AdminTests(unittest.TestCase):
|
||||
"""HEL-560: the console has no accounts — it rides the review site session.
|
||||
|
||||
Every case here drives the console the way a browser does: the review
|
||||
site's `xiaobai_session` cookie plus the stateless CSRF token derived from
|
||||
it. There is no console login endpoint left to exercise.
|
||||
"""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
settings = Settings(
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="z" * 32,
|
||||
admin_password="StartPass1",
|
||||
tushare_token="real-tushare-token-abcdef",
|
||||
db_path=Path(self.tmp.name) / "hub.db",
|
||||
scheduler_enabled=False,
|
||||
review_public_url="http://127.0.0.1:8765",
|
||||
)
|
||||
self.site_auth = StubSiteAuth()
|
||||
self.hub = Hub(
|
||||
settings,
|
||||
adapter=TushareAdapter("real-tushare-token-abcdef", transport=fake_transport),
|
||||
site_auth=self.site_auth,
|
||||
)
|
||||
self.hub = Hub(settings, adapter=TushareAdapter("real-tushare-token-abcdef", transport=fake_transport))
|
||||
handler = make_handler(self.hub)
|
||||
self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
threading.Thread(target=self.server.serve_forever, daemon=True).start()
|
||||
self.base = f"http://127.0.0.1:{self.server.server_address[1]}"
|
||||
self.cookie = f"xiaobai_session={StubSiteAuth.SESSION}"
|
||||
self.csrf = self.site_auth.csrf_token(StubSiteAuth.SESSION)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.server.shutdown()
|
||||
@@ -54,51 +68,96 @@ class AdminTests(unittest.TestCase):
|
||||
set_cookie = resp.headers.get("Set-Cookie", "")
|
||||
return resp.status, json.loads(resp.read().decode()), set_cookie
|
||||
|
||||
def test_login_change_password_and_secret_masking(self) -> None:
|
||||
status, body, cookie_header = self._json(
|
||||
"/admin/api/login", "POST", {"username": "hub_admin", "password": "StartPass1"}
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertTrue(body["must_change"])
|
||||
cookie = cookie_header.split(";")[0]
|
||||
csrf = body["csrf"]
|
||||
status, _, _ = self._json(
|
||||
"/admin/api/change-password",
|
||||
"POST",
|
||||
{"current": "StartPass1", "new_password": "NewPass123"},
|
||||
cookie=cookie,
|
||||
csrf=csrf,
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
_, sources, _ = self._json("/admin/api/sources", cookie=cookie, csrf=csrf)
|
||||
blob = json.dumps(sources)
|
||||
self.assertNotIn("real-tushare-token-abcdef", blob)
|
||||
self.assertTrue(sources["items"][0]["credential"]["configured"])
|
||||
self.assertTrue(str(sources["items"][0]["credential"]["last4"]).endswith("cdef") or "****" in str(sources["items"][0]["credential"]["last4"]))
|
||||
def _admin(self, path, method="GET", body=None):
|
||||
return self._json(path, method, body, cookie=self.cookie, csrf=self.csrf)
|
||||
|
||||
def test_rollback_requires_password_and_confirm(self) -> None:
|
||||
_, body, cookie_header = self._json(
|
||||
"/admin/api/login", "POST", {"username": "hub_admin", "password": "StartPass1"}
|
||||
)
|
||||
cookie = cookie_header.split(";")[0]
|
||||
csrf = body["csrf"]
|
||||
self._json("/admin/api/change-password", "POST", {"current": "StartPass1", "new_password": "NewPass123"}, cookie, csrf)
|
||||
from urllib.error import HTTPError
|
||||
def test_session_reports_the_site_account_and_a_csrf_token(self) -> None:
|
||||
status, body, _ = self._admin("/admin/api/session")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertTrue(body["authenticated"])
|
||||
self.assertTrue(body["is_admin"])
|
||||
self.assertEqual(body["username"], "admin")
|
||||
self.assertEqual(body["csrf"], self.csrf)
|
||||
|
||||
def test_anonymous_session_probe_returns_the_site_login_url(self) -> None:
|
||||
with self.assertRaises(HTTPError) as ctx:
|
||||
self._json("/admin/api/session")
|
||||
self.assertEqual(ctx.exception.code, 401)
|
||||
payload = json.loads(ctx.exception.read().decode())
|
||||
self.assertFalse(payload["authenticated"])
|
||||
self.assertEqual(payload["login_url"], "http://127.0.0.1:8765/login/")
|
||||
|
||||
def test_non_admin_site_accounts_are_refused(self) -> None:
|
||||
self.site_auth.add_session("member-session", StubSiteAuth.MEMBER)
|
||||
with self.assertRaises(HTTPError) as ctx:
|
||||
self._json("/admin/api/sources", cookie="xiaobai_session=member-session")
|
||||
self.assertEqual(ctx.exception.code, 403)
|
||||
payload = json.loads(ctx.exception.read().decode())
|
||||
self.assertEqual(payload["error"]["code"], "PERMISSION_DENIED")
|
||||
self.assertFalse(payload["is_admin"])
|
||||
|
||||
def test_writes_require_the_derived_csrf_token(self) -> None:
|
||||
with self.assertRaises(HTTPError) as ctx:
|
||||
self._json(
|
||||
"/admin/api/credentials/tushare",
|
||||
"POST",
|
||||
{"tushare_token": "new-token-1234"},
|
||||
cookie=self.cookie,
|
||||
)
|
||||
self.assertEqual(ctx.exception.code, 401)
|
||||
|
||||
def test_logout_ends_the_site_session(self) -> None:
|
||||
status, body, _ = self._admin("/admin/api/logout", "POST", {})
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body["login_url"], "http://127.0.0.1:8765/login/")
|
||||
self.assertIn(StubSiteAuth.SESSION, self.site_auth.logged_out)
|
||||
|
||||
def test_stored_credentials_are_masked_in_the_sources_view(self) -> None:
|
||||
_, sources, _ = self._admin("/admin/api/sources")
|
||||
blob = json.dumps(sources)
|
||||
self.assertNotIn("real-tushare-token-abcdef", blob)
|
||||
credential = sources["items"][0]["credential"]
|
||||
self.assertTrue(credential["configured"])
|
||||
self.assertTrue("****" in str(credential["last4"]) or str(credential["last4"]).endswith("cdef"))
|
||||
|
||||
def test_tushare_credential_write_hot_swaps_the_live_adapter(self) -> None:
|
||||
status, _, _ = self._admin(
|
||||
"/admin/api/credentials/tushare", "POST", {"tushare_token": "rotated-token-9876"}
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(self.hub.adapter.token, "rotated-token-9876")
|
||||
self.assertEqual(self.hub.auth.load_credential("tushare_token"), "rotated-token-9876")
|
||||
_, sources, _ = self._admin("/admin/api/sources")
|
||||
self.assertNotIn("rotated-token-9876", json.dumps(sources))
|
||||
|
||||
def test_ifind_credential_write_hot_swaps_the_live_adapter(self) -> None:
|
||||
status, _, _ = self._admin(
|
||||
"/admin/api/credentials/ifind",
|
||||
"POST",
|
||||
{"ifind_refresh_token": "refresh-abcd", "ifind_access_token": "access-efgh"},
|
||||
)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(self.hub.auth.load_credential("ifind_refresh_token"), "refresh-abcd")
|
||||
self.assertEqual(self.hub.auth.load_credential("ifind_access_token"), "access-efgh")
|
||||
|
||||
def test_rollback_confirms_the_site_account_password(self) -> None:
|
||||
with self.assertRaises(HTTPError) as ctx:
|
||||
self._admin(
|
||||
"/admin/api/rollback",
|
||||
"POST",
|
||||
{"dataset": "daily", "trade_date": "20240902", "password": "wrong", "confirm": "daily:20240902"},
|
||||
cookie,
|
||||
csrf,
|
||||
{
|
||||
"dataset": "daily",
|
||||
"trade_date": "20240902",
|
||||
"password": "wrong",
|
||||
"confirm": "daily:20240902",
|
||||
},
|
||||
)
|
||||
self.assertEqual(ctx.exception.code, 401)
|
||||
|
||||
def test_invalid_json_does_not_log_request_body_secrets(self) -> None:
|
||||
secret = "SuperSecretPass1!"
|
||||
token = "hub-token-should-not-leak"
|
||||
raw = json.dumps({"password": secret, "token": token, "username": "hub_admin"}) + "{not-json"
|
||||
raw = json.dumps({"password": secret, "token": token, "username": "admin"}) + "{not-json"
|
||||
stream = io.StringIO()
|
||||
logger = logging.getLogger("datahub")
|
||||
handler = logging.StreamHandler(stream)
|
||||
@@ -108,9 +167,13 @@ class AdminTests(unittest.TestCase):
|
||||
logger.setLevel(logging.DEBUG)
|
||||
try:
|
||||
req = Request(
|
||||
self.base + "/admin/api/login",
|
||||
self.base + "/admin/api/credentials/tushare",
|
||||
data=raw.encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Cookie": self.cookie,
|
||||
"X-CSRF-Token": self.csrf,
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with self.assertRaises(HTTPError) as ctx:
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from datahub.adapters.tushare import TushareAdapter
|
||||
from datahub.crypto import SecretVault
|
||||
from datahub.httpapp import make_handler
|
||||
from datahub.hub import Hub
|
||||
from datahub.settings import Settings
|
||||
from tests.fixtures import TRADE_DATE, StubSiteAuth, fake_transport
|
||||
|
||||
|
||||
class AdminObservabilityApiTests(unittest.TestCase):
|
||||
"""HEL-543: new read-only admin endpoints for provider status, source
|
||||
catalog and lineage. These must never require write access and must
|
||||
never touch the existing routing/publish logic."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
settings = Settings(
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="z" * 32,
|
||||
tushare_token="real-tushare-token-abcdef",
|
||||
db_path=Path(self.tmp.name) / "hub.db",
|
||||
scheduler_enabled=False,
|
||||
)
|
||||
self.site_auth = StubSiteAuth()
|
||||
self.hub = Hub(
|
||||
settings,
|
||||
adapter=TushareAdapter("real-tushare-token-abcdef", transport=fake_transport),
|
||||
site_auth=self.site_auth,
|
||||
)
|
||||
handler = make_handler(self.hub)
|
||||
self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
threading.Thread(target=self.server.serve_forever, daemon=True).start()
|
||||
self.base = f"http://127.0.0.1:{self.server.server_address[1]}"
|
||||
self.cookie = f"xiaobai_session={StubSiteAuth.SESSION}"
|
||||
self.csrf = self.site_auth.csrf_token(StubSiteAuth.SESSION)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _json(self, path, method="GET", body=None, cookie="", csrf=""):
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if cookie:
|
||||
headers["Cookie"] = cookie
|
||||
if csrf:
|
||||
headers["X-CSRF-Token"] = csrf
|
||||
req = Request(self.base + path, data=data, headers=headers, method=method)
|
||||
with urlopen(req, timeout=5) as resp:
|
||||
set_cookie = resp.headers.get("Set-Cookie", "")
|
||||
return resp.status, json.loads(resp.read().decode()), set_cookie
|
||||
|
||||
def _get(self, path):
|
||||
return self._json(path, cookie=self.cookie, csrf=self.csrf)
|
||||
|
||||
def test_providers_status_reflects_real_pipeline_activity(self) -> None:
|
||||
pipeline = self.hub.pipeline
|
||||
pipeline.ingest_reference(TRADE_DATE)
|
||||
pipeline.run_dataset("daily", TRADE_DATE)
|
||||
|
||||
status, body, _ = self._get("/admin/api/providers/status")
|
||||
self.assertEqual(status, 200)
|
||||
health = body["health"]
|
||||
self.assertTrue(any(item["provider"] == "tushare" and item["interface"] == "daily" for item in health))
|
||||
row = next(item for item in health if item["provider"] == "tushare" and item["interface"] == "daily")
|
||||
self.assertEqual(row["state"], "ok")
|
||||
recent = body["recent_calls"]
|
||||
self.assertTrue(any(item["provider"] == "tushare" and item["interface"] == "daily" for item in recent))
|
||||
|
||||
def test_providers_status_filters_by_provider(self) -> None:
|
||||
pipeline = self.hub.pipeline
|
||||
pipeline.ingest_reference(TRADE_DATE)
|
||||
pipeline.run_dataset("daily", TRADE_DATE)
|
||||
|
||||
status, body, _ = self._get("/admin/api/providers/status?provider=tushare")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertTrue(all(item["provider"] == "tushare" for item in body["health"]))
|
||||
self.assertTrue(all(item["provider"] == "tushare" for item in body["recent_calls"]))
|
||||
|
||||
status, body, _ = self._get("/admin/api/providers/status?provider=eastmoney")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body["health"], [])
|
||||
self.assertEqual(body["recent_calls"], [])
|
||||
|
||||
def test_source_catalog_lists_known_providers_without_leaking_secrets(self) -> None:
|
||||
status, body, _ = self._get("/admin/api/source-catalog")
|
||||
self.assertEqual(status, 200)
|
||||
blob = json.dumps(body)
|
||||
self.assertNotIn("real-tushare-token-abcdef", blob)
|
||||
providers = {item["provider"] for item in body["items"]}
|
||||
self.assertIn("tushare", providers)
|
||||
self.assertIn("eastmoney", providers)
|
||||
self.assertIn("tencent", providers)
|
||||
self.assertIn("ifind", providers)
|
||||
|
||||
def test_lineage_snapshot_and_affected_query(self) -> None:
|
||||
status, body, _ = self._get("/admin/api/lineage")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertTrue(len(body["items"]) > 0)
|
||||
datasets = {item["dataset"] for item in body["items"]}
|
||||
self.assertIn("stocks", datasets)
|
||||
|
||||
status, body, _ = self._get("/admin/api/lineage/affected?provider=tushare&interface=daily")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body["provider"], "tushare")
|
||||
self.assertEqual(body["interface"], "daily")
|
||||
|
||||
def test_disabled_kill_switch_reports_enabled_false_with_empty_structure(self) -> None:
|
||||
# HEL-543 total-review 🔴: flip the runtime kill switch the same way
|
||||
# Hub.__init__ wires Settings.observability_enabled onto the db
|
||||
# handle, then confirm every new endpoint reports disabled with an
|
||||
# explicit empty structure rather than silently going quiet.
|
||||
self.hub.db.observability_enabled = False
|
||||
try:
|
||||
pipeline = self.hub.pipeline
|
||||
pipeline.ingest_reference(TRADE_DATE)
|
||||
pipeline.run_dataset("daily", TRADE_DATE) # must still fully succeed
|
||||
|
||||
status, body, _ = self._get("/admin/api/providers/status")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body, {"enabled": False, "health": [], "recent_calls": []})
|
||||
|
||||
status, body, _ = self._get("/admin/api/source-catalog")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body, {"enabled": False, "items": []})
|
||||
|
||||
status, body, _ = self._get("/admin/api/lineage")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertFalse(body["enabled"])
|
||||
self.assertEqual(body["items"], [])
|
||||
|
||||
status, body, _ = self._get("/admin/api/lineage/affected?provider=tushare")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(
|
||||
body, {"enabled": False, "provider": "tushare", "interface": "", "items": []}
|
||||
)
|
||||
|
||||
# Nothing was ever written while disabled.
|
||||
self.assertEqual(self.hub.db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
finally:
|
||||
self.hub.db.observability_enabled = True
|
||||
|
||||
def test_non_admin_site_accounts_cannot_read_the_new_endpoints(self) -> None:
|
||||
from urllib.error import HTTPError
|
||||
|
||||
self.site_auth.add_session("member-session", StubSiteAuth.MEMBER)
|
||||
with self.assertRaises(HTTPError) as ctx:
|
||||
self._json("/admin/api/source-catalog", cookie="xiaobai_session=member-session")
|
||||
self.assertEqual(ctx.exception.code, 403)
|
||||
|
||||
with self.assertRaises(HTTPError) as ctx:
|
||||
self._json("/admin/api/source-catalog")
|
||||
self.assertEqual(ctx.exception.code, 401)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -37,7 +37,6 @@ class ApiContractTests(unittest.TestCase):
|
||||
port=0,
|
||||
encryption_key=key,
|
||||
api_token=self.token,
|
||||
admin_password="StartPass1",
|
||||
tushare_token="tushare-secret-token-xyz",
|
||||
db_path=Path(self.tmp.name) / "hub.db",
|
||||
backup_dir=Path(self.tmp.name) / "backups",
|
||||
|
||||
@@ -52,7 +52,6 @@ def make_pipe(transport: GroupTransport, quality_extra: dict | None = None):
|
||||
settings = Settings(
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="t" * 32,
|
||||
admin_password="admin-pass",
|
||||
tushare_token="test-token",
|
||||
db_path=db.path,
|
||||
quality=quality,
|
||||
@@ -361,18 +360,22 @@ class ForceBoundaryEntryTests(unittest.TestCase):
|
||||
from datahub.scheduler import Scheduler
|
||||
from datahub.serving import ApiError
|
||||
|
||||
from tests.fixtures import StubSiteAuth
|
||||
|
||||
vault = SecretVault(self.pipe.settings.encryption_key)
|
||||
auth = AuthService(self.db, vault, self.pipe.settings.api_token, "StartPass1")
|
||||
admin = AdminAPI(self.db, self.pipe, Scheduler(self.db, self.pipe), auth)
|
||||
auth = AuthService(self.db, vault, self.pipe.settings.api_token)
|
||||
# 危险操作的第二因子现在是主站账号口令(HEL-560),不再有本地管理员密码
|
||||
site_auth = StubSiteAuth(password="StartPass1")
|
||||
admin = AdminAPI(self.db, self.pipe, Scheduler(self.db, self.pipe), auth, site_auth=site_auth)
|
||||
before = publications_map(self.db, TRADE_DATE)
|
||||
result = admin.backfill("moneyflow", TRADE_DATE, "StartPass1", f"moneyflow:{TRADE_DATE}", "tester")
|
||||
result = admin.backfill("moneyflow", TRADE_DATE, "StartPass1", f"moneyflow:{TRADE_DATE}", "tester", 1)
|
||||
self.assertEqual(result["moneyflow"]["state"], "published")
|
||||
after = publications_map(self.db, TRADE_DATE)
|
||||
for name in (*GROUP_A, "stocks"):
|
||||
self.assertNotEqual(after[name], before[name], name)
|
||||
# bad password / wrong confirm still rejected
|
||||
with self.assertRaises(ApiError):
|
||||
admin.backfill("daily", TRADE_DATE, "wrong", f"daily:{TRADE_DATE}", "tester")
|
||||
admin.backfill("daily", TRADE_DATE, "wrong", f"daily:{TRADE_DATE}", "tester", 1)
|
||||
|
||||
def test_admin_backfill_switch_crash_is_failed_precondition(self) -> None:
|
||||
from datahub.admin_api import AdminAPI
|
||||
@@ -381,9 +384,13 @@ class ForceBoundaryEntryTests(unittest.TestCase):
|
||||
from datahub.scheduler import Scheduler
|
||||
from datahub.serving import ApiError
|
||||
|
||||
from tests.fixtures import StubSiteAuth
|
||||
|
||||
vault = SecretVault(self.pipe.settings.encryption_key)
|
||||
auth = AuthService(self.db, vault, self.pipe.settings.api_token, "StartPass1")
|
||||
admin = AdminAPI(self.db, self.pipe, Scheduler(self.db, self.pipe), auth)
|
||||
auth = AuthService(self.db, vault, self.pipe.settings.api_token)
|
||||
# 危险操作的第二因子现在是主站账号口令(HEL-560),不再有本地管理员密码
|
||||
site_auth = StubSiteAuth(password="StartPass1")
|
||||
admin = AdminAPI(self.db, self.pipe, Scheduler(self.db, self.pipe), auth, site_auth=site_auth)
|
||||
before = publications_map(self.db, TRADE_DATE)
|
||||
|
||||
def explode() -> None:
|
||||
@@ -391,7 +398,7 @@ class ForceBoundaryEntryTests(unittest.TestCase):
|
||||
|
||||
self.pipe.before_commit = explode
|
||||
with self.assertRaises(ApiError) as ctx:
|
||||
admin.backfill("valuation", TRADE_DATE, "StartPass1", f"valuation:{TRADE_DATE}", "tester")
|
||||
admin.backfill("valuation", TRADE_DATE, "StartPass1", f"valuation:{TRADE_DATE}", "tester", 1)
|
||||
self.assertEqual(ctx.exception.code, "FAILED_PRECONDITION")
|
||||
self.assertIn("killed mid-switch", ctx.exception.message)
|
||||
# previous complete A/B versions keep serving
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
INDEX = (ROOT / "admin" / "index.html").read_text(encoding="utf-8")
|
||||
STYLES = (ROOT / "admin" / "styles.css").read_text(encoding="utf-8")
|
||||
APP = (ROOT / "admin" / "app.js").read_text(encoding="utf-8")
|
||||
HTTPAPP = (ROOT / "datahub" / "httpapp.py").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
class ConsoleShellTests(unittest.TestCase):
|
||||
"""The console shell must carry the site-session gate, not its own login."""
|
||||
|
||||
def test_independent_login_and_password_change_are_gone(self) -> None:
|
||||
for removed in ("login-form", "change-form", "login-view", "change-view", 'value="hub_admin"'):
|
||||
self.assertNotIn(removed, INDEX, f"{removed} 属于已废弃的独立账号体系")
|
||||
self.assertNotIn("/admin/api/login", APP)
|
||||
self.assertNotIn("/admin/api/change-password", APP)
|
||||
|
||||
def test_gate_offers_a_way_back_to_the_review_site(self) -> None:
|
||||
for element in ("gate-view", "gate-title", "gate-desc", "gate-login", "gate-retry"):
|
||||
self.assertIn(element, INDEX)
|
||||
self.assertIn("/admin/api/session", APP)
|
||||
self.assertIn("login_url", APP)
|
||||
|
||||
def test_nav_exposes_the_pages_this_console_now_owns(self) -> None:
|
||||
for page in ("overview", "sources", "models", "members", "lineage"):
|
||||
self.assertIn(f'data-nav="{page}"', INDEX)
|
||||
# 路由白名单必须与导航一致,否则点了导航会回落到总览
|
||||
routed = re.search(r"return \[([^\]]+)\]\.includes\(h\)", APP)
|
||||
assert routed is not None
|
||||
for page in ("overview", "sources", "models", "members", "lineage"):
|
||||
self.assertIn(f"'{page}'", routed.group(1))
|
||||
|
||||
|
||||
class ThemeTokenTests(unittest.TestCase):
|
||||
"""Day/night is one token set with two value sets — never stacked overrides."""
|
||||
|
||||
def test_both_themes_define_the_same_tokens(self) -> None:
|
||||
night = _token_block(':root,\n:root[data-theme="night"]')
|
||||
day = _token_block(':root[data-theme="day"]')
|
||||
self.assertTrue(night)
|
||||
self.assertEqual(
|
||||
sorted(night),
|
||||
sorted(day),
|
||||
"日间主题必须覆盖同一组变量名,缺一个就会漏出夜间色",
|
||||
)
|
||||
|
||||
def test_no_hardcoded_colours_escape_the_token_set(self) -> None:
|
||||
# 颜色只要写死在 JS 或组件样式里,切主题就会有一块保持夜间色。
|
||||
self.assertEqual([], re.findall(r"#[0-9a-fA-F]{6}", APP))
|
||||
after_tokens = STYLES.split("/* ---------- ambient background", 1)[1]
|
||||
self.assertEqual([], re.findall(r"#[0-9a-fA-F]{6}", after_tokens))
|
||||
|
||||
def test_no_dark_literal_paint_survives_outside_the_token_blocks(self) -> None:
|
||||
"""A literal dark rgba() would stay dark in day mode — tokens only.
|
||||
|
||||
Accent tints are allowed: they are low-opacity washes of the four
|
||||
status hues and read correctly on either background.
|
||||
"""
|
||||
accent = {(34, 211, 238), (52, 211, 153), (251, 191, 36), (248, 113, 113)}
|
||||
offenders = []
|
||||
body = STYLES.split("/* ---------- ambient background", 1)[1]
|
||||
for line in body.splitlines():
|
||||
for match in re.finditer(r"rgba\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)", line):
|
||||
rgb = tuple(int(match.group(index)) for index in (1, 2, 3))
|
||||
if rgb in accent or sum(rgb) >= 250:
|
||||
continue
|
||||
offenders.append(line.strip()[:80])
|
||||
self.assertEqual([], offenders, "深色字面值必须收敛成 token,否则日间主题会漏出夜间底色")
|
||||
|
||||
def test_theme_choice_survives_a_reload(self) -> None:
|
||||
self.assertIn("localStorage.getItem('datahub-theme')", APP)
|
||||
self.assertIn("localStorage.setItem('datahub-theme'", APP)
|
||||
self.assertIn('data-theme="night"', INDEX)
|
||||
|
||||
def test_svg_colours_go_through_style_so_tokens_apply(self) -> None:
|
||||
# var() 在 SVG 呈现属性里支持不稳,必须写进 style 才吃得到主题变量。
|
||||
for attribute in ('fill="var(', 'stroke="var(', 'stop-color="var('):
|
||||
self.assertNotIn(attribute, APP, f"{attribute} 应改写为 style 声明")
|
||||
|
||||
|
||||
class NarrowScreenTests(unittest.TestCase):
|
||||
def test_narrow_layout_stacks_inputs_and_buttons(self) -> None:
|
||||
self.assertIn("@media (max-width: 1100px)", STYLES)
|
||||
narrow = STYLES.split("@media (max-width: 1100px)", 1)[1]
|
||||
self.assertIn(".field-row { grid-template-columns: minmax(0, 1fr); }", narrow)
|
||||
# 凭证行与模型行在窄屏都要竖排,按钮才不会和输入框抢同一行
|
||||
self.assertIn(".cred-line { flex-direction: column;", narrow)
|
||||
self.assertIn(".model-row { flex-direction: column;", narrow)
|
||||
|
||||
|
||||
class ConsoleEndpointTests(unittest.TestCase):
|
||||
def test_every_endpoint_the_page_calls_is_routed(self) -> None:
|
||||
called = {
|
||||
path.split("?")[0]
|
||||
for path in re.findall(r"api\('(/admin/api/[^']+)'", APP)
|
||||
}
|
||||
self.assertTrue(called)
|
||||
for path in called:
|
||||
if path.startswith("/admin/api/sources/"):
|
||||
continue
|
||||
self.assertIn(f'"{path}"', HTTPAPP, f"{path} 前端在调,后端没路由")
|
||||
|
||||
def test_write_endpoints_are_reached_with_the_csrf_header(self) -> None:
|
||||
self.assertIn("X-CSRF-Token", APP)
|
||||
self.assertIn("check_csrf", HTTPAPP)
|
||||
|
||||
|
||||
def _token_block(selector: str) -> list[str]:
|
||||
start = STYLES.index(selector)
|
||||
body = STYLES[start:].split("}", 1)[0]
|
||||
return re.findall(r"(--[a-z0-9-]+):", body)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class StylesheetIntegrityTests(unittest.TestCase):
|
||||
"""HEL-560 改造中曾误把样式表尾部整段截断,抽屉/弹层/toast 全部失样,
|
||||
页面照样能跑、单测照样绿。这里把"每个仍在用的组件都得有样式"钉死。"""
|
||||
|
||||
def test_every_component_the_page_renders_still_has_its_own_rules(self) -> None:
|
||||
for selector in (
|
||||
".auth-wrap", ".drawer", ".drawer-mask", ".drawer-hd", ".drawer-tab",
|
||||
".modal-mask", ".modal-box", ".modal-actions", ".opbtn", ".empty-hint",
|
||||
".toast", ".row-fail", ".row-off", ".spark-end", ".cred-box", ".model-row",
|
||||
".vend-manual", ".gate-panel", ".dtable", ".invite-code", ".table-foot",
|
||||
".pbtn", ".field", ".form-hint",
|
||||
):
|
||||
self.assertIn(f"{selector} ", STYLES, f"{selector} 的样式丢了")
|
||||
|
||||
def test_stylesheet_braces_stay_balanced(self) -> None:
|
||||
body = STYLES[STYLES.index("/* ================= index.css"):]
|
||||
self.assertEqual(body.count("{"), body.count("}"))
|
||||
@@ -20,7 +20,6 @@ class ExtendedEodTests(unittest.TestCase):
|
||||
port=0,
|
||||
encryption_key=key,
|
||||
api_token="k" * 32,
|
||||
admin_password="StartPass1",
|
||||
tushare_token="tushare-secret",
|
||||
db_path=Path(self.tmp.name) / "hub.db",
|
||||
backup_dir=Path(self.tmp.name) / "backups",
|
||||
|
||||
@@ -0,0 +1,344 @@
|
||||
"""HEL-529 rework regressions: staging dedupe, anomaly convergence,
|
||||
source-catalog observation join (dataset-name ↔ interface-name), lineage
|
||||
update_freq. All read-only or within-batch fixes; none touch routing, the
|
||||
8765 main site, or the 问天 frozen zone.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from datahub.adapters.tushare import TushareAdapter
|
||||
from datahub.crypto import SecretVault
|
||||
from datahub.hub import Hub
|
||||
from datahub.pipeline import _dedupe_staging_rows
|
||||
from datahub.settings import Settings
|
||||
from tests.fixtures import fake_transport
|
||||
|
||||
|
||||
class StagingDedupeTests(unittest.TestCase):
|
||||
def test_popularity_within_batch_duplicates_collapse_keep_last(self) -> None:
|
||||
rows = [
|
||||
{"ts_code": "600000.SH", "trade_date": "20260914", "source": "ths", "rank": 1},
|
||||
{"ts_code": "000868.SZ", "trade_date": "20260914", "source": "dc", "rank": 2},
|
||||
{"ts_code": "600000.SH", "trade_date": "20260914", "source": "dc", "rank": 3, "hot": 9.9},
|
||||
{"ts_code": "600000.SH", "trade_date": "20260914", "source": "dc", "rank": 4, "hot": 8.8},
|
||||
]
|
||||
out = _dedupe_staging_rows("popularity", rows)
|
||||
self.assertEqual(len(out), 3) # (600000,ths) (000868,dc) (600000,dc)
|
||||
dup = [r for r in out if r["ts_code"] == "600000.SH" and r["source"] == "dc"][0]
|
||||
self.assertEqual(dup["rank"], 4) # keeps LAST occurrence
|
||||
self.assertEqual(out[0]["ts_code"], "600000.SH") # preserves first-seen order
|
||||
|
||||
def test_dragon_tiger_seat_duplicates_collapse(self) -> None:
|
||||
rows = [
|
||||
{"ts_code": "300010.SZ", "trade_date": "20260914", "hm_name": "T王", "buy_amount": 100},
|
||||
{"ts_code": "300010.SZ", "trade_date": "20260914", "hm_name": "T王", "buy_amount": 200},
|
||||
{"ts_code": "300010.SZ", "trade_date": "20260914", "hm_name": "T王", "buy_amount": 300},
|
||||
]
|
||||
out = _dedupe_staging_rows("dragon_tiger", rows)
|
||||
self.assertEqual(len(out), 1)
|
||||
self.assertEqual(out[0]["buy_amount"], 300)
|
||||
|
||||
def test_different_sources_are_not_duplicates(self) -> None:
|
||||
rows = [
|
||||
{"ts_code": "600000.SH", "trade_date": "20260914", "source": "ths"},
|
||||
{"ts_code": "600000.SH", "trade_date": "20260914", "source": "dc"},
|
||||
]
|
||||
self.assertEqual(len(_dedupe_staging_rows("popularity", rows)), 2)
|
||||
|
||||
def test_unknown_dataset_passthrough(self) -> None:
|
||||
rows = [{"a": 1}, {"a": 1}]
|
||||
self.assertEqual(_dedupe_staging_rows("calendar", rows), rows)
|
||||
|
||||
|
||||
class _Base(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
settings = Settings(
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="z" * 32,
|
||||
tushare_token="real-tushare-token-abcdef",
|
||||
db_path=Path(self.tmp.name) / "hub.db",
|
||||
scheduler_enabled=False,
|
||||
)
|
||||
self.hub = Hub(settings, adapter=TushareAdapter("real-tushare-token-abcdef", transport=fake_transport))
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
|
||||
class StagingDedupePublishTests(_Base):
|
||||
def test_duplicate_popularity_and_dragon_tiger_now_publish(self) -> None:
|
||||
"""Replays the 2026-09-14 production failure: within-response duplicate
|
||||
keys used to abort the whole batch at the staging INSERT; with dedupe
|
||||
the same upstream payload publishes."""
|
||||
db = self.hub.db
|
||||
trade_date = "20240902"
|
||||
# dc_hot returns 600000.SH twice within one response; hm_detail returns
|
||||
# the same (ts_code, hm_name) seat three times (mirrors live evidence).
|
||||
popularity_rows = [
|
||||
{"ts_code": "600000.SH", "trade_date": trade_date, "source": "ths",
|
||||
"ts_name": "浦发银行", "rank": 1, "pct_change": 1.2, "current_price": 10.2,
|
||||
"hot": 90.0, "concept": "银行", "data_type": "热股"},
|
||||
{"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc",
|
||||
"ts_name": "浦发银行", "rank": 2, "pct_change": 1.2, "current_price": 10.2,
|
||||
"hot": 80.0, "concept": "银行", "data_type": "A股市场"},
|
||||
{"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc",
|
||||
"ts_name": "浦发银行", "rank": 3, "pct_change": 1.3, "current_price": 10.3,
|
||||
"hot": 81.0, "concept": "银行", "data_type": "A股市场"},
|
||||
]
|
||||
dragon_rows = [
|
||||
{"trade_date": trade_date, "ts_code": "600000.SH", "ts_name": "浦发银行",
|
||||
"buy_amount": 100, "sell_amount": 200, "net_amount": -100,
|
||||
"hm_name": "测试游资", "hm_orgs": "某某营业部", "tag": "超买"},
|
||||
{"trade_date": trade_date, "ts_code": "600000.SH", "ts_name": "浦发银行",
|
||||
"buy_amount": 300, "sell_amount": 0, "net_amount": 300,
|
||||
"hm_name": "测试游资", "hm_orgs": "某某营业部", "tag": "超买"},
|
||||
]
|
||||
self.hub.pipeline._stage("popularity", "b-dup-pop", popularity_rows)
|
||||
self.hub.pipeline._stage("dragon_tiger", "b-dup-dt", dragon_rows)
|
||||
pop = db.fetchall("SELECT * FROM staging_popularity WHERE batch_id = 'b-dup-pop'")
|
||||
dt = db.fetchall("SELECT * FROM staging_dragon_tiger WHERE batch_id = 'b-dup-dt'")
|
||||
self.assertEqual(len(pop), 2)
|
||||
self.assertEqual(len(dt), 1)
|
||||
kept = dt[0]
|
||||
self.assertEqual(kept["buy_amount"], 300)
|
||||
|
||||
def test_hel562_popularity_collapsed_dups_publish_not_hard_fail(self) -> None:
|
||||
"""HEL-562: staging dedupe alone is not enough — quality gate used to
|
||||
hard-fail on the same within-batch dups after they were already
|
||||
collapsed (live 20260915: duplicate keys: 3 → integrity_gate)."""
|
||||
trade_date = "20240902"
|
||||
# 3 within-batch dups on (ts_code, trade_date, source=dc) — mirrors
|
||||
# ths+dc merge where dc_hot repeats the same keys.
|
||||
rows = [
|
||||
{"ts_code": "600000.SH", "trade_date": trade_date, "source": "ths",
|
||||
"ts_name": "浦发银行", "rank": 1, "pct_change": 1.2, "current_price": 10.2,
|
||||
"hot": 90.0, "concept": "银行", "data_type": "热股"},
|
||||
{"ts_code": "000001.SZ", "trade_date": trade_date, "source": "dc",
|
||||
"ts_name": "平安银行", "rank": 1, "pct_change": 2.0, "current_price": 11.0,
|
||||
"hot": 88.0, "concept": "银行", "data_type": "A股市场"},
|
||||
{"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc",
|
||||
"ts_name": "浦发银行", "rank": 2, "pct_change": 1.2, "current_price": 10.2,
|
||||
"hot": 80.0, "concept": "银行", "data_type": "A股市场"},
|
||||
{"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc",
|
||||
"ts_name": "浦发银行", "rank": 3, "pct_change": 1.3, "current_price": 10.3,
|
||||
"hot": 81.0, "concept": "银行", "data_type": "A股市场"},
|
||||
{"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc",
|
||||
"ts_name": "浦发银行", "rank": 4, "pct_change": 1.4, "current_price": 10.4,
|
||||
"hot": 82.0, "concept": "银行", "data_type": "A股市场"},
|
||||
]
|
||||
report = self.hub.pipeline.validate("popularity", "b-gate", trade_date, rows)
|
||||
self.assertFalse(report["hard_fail"])
|
||||
self.assertEqual(report["errors"], [])
|
||||
self.assertEqual(report["warnings"], ["duplicate keys: 2"])
|
||||
|
||||
result = self.hub.pipeline.run_dataset("popularity", trade_date, prepared_rows=rows)
|
||||
# warnings present → soft_fail → publication state is degraded (still served)
|
||||
self.assertEqual(result["state"], "degraded")
|
||||
self.assertFalse(result["quality"]["hard_fail"])
|
||||
self.assertTrue(result["quality"]["soft_fail"])
|
||||
self.assertIn("duplicate keys: 2", result["quality"]["warnings"])
|
||||
eod = self.hub.db.fetchall(
|
||||
"SELECT * FROM eod_popularity WHERE trade_date = ? AND batch_id = ?",
|
||||
(trade_date, result["batch_id"]),
|
||||
)
|
||||
# 5 raw → 3 unique keys after staging collapse (ths + two dc codes)
|
||||
self.assertEqual(len(eod), 3)
|
||||
pub = self.hub.db.fetchone(
|
||||
"SELECT * FROM publications WHERE dataset='popularity' AND trade_date=?",
|
||||
(trade_date,),
|
||||
)
|
||||
self.assertEqual(pub["active_batch"], result["batch_id"])
|
||||
self.assertEqual(pub["state"], "degraded")
|
||||
# Serving path accepts degraded the same as published (no DATASET_NOT_PUBLISHED)
|
||||
from datahub.serving import V1API
|
||||
|
||||
api = V1API(self.hub.db, self.hub.pipeline, self.hub.settings)
|
||||
payload = api.handle("/v1/popularity", {"date": [trade_date]})
|
||||
self.assertEqual(len(payload["data"]), 3)
|
||||
self.assertEqual(payload["meta"]["state"], "degraded")
|
||||
self.assertEqual(payload["meta"]["batch_id"], result["batch_id"])
|
||||
|
||||
def test_hel562_core_soft_still_hard_fails_on_duplicate_keys(self) -> None:
|
||||
"""moneyflow/auction stay on the old soft gate: raw dups → hard_fail."""
|
||||
trade_date = "20240902"
|
||||
rows = [
|
||||
{"ts_code": "600000.SH", "trade_date": trade_date,
|
||||
"buy_sm_amount": 1, "sell_sm_amount": 1, "buy_md_amount": 1, "sell_md_amount": 1,
|
||||
"buy_lg_amount": 1, "sell_lg_amount": 1, "buy_elg_amount": 1, "sell_elg_amount": 1,
|
||||
"net_mf_amount": 0},
|
||||
{"ts_code": "600000.SH", "trade_date": trade_date,
|
||||
"buy_sm_amount": 2, "sell_sm_amount": 2, "buy_md_amount": 2, "sell_md_amount": 2,
|
||||
"buy_lg_amount": 2, "sell_lg_amount": 2, "buy_elg_amount": 2, "sell_elg_amount": 2,
|
||||
"net_mf_amount": 0},
|
||||
]
|
||||
report = self.hub.pipeline.validate("moneyflow", "b-mf", trade_date, rows)
|
||||
self.assertTrue(report["hard_fail"])
|
||||
self.assertIn("duplicate keys: 1", report["errors"])
|
||||
self.assertEqual(report["warnings"], [])
|
||||
|
||||
def test_hel562_popularity_date_mismatch_still_hard_fails(self) -> None:
|
||||
"""Collapsed-dup carve-out must not weaken other soft integrity checks."""
|
||||
rows = [
|
||||
{"ts_code": "600000.SH", "trade_date": "20240901", "source": "ths",
|
||||
"ts_name": "浦发银行", "rank": 1, "pct_change": 1.2, "current_price": 10.2,
|
||||
"hot": 90.0, "concept": "银行", "data_type": "热股"},
|
||||
]
|
||||
report = self.hub.pipeline.validate("popularity", "b-bad-date", "20240902", rows)
|
||||
self.assertTrue(report["hard_fail"])
|
||||
self.assertIn("date mismatch rows: 1", report["errors"])
|
||||
|
||||
|
||||
class OverviewAnomalyConvergenceTests(_Base):
|
||||
def _seed_batches(self, today: str) -> None:
|
||||
db = self.hub.db
|
||||
rows = [
|
||||
# resolved history: earlier failures/stalls, later success
|
||||
("x-daily-001", "daily", "staged", "empty official batch", "2026-09-14T15:05:00+08:00"),
|
||||
("x-daily-002", "daily", "failed", "release group not switched", "2026-09-14T16:10:00+08:00"),
|
||||
("x-daily-006", "daily", "published", "", "2026-09-14T20:00:00+08:00"),
|
||||
# current unresolved faults
|
||||
("x-pop-001", "popularity", "failed", "UNIQUE constraint failed: staging_popularity", "2026-09-14T22:40:00+08:00"),
|
||||
("x-dt-001", "dragon_tiger", "failed", "UNIQUE constraint failed: staging_dragon_tiger", "2026-09-14T16:45:00+08:00"),
|
||||
("x-dt-002", "dragon_tiger", "failed", "UNIQUE constraint failed: staging_dragon_tiger", "2026-09-14T21:48:00+08:00"),
|
||||
# staged-empty later published
|
||||
("x-idx-001", "index_daily", "staged", "empty official batch", "2026-09-14T15:10:00+08:00"),
|
||||
("x-idx-003", "index_daily", "published", "", "2026-09-14T16:10:00+08:00"),
|
||||
]
|
||||
for batch_id, dataset, state, error, started in rows:
|
||||
db.execute(
|
||||
"INSERT INTO batches(batch_id, dataset, trade_date, state, attempt, rows_in, rows_out,"
|
||||
" quality_json, started_at, finished_at, error) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(batch_id, dataset, today, state, 1, None, None, None, started, started if state != "staged" else None, error or None),
|
||||
)
|
||||
db.execute(
|
||||
"INSERT INTO publications(dataset, trade_date, active_batch, prev_batch, state, published_at)"
|
||||
" VALUES ('daily', ?, 'x-daily-006', 'x-daily-005', 'published', '2026-09-14T20:00:22+08:00')",
|
||||
(today,),
|
||||
)
|
||||
db.execute(
|
||||
"INSERT INTO publications(dataset, trade_date, active_batch, prev_batch, state, published_at)"
|
||||
" VALUES ('index_daily', ?, 'x-idx-003', 'x-idx-002', 'published', '2026-09-14T16:10:12+08:00')",
|
||||
(today,),
|
||||
)
|
||||
|
||||
def test_anomalies_only_latest_unresolved(self) -> None:
|
||||
overview = self.hub.admin.overview()
|
||||
today = overview["trade_date"]
|
||||
self._seed_batches(today)
|
||||
anomalies = self.hub.admin.overview()["anomalies"]
|
||||
got = sorted((a["dataset"], a["batch_id"]) for a in anomalies)
|
||||
self.assertEqual(
|
||||
got,
|
||||
[
|
||||
("dragon_tiger", "x-dt-002"), # latest failed, never published
|
||||
("popularity", "x-pop-001"), # latest failed, never published
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class SourceCatalogJoinTests(_Base):
|
||||
def test_observation_join_matches_dataset_named_health_rows(self) -> None:
|
||||
db = self.hub.db
|
||||
now = "2026-09-15T08:00:00+08:00"
|
||||
# tushare observability writes dataset names (real legacy behavior)
|
||||
for iface, state in [("valuation", "ok"), ("popularity", "ok"), ("stocks", "ok")]:
|
||||
db.execute(
|
||||
"INSERT INTO provider_health(provider, interface, state, last_ok_at, last_error,"
|
||||
" last_fallback_reason, consec_failures, last_latency_ms, last_data_age_seconds, updated_at)"
|
||||
" VALUES ('tushare', ?, ?, ?, '', '', 0, 300, NULL, ?)",
|
||||
(iface, state, now, now),
|
||||
)
|
||||
# realtime providers write interface names
|
||||
db.execute(
|
||||
"INSERT INTO provider_health(provider, interface, state, last_ok_at, last_error,"
|
||||
" last_fallback_reason, consec_failures, last_latency_ms, last_data_age_seconds, updated_at)"
|
||||
" VALUES ('eastmoney', 'indices', 'ok', ?, '', '', 0, 153, NULL, ?)",
|
||||
(now, now),
|
||||
)
|
||||
items = self.hub.admin.source_catalog()["items"]
|
||||
tushare = [i for i in items if i["provider"] == "tushare"][0]
|
||||
by_iface = {i["interface"]: i for i in tushare["interfaces"]}
|
||||
# daily_basic serves valuation → observed via dataset name
|
||||
self.assertTrue(by_iface["daily_basic"]["observed"])
|
||||
self.assertEqual(by_iface["daily_basic"]["observed_basis"], "dataset")
|
||||
self.assertEqual(by_iface["daily_basic"]["observed_state"], "ok")
|
||||
# ths_hot + dc_hot serve popularity → observed via dataset name
|
||||
self.assertTrue(by_iface["ths_hot"]["observed"])
|
||||
self.assertTrue(by_iface["dc_hot"]["observed"])
|
||||
# stock_basic serves stocks
|
||||
self.assertTrue(by_iface["stock_basic"]["observed"])
|
||||
# never-observed interface stays honestly unobserved (not "unconfigured")
|
||||
self.assertFalse(by_iface["trade_cal"]["observed"])
|
||||
self.assertEqual(by_iface["trade_cal"]["observed_state"], "")
|
||||
# interfaces carry real batch groups
|
||||
groups = {i["interface"]: i["group"] for i in tushare["interfaces"]}
|
||||
self.assertEqual(groups["daily"], "盘后 A 批")
|
||||
self.assertEqual(groups["ths_hot"], "扩展软批")
|
||||
self.assertEqual(groups["index_daily"], "指数 B 批")
|
||||
eastmoney = [i for i in items if i["provider"] == "eastmoney"][0]
|
||||
em = {i["interface"]: i for i in eastmoney["interfaces"]}
|
||||
self.assertTrue(em["indices"]["observed"])
|
||||
self.assertEqual(em["indices"]["observed_basis"], "interface")
|
||||
self.assertFalse(em["market_quotes"]["observed"])
|
||||
|
||||
def test_lineage_update_freq_present(self) -> None:
|
||||
items = self.hub.admin.lineage("20240902")["items"]
|
||||
self.assertTrue(items)
|
||||
for item in items:
|
||||
self.assertTrue(item.get("update_freq"), f"missing update_freq for {item['dataset']}")
|
||||
|
||||
|
||||
class LineageMentorIfindTests(_Base):
|
||||
"""问师 → iFinD 血缘修正(第二轮返工):不按名称猜关系,按真实调用代码。"""
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
|
||||
def test_no_mentor_dependency_on_ifind_wencai(self) -> None:
|
||||
items = self.hub.admin.lineage("20240902")["items"]
|
||||
wencai = [i for i in items if i["dataset"] == "ifind_wencai"]
|
||||
self.assertEqual(len(wencai), 1)
|
||||
consumers = wencai[0]["known_consumers"]
|
||||
for consumer in consumers:
|
||||
self.assertNotIn("问师", consumer, f"wencai consumer must not be 问师: {consumer}")
|
||||
all_consumers = " ".join(c for i in items for c in i["known_consumers"])
|
||||
self.assertNotIn("问师(自然语言选股", all_consumers)
|
||||
|
||||
def test_wencai_real_consumer_is_pools_enrichment_with_code_evidence(self) -> None:
|
||||
items = self.hub.admin.lineage("20240902")["items"]
|
||||
wencai = [i for i in items if i["dataset"] == "ifind_wencai"][0]
|
||||
self.assertTrue(any("股票池" in c for c in wencai["known_consumers"]))
|
||||
# Real call evidence in the main-site source tree:
|
||||
pools_src = (self.REPO / "backend" / "features" / "pools" / "service.py").read_text(encoding="utf-8")
|
||||
self.assertIn("ifind.wencai(", pools_src)
|
||||
self.assertIn("ifind_event_enrichment_v1", pools_src)
|
||||
# And 问师 itself never calls wencai:
|
||||
mentor_src = (self.REPO / "backend" / "features" / "mentor" / "service.py").read_text(encoding="utf-8")
|
||||
self.assertNotIn(".wencai(", mentor_src)
|
||||
|
||||
def test_mentor_optional_ifind_history_subcapabilities(self) -> None:
|
||||
items = self.hub.admin.lineage("20240902")["items"]
|
||||
history = [i for i in items if i["dataset"] == "ifind_history"]
|
||||
self.assertEqual(len(history), 1)
|
||||
consumers = history[0]["known_consumers"]
|
||||
self.assertTrue(any("趋势思维模型" in c for c in consumers))
|
||||
self.assertTrue(any("宏观思维模型" in c for c in consumers))
|
||||
# every consumer must be a 问师 sub-capability, not the whole board
|
||||
for consumer in consumers:
|
||||
self.assertIn("·", consumer, f"not a sub-capability mapping: {consumer}")
|
||||
# Real call evidence: mentor builds market matrices via ifind.history
|
||||
mentor_src = (self.REPO / "backend" / "features" / "mentor" / "service.py").read_text(encoding="utf-8")
|
||||
self.assertIn("ifind.history(", mentor_src)
|
||||
self.assertIn("_mentor_market_matrix", mentor_src)
|
||||
self.assertIn("MENTOR_INDEX_UNIVERSE", mentor_src)
|
||||
self.assertIn("MENTOR_ETF_UNIVERSE", mentor_src)
|
||||
# Optional dependency: fails open when ifind is not configured
|
||||
self.assertIn("if not ifind or not ifind.configured", mentor_src)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,105 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from datahub.adapters.ifind import IfindAdapter
|
||||
from datahub.db import HubDB
|
||||
from datahub.serving import ApiError
|
||||
from datahub.steward import steward_query
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, payload: dict, status: int = 200) -> None:
|
||||
import json
|
||||
|
||||
self.status = status
|
||||
self._raw = json.dumps(payload).encode("utf-8")
|
||||
|
||||
def read(self):
|
||||
return self._raw
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
|
||||
class IfindObservabilityTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db = HubDB(Path(self.tmp.name) / "hub.db")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _adapter_with_urlopen(self, urlopen) -> IfindAdapter:
|
||||
return IfindAdapter(refresh_token="rt", access_token="at", urlopen=urlopen)
|
||||
|
||||
def test_successful_fetch_is_logged_without_changing_rows(self) -> None:
|
||||
def urlopen(request, timeout=None):
|
||||
return _Resp(
|
||||
{
|
||||
"errorcode": 0,
|
||||
"tables": [{"thscode": ["000001.SZ"], "table": {"涨停原因": ["重组"]}}],
|
||||
}
|
||||
)
|
||||
|
||||
adapter = self._adapter_with_urlopen(urlopen)
|
||||
|
||||
class _Api:
|
||||
ifind = adapter
|
||||
db = self.db
|
||||
|
||||
payload = steward_query(
|
||||
_Api(), {"api_name": "ifind_wencai", "params": {"query": "涨停"}}
|
||||
)
|
||||
self.assertEqual(payload["data"][0]["thscode"], "000001.SZ")
|
||||
log = self.db.fetchone("SELECT * FROM provider_call_log WHERE provider = 'ifind'")
|
||||
self.assertIsNotNone(log)
|
||||
self.assertEqual(log["interface"], "wencai")
|
||||
self.assertEqual(log["status"], "ok")
|
||||
|
||||
def test_failed_fetch_reraises_and_logs_error(self) -> None:
|
||||
def urlopen(request, timeout=None):
|
||||
return _Resp({"errorcode": -9999, "errmsg": "quota exceeded"})
|
||||
|
||||
adapter = self._adapter_with_urlopen(urlopen)
|
||||
|
||||
class _Api:
|
||||
ifind = adapter
|
||||
db = self.db
|
||||
|
||||
with self.assertRaises(ApiError) as ctx:
|
||||
steward_query(_Api(), {"api_name": "ifind_wencai", "params": {"query": "涨停"}})
|
||||
self.assertEqual(ctx.exception.code, "SOURCE_UNAVAILABLE")
|
||||
log = self.db.fetchone("SELECT * FROM provider_call_log WHERE provider = 'ifind'")
|
||||
self.assertIsNotNone(log)
|
||||
self.assertEqual(log["status"], "error")
|
||||
|
||||
def test_status_check_alone_does_not_dial_or_log_a_fetch_call(self) -> None:
|
||||
class _Api:
|
||||
ifind = IfindAdapter()
|
||||
db = self.db
|
||||
|
||||
payload = steward_query(_Api(), {"api_name": "ifind_status", "params": {}})
|
||||
self.assertFalse(payload["data"][0]["configured"])
|
||||
log = self.db.fetchall("SELECT * FROM provider_call_log WHERE provider = 'ifind'")
|
||||
self.assertEqual(log, [])
|
||||
|
||||
def test_api_double_without_db_attribute_still_works(self) -> None:
|
||||
# Mirrors tests/test_ifind_adapter.py's `_Api` double, which has no
|
||||
# `db` attribute at all. Observability must not require it.
|
||||
class _Api:
|
||||
ifind = IfindAdapter()
|
||||
|
||||
payload = steward_query(_Api(), {"api_name": "ifind_status", "params": {}})
|
||||
self.assertFalse(payload["data"][0]["configured"])
|
||||
with self.assertRaises(ApiError):
|
||||
steward_query(_Api(), {"api_name": "ifind_wencai", "params": {"query": "涨停"}})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,306 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from datahub import observability
|
||||
from datahub.db import HubDB
|
||||
|
||||
|
||||
class _BrokenDB:
|
||||
"""A db double whose write() always raises, to prove fail-open."""
|
||||
|
||||
@contextmanager
|
||||
def write(self):
|
||||
raise RuntimeError("disk is full")
|
||||
yield None # pragma: no cover - unreachable, keeps this a generator
|
||||
|
||||
def fetchall(self, sql, params=()):
|
||||
raise RuntimeError("disk is full")
|
||||
|
||||
def fetchone(self, sql, params=()):
|
||||
raise RuntimeError("disk is full")
|
||||
|
||||
|
||||
class ClassifyRowsTests(unittest.TestCase):
|
||||
def test_empty_list_is_flagged_empty(self):
|
||||
status, reason, age = observability.classify_rows([])
|
||||
self.assertEqual(status, "empty")
|
||||
self.assertEqual(reason, "no_rows_returned")
|
||||
self.assertIsNone(age)
|
||||
|
||||
def test_empty_dict_result_is_flagged_empty(self):
|
||||
status, reason, _ = observability.classify_rows({})
|
||||
self.assertEqual(status, "empty")
|
||||
self.assertEqual(reason, "no_rows_returned")
|
||||
|
||||
def test_missing_required_field_is_flagged(self):
|
||||
rows = [{"ts_code": "600000.SH", "close": 10.2}, {"ts_code": "000001.SZ"}]
|
||||
status, reason, _ = observability.classify_rows(rows, required_fields=("close",), freshness_field=None)
|
||||
self.assertEqual(status, "missing_fields")
|
||||
self.assertIn("close", reason)
|
||||
|
||||
def test_fresh_rows_are_ok(self):
|
||||
import time
|
||||
|
||||
rows = [{"ts_code": "600000.SH", "close": 10.2, "quote_time_epoch": int(time.time())}]
|
||||
status, reason, age = observability.classify_rows(rows)
|
||||
self.assertEqual(status, "ok")
|
||||
self.assertEqual(reason, "")
|
||||
self.assertIsNotNone(age)
|
||||
self.assertLess(age, 5)
|
||||
|
||||
def test_stale_rows_are_flagged(self):
|
||||
import time
|
||||
|
||||
rows = [{"ts_code": "600000.SH", "close": 10.2, "quote_time_epoch": int(time.time()) - 3600}]
|
||||
status, reason, age = observability.classify_rows(rows, max_age_seconds=300)
|
||||
self.assertEqual(status, "stale")
|
||||
self.assertEqual(reason, "data_age_exceeds_threshold")
|
||||
self.assertGreaterEqual(age, 3600 - 5)
|
||||
|
||||
def test_classifier_never_raises_on_garbage_input(self):
|
||||
status, reason, age = observability.classify_rows(object())
|
||||
self.assertEqual(status, "empty")
|
||||
self.assertIsNone(age)
|
||||
# Malformed rows inside a list must not raise either.
|
||||
status, _, _ = observability.classify_rows(["not-a-dict", 123, None])
|
||||
self.assertEqual(status, "empty")
|
||||
|
||||
|
||||
class ClassifyErrorTests(unittest.TestCase):
|
||||
def test_blocked_page_markers_are_detected(self):
|
||||
status, reason = observability.classify_error("eastmoney request failed: Expecting value: line 1 column 1")
|
||||
self.assertEqual(status, "blocked")
|
||||
self.assertEqual(reason, "response_looks_like_intercept_page")
|
||||
|
||||
def test_timeout_is_detected(self):
|
||||
status, _ = observability.classify_error("tencent request failed: timed out")
|
||||
self.assertEqual(status, "timeout")
|
||||
|
||||
def test_generic_error_falls_back(self):
|
||||
status, reason = observability.classify_error("connection reset by peer")
|
||||
self.assertEqual(status, "error")
|
||||
self.assertEqual(reason, "")
|
||||
|
||||
def test_never_raises_on_none(self):
|
||||
status, reason = observability.classify_error(None)
|
||||
self.assertEqual(status, "error")
|
||||
|
||||
|
||||
class RecordCallTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db = HubDB(Path(self.tmp.name) / "hub.db")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_record_call_writes_log_and_health(self):
|
||||
observability.record_call(self.db, "eastmoney", "indices", status="ok", latency_ms=42)
|
||||
log_rows = self.db.fetchall("SELECT * FROM provider_call_log")
|
||||
self.assertEqual(len(log_rows), 1)
|
||||
self.assertEqual(log_rows[0]["provider"], "eastmoney")
|
||||
self.assertEqual(log_rows[0]["interface"], "indices")
|
||||
self.assertEqual(log_rows[0]["status"], "ok")
|
||||
health = self.db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = ? AND interface = ?",
|
||||
("eastmoney", "indices"),
|
||||
)
|
||||
self.assertIsNotNone(health)
|
||||
self.assertEqual(health["state"], "ok")
|
||||
self.assertEqual(health["consec_failures"], 0)
|
||||
|
||||
def test_consecutive_failures_increment_and_reset(self):
|
||||
observability.record_call(self.db, "tencent", "named_quotes", status="error", error="boom")
|
||||
observability.record_call(self.db, "tencent", "named_quotes", status="error", error="boom again")
|
||||
health = self.db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = ? AND interface = ?",
|
||||
("tencent", "named_quotes"),
|
||||
)
|
||||
self.assertEqual(health["consec_failures"], 2)
|
||||
self.assertEqual(health["state"], "error")
|
||||
observability.record_call(self.db, "tencent", "named_quotes", status="ok")
|
||||
health = self.db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = ? AND interface = ?",
|
||||
("tencent", "named_quotes"),
|
||||
)
|
||||
self.assertEqual(health["consec_failures"], 0)
|
||||
self.assertEqual(health["state"], "ok")
|
||||
|
||||
def test_none_db_is_a_silent_noop(self):
|
||||
# Must not raise even though there is nowhere to write.
|
||||
observability.record_call(None, "ifind", "wencai", status="ok")
|
||||
|
||||
def test_broken_db_write_does_not_raise(self):
|
||||
observability.record_call(_BrokenDB(), "eastmoney", "indices", status="ok")
|
||||
|
||||
|
||||
class ObserveTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db = HubDB(Path(self.tmp.name) / "hub.db")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_returns_exact_success_value_unmodified(self):
|
||||
sentinel = {"ts_code": "600000.SH", "close": 10.2}
|
||||
result = observability.observe(self.db, "eastmoney", "indices", lambda: sentinel)
|
||||
self.assertIs(result, sentinel)
|
||||
rows = self.db.fetchall("SELECT * FROM provider_call_log")
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["status"], "ok")
|
||||
|
||||
def test_reraises_exact_exception_on_failure(self):
|
||||
boom = ValueError("upstream exploded")
|
||||
|
||||
def fn():
|
||||
raise boom
|
||||
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
observability.observe(self.db, "eastmoney", "indices", fn)
|
||||
self.assertIs(ctx.exception, boom)
|
||||
rows = self.db.fetchall("SELECT * FROM provider_call_log")
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["status"], "error")
|
||||
self.assertIn("upstream exploded", rows[0]["error"])
|
||||
|
||||
def test_classify_downgrades_success_to_stale_without_changing_return_value(self):
|
||||
sentinel = [{"ts_code": "600000.SH", "quote_time_epoch": 1}]
|
||||
result = observability.observe(
|
||||
self.db, "eastmoney", "indices", lambda: sentinel,
|
||||
classify=lambda rows: observability.classify_rows(rows),
|
||||
)
|
||||
self.assertIs(result, sentinel)
|
||||
row = self.db.fetchone("SELECT * FROM provider_call_log")
|
||||
self.assertEqual(row["status"], "stale")
|
||||
|
||||
def test_broken_db_never_breaks_a_successful_call(self):
|
||||
sentinel = {"ok": True}
|
||||
result = observability.observe(_BrokenDB(), "eastmoney", "indices", lambda: sentinel)
|
||||
self.assertIs(result, sentinel)
|
||||
|
||||
def test_broken_db_never_masks_a_real_failure(self):
|
||||
def fn():
|
||||
raise RuntimeError("real upstream failure")
|
||||
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
observability.observe(_BrokenDB(), "eastmoney", "indices", fn)
|
||||
self.assertEqual(str(ctx.exception), "real upstream failure")
|
||||
|
||||
def test_classifier_exception_does_not_break_the_call(self):
|
||||
sentinel = {"ok": True}
|
||||
|
||||
def bad_classify(_result):
|
||||
raise KeyError("classifier bug")
|
||||
|
||||
result = observability.observe(self.db, "eastmoney", "indices", lambda: sentinel, classify=bad_classify)
|
||||
self.assertIs(result, sentinel)
|
||||
row = self.db.fetchone("SELECT * FROM provider_call_log")
|
||||
# A classifier bug must degrade to "ok", never silently drop the row
|
||||
# nor claim the call failed when it did not.
|
||||
self.assertEqual(row["status"], "ok")
|
||||
|
||||
def test_none_db_is_transparent_passthrough(self):
|
||||
sentinel = object()
|
||||
result = observability.observe(None, "eastmoney", "indices", lambda: sentinel)
|
||||
self.assertIs(result, sentinel)
|
||||
|
||||
|
||||
class _ToggleDB(HubDB):
|
||||
"""A real HubDB subclass so we can flip the HEL-543 kill switch the same
|
||||
way Hub.__init__ does, without needing a full Hub/Settings wiring."""
|
||||
|
||||
|
||||
class KillSwitchTests(unittest.TestCase):
|
||||
"""HEL-543 total-review 🔴: the observability side channel must be
|
||||
disable-able at runtime, and disabling it must leave existing behavior
|
||||
completely unchanged (pure passthrough, zero db access)."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db = _ToggleDB(Path(self.tmp.name) / "hub.db")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_is_enabled_defaults_true_when_attribute_absent(self):
|
||||
# A bare HubDB (as used throughout the rest of this test suite, and
|
||||
# by any pre-HEL-543 call site) must default to enabled.
|
||||
self.assertTrue(observability.is_enabled(self.db))
|
||||
self.assertTrue(observability.is_enabled(None))
|
||||
|
||||
def test_disabled_record_call_writes_nothing(self):
|
||||
self.db.observability_enabled = False
|
||||
observability.record_call(self.db, "eastmoney", "indices", status="ok", latency_ms=1)
|
||||
self.assertEqual(self.db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
self.assertEqual(self.db.fetchall("SELECT * FROM provider_health"), [])
|
||||
|
||||
def test_disabled_observe_is_a_pure_passthrough_on_success(self):
|
||||
self.db.observability_enabled = False
|
||||
sentinel = {"ts_code": "600000.SH"}
|
||||
calls = {"n": 0}
|
||||
|
||||
def fn():
|
||||
calls["n"] += 1
|
||||
return sentinel
|
||||
|
||||
result = observability.observe(self.db, "eastmoney", "indices", fn)
|
||||
self.assertIs(result, sentinel)
|
||||
self.assertEqual(calls["n"], 1)
|
||||
self.assertEqual(self.db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
|
||||
def test_disabled_observe_still_reraises_the_exact_exception(self):
|
||||
self.db.observability_enabled = False
|
||||
boom = RuntimeError("upstream exploded")
|
||||
|
||||
def fn():
|
||||
raise boom
|
||||
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
observability.observe(self.db, "eastmoney", "indices", fn)
|
||||
self.assertIs(ctx.exception, boom)
|
||||
self.assertEqual(self.db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
|
||||
def test_re_enabling_resumes_recording(self):
|
||||
self.db.observability_enabled = False
|
||||
observability.observe(self.db, "eastmoney", "indices", lambda: {"ok": True})
|
||||
self.assertEqual(self.db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
self.db.observability_enabled = True
|
||||
observability.observe(self.db, "eastmoney", "indices", lambda: {"ok": True})
|
||||
self.assertEqual(len(self.db.fetchall("SELECT * FROM provider_call_log")), 1)
|
||||
|
||||
|
||||
class SettingsToggleTests(unittest.TestCase):
|
||||
"""The kill switch follows the same env-var pattern as DATAHUB_SCHEDULER."""
|
||||
|
||||
def test_defaults_to_enabled(self):
|
||||
from datahub.settings import load_settings
|
||||
|
||||
settings = load_settings(env={})
|
||||
self.assertTrue(settings.observability_enabled)
|
||||
|
||||
def test_datahub_observability_zero_disables(self):
|
||||
from datahub.settings import load_settings
|
||||
|
||||
settings = load_settings(env={"DATAHUB_OBSERVABILITY": "0"})
|
||||
self.assertFalse(settings.observability_enabled)
|
||||
|
||||
def test_datahub_observability_off_disables(self):
|
||||
from datahub.settings import load_settings
|
||||
|
||||
settings = load_settings(env={"DATAHUB_OBSERVABILITY": "off"})
|
||||
self.assertFalse(settings.observability_enabled)
|
||||
|
||||
def test_datahub_observability_one_keeps_enabled(self):
|
||||
from datahub.settings import load_settings
|
||||
|
||||
settings = load_settings(env={"DATAHUB_OBSERVABILITY": "1"})
|
||||
self.assertTrue(settings.observability_enabled)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -53,7 +53,6 @@ def make_pipeline(before_commit=None, clock=None, quality=None) -> tuple[Pipelin
|
||||
settings = Settings(
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="t" * 32,
|
||||
admin_password="admin-pass",
|
||||
tushare_token="test-token",
|
||||
db_path=db.path,
|
||||
quality=quality_cfg,
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from datahub.pipeline import RetryError
|
||||
from tests.fixtures import TRADE_DATE
|
||||
from tests.test_pipeline import make_pipeline
|
||||
|
||||
|
||||
class PipelineObservabilityTests(unittest.TestCase):
|
||||
"""HEL-543: Tushare calls must keep writing the existing `src_calls`
|
||||
record unchanged, while also feeding the new cross-provider
|
||||
`provider_call_log` / `provider_health` side channel."""
|
||||
|
||||
def test_successful_fetch_logs_to_both_src_calls_and_provider_call_log(self) -> None:
|
||||
pipe, db = make_pipeline()
|
||||
pipe.ingest_reference(TRADE_DATE)
|
||||
result = pipe.run_dataset("daily", TRADE_DATE)
|
||||
self.assertEqual(result["state"], "published")
|
||||
|
||||
src_calls = db.fetchall("SELECT * FROM src_calls WHERE provider = 'tushare' AND endpoint = 'daily'")
|
||||
self.assertTrue(any(row["ok"] == 1 for row in src_calls))
|
||||
|
||||
log = db.fetchall(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'tushare' AND interface = 'daily'"
|
||||
)
|
||||
self.assertTrue(len(log) >= 1)
|
||||
self.assertEqual(log[-1]["status"], "ok")
|
||||
|
||||
health = db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = 'tushare' AND interface = 'daily'"
|
||||
)
|
||||
self.assertIsNotNone(health)
|
||||
self.assertEqual(health["state"], "ok")
|
||||
self.assertEqual(health["consec_failures"], 0)
|
||||
|
||||
def test_failed_fetch_logs_error_to_both_channels_and_still_raises(self) -> None:
|
||||
pipe, db = make_pipeline()
|
||||
pipe.ingest_reference(TRADE_DATE)
|
||||
|
||||
def boom(dataset, params):
|
||||
raise RuntimeError("tushare upstream 500")
|
||||
|
||||
pipe.adapter.fetch = boom # type: ignore[assignment]
|
||||
|
||||
with self.assertRaises(RetryError):
|
||||
pipe.run_dataset("daily", TRADE_DATE, attempts=1)
|
||||
|
||||
src_calls = db.fetchall("SELECT * FROM src_calls WHERE provider = 'tushare' AND endpoint = 'daily' AND ok = 0")
|
||||
self.assertTrue(len(src_calls) >= 1)
|
||||
self.assertIn("tushare upstream 500", src_calls[-1]["error"])
|
||||
|
||||
log = db.fetchall(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'tushare' AND interface = 'daily' AND status != 'ok'"
|
||||
)
|
||||
self.assertTrue(len(log) >= 1)
|
||||
self.assertIn("tushare upstream 500", log[-1]["error"])
|
||||
|
||||
health = db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = 'tushare' AND interface = 'daily'"
|
||||
)
|
||||
self.assertIsNotNone(health)
|
||||
self.assertNotEqual(health["state"], "ok")
|
||||
self.assertGreaterEqual(health["consec_failures"], 1)
|
||||
|
||||
def test_provider_call_log_is_purged_by_existing_cleanup_job(self) -> None:
|
||||
pipe, db = make_pipeline()
|
||||
pipe.ingest_reference(TRADE_DATE)
|
||||
pipe.run_dataset("daily", TRADE_DATE)
|
||||
self.assertTrue(db.fetchall("SELECT * FROM provider_call_log"))
|
||||
|
||||
# Force everything to look ancient so cleanup() sweeps it.
|
||||
db.execute("UPDATE provider_call_log SET created_at = '2000-01-01T00:00:00+08:00'")
|
||||
db.execute("UPDATE src_calls SET created_at = '2000-01-01T00:00:00+08:00'")
|
||||
db.execute("UPDATE job_runs SET started_at = '2000-01-01T00:00:00+08:00'")
|
||||
|
||||
pipe.cleanup()
|
||||
self.assertEqual(db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -76,7 +76,6 @@ def make_pipe(transport, quality_extra=None, clock=None):
|
||||
settings = Settings(
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="t" * 32,
|
||||
admin_password="admin-pass",
|
||||
tushare_token="test-token",
|
||||
db_path=db.path,
|
||||
quality=quality,
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from datahub.adapters.base import AdapterError
|
||||
from datahub.db import HubDB
|
||||
from datahub.realtime_serve import fetch_index_quotes, fetch_intraday, fetch_quotes
|
||||
|
||||
|
||||
class _WriteBreaksDB:
|
||||
"""Wraps a real HubDB but breaks only the write path, to prove the
|
||||
real serving path (reads/caches) is untouched by an observability
|
||||
failure while still exercising real fetch/cache code around it."""
|
||||
|
||||
def __init__(self, real: HubDB) -> None:
|
||||
self._real = real
|
||||
|
||||
def fetchall(self, sql, params=()):
|
||||
return self._real.fetchall(sql, params)
|
||||
|
||||
def fetchone(self, sql, params=()):
|
||||
return self._real.fetchone(sql, params)
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
return self._real.execute(sql, params)
|
||||
|
||||
def executemany(self, sql, rows):
|
||||
return self._real.executemany(sql, rows)
|
||||
|
||||
@contextmanager
|
||||
def write(self):
|
||||
raise RuntimeError("db is not writable right now")
|
||||
yield None # pragma: no cover
|
||||
|
||||
|
||||
class RealtimeObservabilityTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db = HubDB(Path(self.tmp.name) / "hub.db")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_eastmoney_success_is_logged_without_changing_payload(self) -> None:
|
||||
rows = [
|
||||
{"ts_code": "000001.SH", "code": "000001", "name": "上证指数", "price": 3000.0,
|
||||
"previous_close": 2990.0, "quote_time_epoch": 0, "source": "eastmoney_push2"},
|
||||
{"ts_code": "399001.SZ", "code": "399001", "name": "深证成指", "price": 9000.0,
|
||||
"previous_close": 8990.0, "quote_time_epoch": 0, "source": "eastmoney_push2"},
|
||||
{"ts_code": "399006.SZ", "code": "399006", "name": "创业板指", "price": 1800.0,
|
||||
"previous_close": 1790.0, "quote_time_epoch": 0, "source": "eastmoney_push2"},
|
||||
]
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as mocked:
|
||||
mocked.return_value.fetch_indices.return_value = rows
|
||||
payload = fetch_index_quotes(self.db)
|
||||
self.assertEqual(payload["data"], rows)
|
||||
self.assertEqual(payload["meta"]["source"], "eastmoney:ulist")
|
||||
log = self.db.fetchall("SELECT * FROM provider_call_log WHERE provider = 'eastmoney'")
|
||||
self.assertEqual(len(log), 1)
|
||||
self.assertEqual(log[0]["interface"], "indices")
|
||||
self.assertEqual(log[0]["status"], "ok")
|
||||
health = self.db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = 'eastmoney' AND interface = 'indices'"
|
||||
)
|
||||
self.assertEqual(health["state"], "ok")
|
||||
|
||||
def test_eastmoney_failure_falls_back_to_tencent_and_logs_both(self) -> None:
|
||||
tencent_rows = [
|
||||
{"ts_code": "000001.SH", "code": "000001", "name": "上证指数", "price": 3000.0,
|
||||
"previous_close": 2990.0, "quote_time_epoch": 0, "source": "tencent_qt"},
|
||||
{"ts_code": "399001.SZ", "code": "399001", "name": "深证成指", "price": 9000.0,
|
||||
"previous_close": 8990.0, "quote_time_epoch": 0, "source": "tencent_qt"},
|
||||
{"ts_code": "399006.SZ", "code": "399006", "name": "创业板指", "price": 1800.0,
|
||||
"previous_close": 1790.0, "quote_time_epoch": 0, "source": "tencent_qt"},
|
||||
]
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as eastmoney, patch(
|
||||
"datahub.realtime_serve.TencentAdapter"
|
||||
) as tencent:
|
||||
eastmoney.return_value.fetch_indices.side_effect = AdapterError("Eastmoney returned 0/3 indices")
|
||||
tencent.return_value.fetch_indices.return_value = tencent_rows
|
||||
payload = fetch_index_quotes(self.db)
|
||||
self.assertEqual(payload["meta"]["source"], "tencent:qt")
|
||||
self.assertEqual(payload["data"], tencent_rows)
|
||||
east_log = self.db.fetchone("SELECT * FROM provider_call_log WHERE provider = 'eastmoney'")
|
||||
self.assertEqual(east_log["status"], "empty")
|
||||
tencent_log = self.db.fetchone("SELECT * FROM provider_call_log WHERE provider = 'tencent'")
|
||||
self.assertEqual(tencent_log["status"], "ok")
|
||||
|
||||
def test_observability_db_failure_never_breaks_a_real_successful_fetch(self) -> None:
|
||||
rows = [
|
||||
{"ts_code": "000001.SH", "price": 3000.0, "previous_close": 2990.0, "quote_time_epoch": 0},
|
||||
{"ts_code": "399001.SZ", "price": 9000.0, "previous_close": 8990.0, "quote_time_epoch": 0},
|
||||
{"ts_code": "399006.SZ", "price": 1800.0, "previous_close": 1790.0, "quote_time_epoch": 0},
|
||||
]
|
||||
broken = _WriteBreaksDB(self.db)
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as mocked:
|
||||
mocked.return_value.fetch_indices.return_value = rows
|
||||
payload = fetch_index_quotes(broken)
|
||||
self.assertEqual(payload["data"], rows)
|
||||
self.assertEqual(payload["meta"]["source"], "eastmoney:ulist")
|
||||
|
||||
def test_observability_db_failure_never_masks_a_real_source_outage(self) -> None:
|
||||
broken = _WriteBreaksDB(self.db)
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as eastmoney, patch(
|
||||
"datahub.realtime_serve.TencentAdapter"
|
||||
) as tencent:
|
||||
eastmoney.return_value.fetch_indices.side_effect = AdapterError("down")
|
||||
tencent.return_value.fetch_indices.side_effect = AdapterError("also down")
|
||||
with self.assertRaises(Exception):
|
||||
fetch_index_quotes(broken)
|
||||
|
||||
def test_named_quotes_records_both_providers_on_partial_merge(self) -> None:
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as eastmoney, patch(
|
||||
"datahub.realtime_serve.TencentAdapter"
|
||||
) as tencent:
|
||||
eastmoney.return_value.fetch_quotes.return_value = [
|
||||
{"ts_code": "000001.SZ", "close": 10, "pre_close": 9, "quote_date": "20260907"},
|
||||
]
|
||||
tencent.return_value.fetch_quotes.return_value = [
|
||||
{"ts_code": "000002.SZ", "close": 20, "pre_close": 19, "quote_date": "20260907"},
|
||||
]
|
||||
payload = fetch_quotes(self.db, ["000001.SZ", "000002.SZ"])
|
||||
self.assertEqual(payload["meta"]["complete"], True)
|
||||
east_log = self.db.fetchone(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'eastmoney' AND interface = 'named_quotes'"
|
||||
)
|
||||
self.assertEqual(east_log["status"], "ok")
|
||||
tencent_log = self.db.fetchone(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'tencent' AND interface = 'named_quotes'"
|
||||
)
|
||||
self.assertEqual(tencent_log["status"], "ok")
|
||||
|
||||
def test_intraday_success_is_logged_as_ok(self) -> None:
|
||||
payload_data = {
|
||||
"entity_type": "stock", "ts_code": "601318.SH", "trade_date": "2026-09-07",
|
||||
"previous_close": 55.8, "points": [{"date": "2026-09-07", "time": "09:30", "close": 55.9}],
|
||||
}
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as mocked:
|
||||
mocked.return_value.fetch_intraday.return_value = payload_data
|
||||
payload = fetch_intraday(self.db, "601318.SH")
|
||||
self.assertEqual(payload["data"], payload_data)
|
||||
log = self.db.fetchone(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'eastmoney' AND interface = 'intraday'"
|
||||
)
|
||||
self.assertEqual(log["status"], "ok")
|
||||
|
||||
def test_intraday_failure_is_logged_as_empty(self) -> None:
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as mocked:
|
||||
mocked.return_value.fetch_intraday.side_effect = AdapterError("No intraday chart data returned")
|
||||
with self.assertRaises(Exception):
|
||||
fetch_intraday(self.db, "000001.SZ")
|
||||
log = self.db.fetchone(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'eastmoney' AND interface = 'intraday'"
|
||||
)
|
||||
self.assertEqual(log["status"], "empty")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,213 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unittest
|
||||
from http import HTTPStatus
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
REVIEW_ROOT = ROOT.parent
|
||||
|
||||
from datahub.serving import ApiError
|
||||
from datahub.siteauth import SiteBridgeError
|
||||
from datahub.siteauth import SiteBridgeError
|
||||
from datahub.siteconsole import SiteConsole
|
||||
|
||||
|
||||
class RecordingBridge:
|
||||
"""Stands in for the review site so these tests exercise only the mapping."""
|
||||
|
||||
def __init__(self, replies: dict[str, dict[str, Any]] | None = None) -> None:
|
||||
self.replies = replies or {}
|
||||
self.calls: list[tuple[str, dict[str, Any]]] = []
|
||||
self.fail_with = ""
|
||||
|
||||
def call(self, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
self.calls.append((path, dict(payload or {})))
|
||||
if self.fail_with:
|
||||
raise SiteBridgeError(self.fail_with)
|
||||
return self.replies.get(path, {})
|
||||
|
||||
def paths(self) -> list[str]:
|
||||
return [path for path, _ in self.calls]
|
||||
|
||||
|
||||
STATUS = {
|
||||
"llm": {
|
||||
"primary_model_id": "gpt-main",
|
||||
"fallback_model_id": "",
|
||||
"models": [
|
||||
{"id": "gpt-main", "name": "GPT 主力", "model": "gpt-4o", "base_url": "https://api.openai.com/v1", "configured": True, "api_key_last4": "7f21"},
|
||||
{"id": "gpt-mini", "name": "GPT 轻量", "model": "gpt-4o-mini", "base_url": "https://api.openai.com/v1", "configured": True, "api_key_last4": "7f21"},
|
||||
{"id": "self-host", "name": "自建 Qwen", "model": "qwen2.5", "base_url": "https://llm.intra.example.com/v1", "configured": False, "api_key_last4": ""},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class ModelPoolTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.bridge = RecordingBridge({"/api/hub-admin/status": STATUS})
|
||||
self.console = SiteConsole(self.bridge)
|
||||
|
||||
def test_models_are_grouped_per_vendor_so_one_vendor_can_hold_many(self) -> None:
|
||||
payload = self.console.models()
|
||||
groups = {group["base_url"]: group for group in payload["groups"]}
|
||||
self.assertEqual(2, len(groups))
|
||||
openai = groups["https://api.openai.com/v1"]
|
||||
self.assertEqual("OpenAI", openai["label"])
|
||||
self.assertEqual(2, len(openai["models"]))
|
||||
self.assertEqual("7f21", openai["key_last4"])
|
||||
self.assertTrue(openai["configured"])
|
||||
|
||||
def test_unknown_vendors_fall_back_to_their_host_as_a_label(self) -> None:
|
||||
groups = {group["base_url"]: group for group in self.console.models()["groups"]}
|
||||
self.assertEqual("llm.intra.example.com", groups["https://llm.intra.example.com/v1"]["label"])
|
||||
self.assertFalse(groups["https://llm.intra.example.com/v1"]["configured"])
|
||||
|
||||
def test_vendor_presets_are_offered_for_the_new_vendor_picker(self) -> None:
|
||||
vendors = self.console.models()["vendors"]
|
||||
self.assertIn("OpenAI", [vendor["label"] for vendor in vendors])
|
||||
self.assertTrue(all(vendor["base_url"].startswith("http") for vendor in vendors))
|
||||
|
||||
def test_saving_only_forwards_the_keys_the_caller_actually_sent(self) -> None:
|
||||
self.console.save_models({"primary_model_id": "gpt-mini"})
|
||||
path, payload = self.bridge.calls[0]
|
||||
self.assertEqual("/api/hub-admin/settings/save", path)
|
||||
self.assertEqual({"primary_model_id": "gpt-mini"}, payload)
|
||||
|
||||
def test_saving_nothing_is_refused_rather_than_wiping_the_pool(self) -> None:
|
||||
with self.assertRaises(ApiError):
|
||||
self.console.save_models({})
|
||||
self.assertEqual([], self.bridge.paths())
|
||||
|
||||
def test_fetching_a_model_list_needs_a_vendor_endpoint(self) -> None:
|
||||
with self.assertRaises(ApiError):
|
||||
self.console.fetch_models({"api_key": "sk-test"})
|
||||
|
||||
def test_fetch_passes_the_typed_key_through_for_first_time_vendors(self) -> None:
|
||||
self.bridge.replies["/api/hub-admin/models/fetch"] = {"models": ["gpt-4o", "gpt-4o-mini"]}
|
||||
result = self.console.fetch_models({"base_url": "https://api.openai.com/v1", "api_key": "sk-new"})
|
||||
self.assertEqual(["gpt-4o", "gpt-4o-mini"], result["models"])
|
||||
self.assertEqual({"base_url": "https://api.openai.com/v1", "api_key": "sk-new"}, self.bridge.calls[0][1])
|
||||
|
||||
def test_a_site_outage_becomes_a_console_error_not_a_traceback(self) -> None:
|
||||
self.bridge.fail_with = "主站不可达"
|
||||
with self.assertRaises(ApiError) as caught:
|
||||
self.console.models()
|
||||
self.assertIn("主站不可达", str(caught.exception))
|
||||
|
||||
|
||||
class MemberAndQuotaTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.bridge = RecordingBridge({"/api/hub-admin/members": {"users": [{"id": 2}], "membership": {"member_daily_limit": 50}}})
|
||||
self.console = SiteConsole(self.bridge)
|
||||
|
||||
def test_saving_a_member_reads_the_roster_back_so_the_page_shows_truth(self) -> None:
|
||||
payload = self.console.save_member({"user_id": 2, "status": "active", "duration": "3_months"})
|
||||
self.assertEqual(["/api/hub-admin/membership/save", "/api/hub-admin/members"], self.bridge.paths())
|
||||
self.assertEqual([{"id": 2}], payload["users"])
|
||||
|
||||
def test_quota_is_a_system_setting_not_a_membership_row(self) -> None:
|
||||
self.console.save_quota({"member_daily_limit": 80})
|
||||
self.assertEqual("/api/hub-admin/settings/save", self.bridge.paths()[0])
|
||||
self.assertEqual({"member_daily_limit": 80}, self.bridge.calls[0][1])
|
||||
|
||||
def test_quota_below_one_is_rejected_before_it_reaches_the_site(self) -> None:
|
||||
for bad in (0, -5, "abc"):
|
||||
with self.subTest(bad=bad):
|
||||
with self.assertRaises(ApiError):
|
||||
self.console.save_quota({"member_daily_limit": bad})
|
||||
self.assertEqual([], self.bridge.paths())
|
||||
|
||||
|
||||
class InviteTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.bridge = RecordingBridge(
|
||||
{
|
||||
"/api/hub-admin/invites": {"summary": {"unused": 1}, "codes": [{"code_id": "abc", "code_masked": "XB-9Q2F-••••"}]},
|
||||
"/api/hub-admin/invites/create": {
|
||||
"created": [{"code_id": "abc", "code": "XB-9Q2F-7K3M-2P8T"}],
|
||||
"summary": {"unused": 1},
|
||||
"codes": [{"code_id": "abc", "code_masked": "XB-9Q2F-••••"}],
|
||||
},
|
||||
}
|
||||
)
|
||||
self.console = SiteConsole(self.bridge)
|
||||
|
||||
def test_plaintext_codes_come_back_only_from_the_create_call(self) -> None:
|
||||
created = self.console.create_invites({"count": 1}, created_by=1)
|
||||
self.assertEqual("XB-9Q2F-7K3M-2P8T", created["created"][0]["code"])
|
||||
# 列表里永远只有掩码,完整码不会再出现第二次
|
||||
listed = self.console.invites()
|
||||
self.assertEqual("XB-9Q2F-••••", listed["codes"][0]["code_masked"])
|
||||
self.assertNotIn("code", listed["codes"][0])
|
||||
|
||||
def test_the_operator_is_recorded_as_the_issuer(self) -> None:
|
||||
self.console.create_invites({"count": 3, "note": "给张总"}, created_by=7)
|
||||
payload = self.bridge.calls[0][1]
|
||||
self.assertEqual(7, payload["created_by"])
|
||||
self.assertEqual(3, payload["count"])
|
||||
self.assertEqual("给张总", payload["note"])
|
||||
|
||||
def test_revoking_uses_the_public_handle_never_the_raw_code(self) -> None:
|
||||
self.console.revoke_invite({"code_id": "abc"})
|
||||
self.assertEqual("/api/hub-admin/invites/revoke", self.bridge.paths()[0])
|
||||
self.assertEqual({"code_id": "abc"}, self.bridge.calls[0][1])
|
||||
|
||||
def test_revoking_without_a_target_is_refused(self) -> None:
|
||||
with self.assertRaises(ApiError):
|
||||
self.console.revoke_invite({})
|
||||
self.assertEqual([], self.bridge.paths())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class BridgeContractTests(unittest.TestCase):
|
||||
"""Both halves of the bridge must agree on the path spelling.
|
||||
|
||||
A typo here fails only at runtime with a confusing 401 (the review site
|
||||
falls through to its browser-session guard), so it is worth a static check.
|
||||
"""
|
||||
|
||||
def test_every_path_the_console_calls_is_registered_on_the_review_site(self) -> None:
|
||||
console_paths = set()
|
||||
for module in ("siteauth.py", "siteconsole.py"):
|
||||
source = (ROOT / "datahub" / module).read_text(encoding="utf-8")
|
||||
console_paths.update(re.findall(r'"(/api/hub-admin/[a-z/-]+)"', source))
|
||||
self.assertTrue(console_paths)
|
||||
registry = (REVIEW_ROOT / "backend" / "http" / "dispatch.py").read_text(encoding="utf-8")
|
||||
registered = set(re.findall(r'"(/api/hub-admin/[a-z/-]+)":', registry))
|
||||
self.assertEqual(
|
||||
set(),
|
||||
console_paths - registered,
|
||||
"控制台在调、主站没注册的桥接路径会静默变成 401",
|
||||
)
|
||||
|
||||
|
||||
class BridgeErrorMappingTests(unittest.TestCase):
|
||||
"""主站拒绝入参(Key 不对)不能在中枢这边冒成 500。"""
|
||||
|
||||
def _console(self, error: SiteBridgeError) -> SiteConsole:
|
||||
class Failing:
|
||||
def call(self, path, payload=None):
|
||||
raise error
|
||||
|
||||
return SiteConsole(Failing())
|
||||
|
||||
def test_upstream_rejection_comes_back_as_a_bad_request(self) -> None:
|
||||
console = self._console(SiteBridgeError("模型列表拉取失败(HTTP 401)", 400))
|
||||
with self.assertRaises(ApiError) as caught:
|
||||
console.fetch_models({"base_url": "https://api.openai.com/v1", "api_key": "sk-bad"})
|
||||
self.assertEqual(caught.exception.code, "INVALID_ARGUMENT")
|
||||
self.assertEqual(caught.exception.status, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
def test_unreachable_site_comes_back_as_service_unavailable(self) -> None:
|
||||
console = self._console(SiteBridgeError("主站不可达:connection refused"))
|
||||
with self.assertRaises(ApiError) as caught:
|
||||
console.models()
|
||||
self.assertEqual(caught.exception.code, "SOURCE_UNAVAILABLE")
|
||||
self.assertEqual(caught.exception.status, HTTPStatus.SERVICE_UNAVAILABLE)
|
||||
@@ -20,7 +20,6 @@ class StewardQueryTests(unittest.TestCase):
|
||||
port=0,
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="k" * 32,
|
||||
admin_password="StartPass1",
|
||||
tushare_token="tushare-secret-token-xyz",
|
||||
db_path=Path(self.tmp.name) / "hub.db",
|
||||
backup_dir=Path(self.tmp.name) / "backups",
|
||||
|
||||
@@ -45,7 +45,6 @@ def make_pipe(transport):
|
||||
settings = Settings(
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="t" * 32,
|
||||
admin_password="admin-pass",
|
||||
tushare_token="test-token",
|
||||
db_path=db.path,
|
||||
quality={"max_publish_attempts": 3, "publication_generations": 3},
|
||||
|
||||
Reference in New Issue
Block a user