rebuild(llm): add audited model connectivity tests

2026-07-30 10:25:07 +08:00
parent d8f0dd930c
commit 08f69b0641
9 changed files with 197 additions and 8 deletions
@@ -290,6 +290,13 @@ class ModelPoolService:
            raise BusinessError("model_not_configured", "智能解读服务尚未配置。")
        return ModelRuntimeConfig(primary=primary, fallback=fallback)

+    def runtime_model(self, model_id: int) -> ModelPoolRecord:
+        with self._database.read() as connection:
+            record = self._repository.get(connection, model_id)
+        if record is None:
+            raise BusinessError("model_not_found", "模型不存在。")
+        return record
+
    def decrypt_api_key(self, record: ModelPoolRecord) -> str:
        return self._cipher.decrypt(record.encrypted_api_key)

@@ -35,9 +35,12 @@ from backend.features.accounts.schemas import (
    ModelInput,
    ModelPoolItemResponse,
    ModelSelectionInput,
+    ModelTestResponse,
    ModelUpdateInput,
    PasswordChangeInput,
 )
+from backend.http.errors import AppError
+from backend.llm.gateway import LLMGatewayError

 router = APIRouter()

@@ -312,3 +315,16 @@ def delete_model(
 ) -> MessageResponse:
    request.app.state.container.model_pool.delete(model_id)
    return MessageResponse(message="模型已删除。")
+
+
+@router.post("/admin/models/{model_id}/test", response_model=ModelTestResponse)
+def test_model(
+    request: Request,
+    model_id: Annotated[int, Path(ge=1)],
+    principal: AdminWritePrincipal,
+) -> dict[str, object]:
+    try:
+        return request.app.state.container.llm.test_model(principal, model_id)
+    except LLMGatewayError as exc:
+        status = 404 if exc.code == "model_not_found" else 503
+        raise AppError(exc.code, str(exc), status) from exc
@@ -119,3 +119,10 @@ class ModelPoolItemResponse(BaseModel):
 class ModelSelectionInput(BaseModel):
    primary_model_id: int = Field(ge=1)
    fallback_model_id: int | None = Field(default=None, ge=1)
+
+
+class ModelTestResponse(BaseModel):
+    connected: bool
+    message: str
+    duration_ms: int = Field(ge=0)
+    request_id: str
@@ -49,6 +49,8 @@ class ModelRuntimeAccess(Protocol):
 class ModelPoolAccess(Protocol):
    def runtime_config(self) -> ModelRuntimeAccess: ...

+    def runtime_model(self, model_id: int) -> ModelRecordAccess: ...
+
    def decrypt_api_key(self, record: ModelRecordAccess) -> str: ...


@@ -237,6 +239,57 @@ class LLMGateway:
        code, message = _safe_error(last_error)
        raise LLMGatewayError(code, message)

+    def test_model(self, principal: PrincipalAccess, model_id: int) -> dict[str, object]:
+        try:
+            record = self._model_pool.runtime_model(model_id)
+        except BusinessError as exc:
+            raise LLMGatewayError("model_not_found", "模型不存在。") from exc
+        now = datetime.now(SHANGHAI)
+        request_id = uuid.uuid4().hex
+        call = LLMCall(
+            request_id=request_id,
+            user_id=principal.user.id,
+            feature="model_connectivity",
+            prompt_version="system:model-connectivity:v1",
+            business_id=f"model:{model_id}",
+            started_at=now,
+            usage_date=now.date().isoformat(),
+            input_chars=15,
+            quota_exempt=True,
+            profiles=(
+                LLMProfile(
+                    model_id=record.id,
+                    role="primary",
+                    base_url=record.base_url,
+                    model_identifier=record.model_identifier,
+                    api_key=self._model_pool.decrypt_api_key(record),
+                ),
+            ),
+        )
+        with self._database.transaction() as connection:
+            self._repository.reserve(
+                connection,
+                request_id=request_id,
+                user_id=principal.user.id,
+                feature=call.feature,
+                business_id=call.business_id,
+                prompt_version=call.prompt_version,
+                started_at=now.isoformat(timespec="seconds"),
+                input_chars=call.input_chars,
+            )
+        started = time.perf_counter()
+        for _event in self.stream(
+            call,
+            [{"role": "user", "content": "连接测试，只回复：连接成功"}],
+        ):
+            pass
+        return {
+            "connected": True,
+            "message": "连接成功",
+            "duration_ms": round((time.perf_counter() - started) * 1000),
+            "request_id": request_id,
+        }
+
    def _finish_attempt(
        self,
        attempt_id: int,
@@ -8,8 +8,8 @@

 ## 减法结果

- 旧运行时代码：55个文件、61,793行；当前重建运行时代码：216个职责文件、24,581行。
- 运行时代码净减少37,212行，约60.2%；迁移/备份工具、测试和文档未混入运行时比较。
+- 旧运行时代码：55个文件、61,793行；当前重建运行时代码：216个职责文件、24,677行。
+- 运行时代码净减少37,116行，约60.1%；迁移/备份工具、测试和文档未混入运行时比较。
 - 唯一浏览器API、数据网关、LLM网关、弹窗Host、设计令牌和移动规则均通过扫描。
 - LLM网关原有3个账户领域具体类型反向导入已改为最小Protocol，未增加第二套服务。
 - 超过章程建议行数的算法、Provider和CSS已逐项登记保留原因及拆分触发条件，见`../../final/subtraction-audit.md`。
@@ -28,7 +28,7 @@
 - 生产响应统一设置CSP、`nosniff`、拒绝Frame、Permissions Policy、Referrer Policy；HTTPS增加HSTS。
 - CSP保留Vue动态宽度样式所需的`style-src 'unsafe-inline'`，脚本仍只允许同源。
 - 静态哈希资源长期缓存，SPA入口不缓存；未知API不被SPA接管。
- 真实迁移库副本市场摘要中位数20.90ms、P95 23.55ms；前端生产JS 296.29KB、CSS 92.08KB（未压缩）。
+- 真实迁移库副本市场摘要中位数20.90ms、P95 23.55ms；前端生产JS 296.90KB、CSS 92.08KB（未压缩）。

 ## 最终门禁

@@ -37,7 +37,7 @@
 - Vue TypeScript：通过。
 - Vitest：3个文件、7项通过。
 - Vite生产构建：通过。
- Playwright：20项通过，单worker，最后一轮耗时约1.7分钟。
+- Playwright：21项通过，单worker，最后一轮耗时约1.6分钟。
 - `git diff --check`和已知敏感值扫描：通过。
 - Docker：本机未安装，未虚报实构建；列为NAS切换前阻断项。

@@ -25,8 +25,7 @@

 ### P0：用户可见功能缺失

-1. **模型连通性测试**：模型池支持增删改和主辅选择，但每个模型的独立测试入口与服务端测试调用尚未实现。
-2. **自定义选股完整能力**：缺少自然语言转换受控公式和滚动回测；精选策略详情尚未完整显示评分权重、
+1. **自定义选股完整能力**：缺少自然语言转换受控公式和滚动回测；精选策略详情尚未完整显示评分权重、
   执行频率和风险等级。

 ### P1：验收和交互覆盖不足
@@ -58,7 +57,9 @@
 - 涨停、炸板、跌停原因可由iFinD盘后补充，管理员修订具有更高优先级，全部修订历史永久保留。
 - 阶段选股四步和候选区已直接表达未执行、执行中、失败、已完成；策略库不再把未执行或失败误写为暂无信号。
 - 状态栏读取真实后台任务与最后成功行情状态，不再显示固定占位文字。
- 本轮验证为Ruff、104项pytest、Vue类型检查、7项Vitest、生产构建和20项Playwright全部通过。
+- 模型池中的每个模型可由管理员独立测试；测试复用统一LLM网关并记录模型、耗时、成功或失败，
+  不返回密钥和上游正文，也不计入会员每日额度。
+- 本轮验证为Ruff、104项pytest、Vue类型检查、7项Vitest、生产构建和21项Playwright全部通过。

 ## 外部环境阻断项

@@ -22,12 +22,15 @@ const primaryId = ref<number | null>(null);
 const fallbackId = ref<number | null>(null);
 const errorMessage = ref("");
 const deleteConfirmation = ref<number | null>(null);
+const testingId = ref<number | null>(null);
+const testMessage = ref("");
 const form = reactive({ display_name: "", base_url: "", model_identifier: "", api_key: "" });
 const editing = computed(() => models.value.find((item) => item.id === editingId.value) ?? null);

 function resetForm(): void {
  editingId.value = null;
  Object.assign(form, { display_name: "", base_url: "", model_identifier: "", api_key: "" });
+  testMessage.value = "";
 }

 function edit(model: ModelItem): void {
@@ -111,6 +114,23 @@ async function remove(model: ModelItem): Promise<void> {
  }
 }

+async function testConnection(model: ModelItem): Promise<void> {
+  testingId.value = model.id;
+  testMessage.value = "";
+  errorMessage.value = "";
+  try {
+    const result = await api.post<{ message: string; duration_ms: number }>(
+      `/admin/models/${model.id}/test`,
+    );
+    testMessage.value = `${result.message} · ${result.duration_ms} ms`;
+    ui.showToast(`${model.display_name} 连接成功`);
+  } catch (error) {
+    errorMessage.value = error instanceof Error ? error.message : "模型连接测试失败。";
+  } finally {
+    testingId.value = null;
+  }
+}
+
 onMounted(load);
 </script>

@@ -134,8 +154,9 @@ onMounted(load);
          <div class="field"><label for="model-url">服务地址</label><input id="model-url" v-model="form.base_url" class="input" type="url" required /></div>
          <div class="field"><label for="model-identifier">模型标识</label><input id="model-identifier" v-model="form.model_identifier" class="input" required /></div>
          <div class="field"><label for="model-key">密钥</label><input id="model-key" v-model="form.api_key" class="input" type="password" autocomplete="off" :placeholder="editing ? '留空则保持原密钥' : '请输入密钥'" /></div>
+          <p v-if="testMessage" class="up" role="status">{{ testMessage }}</p>
          <p v-if="errorMessage" class="field-error" role="alert">{{ errorMessage }}</p>
-          <div class="form-actions"><button v-if="editing && !editing.is_primary && !editing.is_fallback" class="btn" type="button" @click="remove(editing)">{{ deleteConfirmation === editing.id ? "确认删除" : "删除" }}</button><button class="btn btn-primary" type="submit">{{ editing ? "保存修改" : "添加模型" }}</button></div>
+          <div class="form-actions"><button v-if="editing" class="btn" type="button" :disabled="testingId === editing.id" @click="testConnection(editing)">{{ testingId === editing.id ? "测试中" : "测试连接" }}</button><button v-if="editing && !editing.is_primary && !editing.is_fallback" class="btn" type="button" @click="remove(editing)">{{ deleteConfirmation === editing.id ? "确认删除" : "删除" }}</button><button class="btn btn-primary" type="submit">{{ editing ? "保存修改" : "添加模型" }}</button></div>
        </form>
      </section>
      <section class="card">
@@ -102,3 +102,36 @@ test("administrator can audit jobs, backfill and govern market events", async ({
    { kind: "supplement" },
  ]);
 });
+
+test("administrator tests a selected model without exposing its key", async ({ page }) => {
+  await page.route("**/api/admin/models", (route) => route.fulfill({
+    contentType: "application/json",
+    body: JSON.stringify([{
+      id: 1,
+      display_name: "主模型",
+      base_url: "https://model.example.com/v1",
+      model_identifier: "reasoning-model",
+      has_api_key: true,
+      is_primary: true,
+      is_fallback: false,
+      updated_at: "2026-07-30T10:00:00+08:00",
+    }]),
+  }));
+  await page.route("**/api/admin/models/1/test", (route) => route.fulfill({
+    contentType: "application/json",
+    body: JSON.stringify({
+      connected: true,
+      message: "连接成功",
+      duration_ms: 286,
+      request_id: "audit-request",
+    }),
+  }));
+
+  await authenticate(page);
+  await page.getByRole("button", { name: "系统管理" }).click();
+  await page.getByRole("button", { name: "模型池" }).click();
+  await page.getByRole("button", { name: "主模型 reasoning-model 主模型" }).click();
+  await page.getByRole("button", { name: "测试连接" }).click();
+  await expect(page.getByText("连接成功 · 286 ms", { exact: true })).toBeVisible();
+  await expect(page.locator("body")).not.toContainText(/secret|api[_ -]?key/i);
+});
@@ -6,6 +6,7 @@ import httpx

 from backend.bootstrap.application import create_application
 from backend.bootstrap.settings import Settings
+from backend.llm.provider import ProviderFailure
 from tests.support import run_scenario
 from tests.test_accounts import (
    ADMIN_PASSWORD,
@@ -25,6 +26,17 @@ def model_payload(index: int, api_key: str | None = None) -> dict[str, str]:
    }


+class ScriptedProvider:
+    def __init__(self, scripts: list[object]) -> None:
+        self.scripts = scripts
+
+    def stream(self, _profile, _messages):
+        script = self.scripts.pop(0)
+        if isinstance(script, Exception):
+            raise script
+        yield from script
+
+
 def test_model_pool_is_admin_only_and_never_exposes_keys(tmp_path) -> None:
    application = create_application(Settings.for_test(tmp_path))

@@ -62,6 +74,45 @@ def test_model_pool_is_admin_only_and_never_exposes_keys(tmp_path) -> None:
        assert encrypted_before != "private-alpha-key"
        assert "private-alpha-key" not in encrypted_before

+        application.state.container.llm._provider = ScriptedProvider(
+            [["连接", "成功"], ProviderFailure("authentication")]
+        )
+        tested = await client.post(
+            "/api/admin/models/1/test", headers=csrf_headers(admin_session)
+        )
+        assert tested.status_code == 200
+        assert tested.json()["connected"] is True
+        assert tested.json()["message"] == "连接成功"
+        assert "private-alpha-key" not in tested.text
+        failed_test = await client.post(
+            "/api/admin/models/1/test", headers=csrf_headers(admin_session)
+        )
+        assert failed_test.status_code == 503
+        assert failed_test.json()["error"]["code"] == "model_authentication"
+        with sqlite3.connect(application.state.settings.database_path) as connection:
+            requests = connection.execute(
+                """
+                SELECT feature, business_id, status, error_type
+                FROM llm_requests ORDER BY started_at, rowid
+                """
+            ).fetchall()
+            attempts = connection.execute(
+                """
+                SELECT model_id, role, status, error_type
+                FROM llm_attempts ORDER BY id
+                """
+            ).fetchall()
+            usage = connection.execute("SELECT COUNT(*) FROM llm_usage_daily").fetchone()[0]
+        assert requests == [
+            ("model_connectivity", "model:1", "success", ""),
+            ("model_connectivity", "model:1", "failed", "authentication"),
+        ]
+        assert attempts == [
+            (1, "primary", "success", ""),
+            (1, "primary", "failed", "authentication"),
+        ]
+        assert usage == 0
+
        updated_payload = model_payload(1)
        updated_payload.pop("api_key")
        updated_payload["display_name"] = "主模型"