Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
abd4d22a67 | ||
|
|
f014eb11bd |
@@ -0,0 +1,299 @@
|
||||
const state = { csrf: "", page: "overview" };
|
||||
|
||||
function $(id) { return document.getElementById(id); }
|
||||
|
||||
async function api(path, options = {}) {
|
||||
const headers = Object.assign({ "Content-Type": "application/json" }, options.headers || {});
|
||||
if (state.csrf && (options.method || "GET") !== "GET") headers["X-CSRF-Token"] = state.csrf;
|
||||
const res = await fetch(path, Object.assign({}, options, { headers, credentials: "same-origin" }));
|
||||
const body = await res.json();
|
||||
if (!res.ok) {
|
||||
const msg = (body.error && body.error.message) || body.error || res.statusText;
|
||||
throw new Error(msg);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
function show(id) {
|
||||
["login-view", "change-view", "shell"].forEach((key) => { $(key).hidden = key !== id; });
|
||||
}
|
||||
|
||||
function esc(value) {
|
||||
return String(value ?? "").replace(/[&<>"]/g, (ch) => ({ "&": "&", "<": "<", ">": ">", '"': """ }[ch]));
|
||||
}
|
||||
|
||||
function table(headers, rows) {
|
||||
const thead = headers.map((h) => `<th>${esc(h)}</th>`).join("");
|
||||
const body = rows.length
|
||||
? rows.map((cols) => `<tr>${cols.map((c) => `<td>${c}</td>`).join("")}</tr>`).join("")
|
||||
: `<tr><td colspan="${headers.length}">暂无数据</td></tr>`;
|
||||
return `<table><thead><tr>${thead}</tr></thead><tbody>${body}</tbody></table>`;
|
||||
}
|
||||
|
||||
async function boot() {
|
||||
try {
|
||||
const session = await api("/admin/api/session");
|
||||
state.csrf = session.csrf;
|
||||
$("who").textContent = session.username;
|
||||
if (session.must_change) { show("change-view"); return; }
|
||||
show("shell");
|
||||
await render();
|
||||
} catch {
|
||||
show("login-view");
|
||||
}
|
||||
}
|
||||
|
||||
$("login-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const form = new FormData(event.target);
|
||||
$("login-error").hidden = true;
|
||||
try {
|
||||
const result = await api("/admin/api/login", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ username: form.get("username"), password: form.get("password") }),
|
||||
});
|
||||
state.csrf = result.csrf;
|
||||
if (result.must_change) show("change-view");
|
||||
else { show("shell"); await render(); }
|
||||
} catch (err) {
|
||||
$("login-error").hidden = false;
|
||||
$("login-error").textContent = err.message;
|
||||
}
|
||||
});
|
||||
|
||||
$("change-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const form = new FormData(event.target);
|
||||
try {
|
||||
await api("/admin/api/change-password", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ current: form.get("current"), new_password: form.get("new_password") }),
|
||||
});
|
||||
show("shell");
|
||||
await render();
|
||||
} catch (err) {
|
||||
$("change-error").hidden = false;
|
||||
$("change-error").textContent = err.message;
|
||||
}
|
||||
});
|
||||
|
||||
$("logout-btn").addEventListener("click", async () => {
|
||||
await api("/admin/api/logout", { method: "POST", body: "{}" });
|
||||
show("login-view");
|
||||
});
|
||||
|
||||
$("theme-btn").addEventListener("click", () => {
|
||||
const root = document.documentElement;
|
||||
const next = root.getAttribute("data-theme") === "night" ? "" : "night";
|
||||
if (next) root.setAttribute("data-theme", next);
|
||||
else root.removeAttribute("data-theme");
|
||||
$("theme-btn").textContent = next ? "日间" : "夜间";
|
||||
});
|
||||
|
||||
document.querySelectorAll("nav button").forEach((btn) => {
|
||||
btn.addEventListener("click", () => {
|
||||
document.querySelectorAll("nav button").forEach((item) => item.classList.remove("active"));
|
||||
btn.classList.add("active");
|
||||
state.page = btn.dataset.page;
|
||||
render();
|
||||
});
|
||||
});
|
||||
|
||||
async function render() {
|
||||
const page = $("page");
|
||||
if (state.page === "overview") {
|
||||
const data = await api("/admin/api/overview");
|
||||
$("phase").textContent = data.session_phase;
|
||||
const eod = data.eod_status || {};
|
||||
const rev = data.revision_status || {};
|
||||
const eodLabels = {
|
||||
pending_first_attempt: "等待首次尝试",
|
||||
waiting_upstream: "等待上游",
|
||||
done: "已成功",
|
||||
cutoff_failed: "已截止失败",
|
||||
closed_day: "休市",
|
||||
};
|
||||
const revLabels = {
|
||||
waiting_review: "等待复核",
|
||||
review_failed: "复核失败",
|
||||
aligned: "已追平",
|
||||
cutoff: "已截止",
|
||||
pending_publish: "待发布",
|
||||
closed_day: "休市",
|
||||
};
|
||||
const eodExtra = [];
|
||||
if (eod.state === "waiting_upstream") {
|
||||
eodExtra.push(`已试 ${eod.attempts} 次`);
|
||||
if (eod.next_retry_at) eodExtra.push(`下次重试 ${esc(String(eod.next_retry_at).replace("T", " ").slice(11, 16))}`);
|
||||
if (eod.missing_datasets && eod.missing_datasets.length) eodExtra.push(`缺 ${esc(eod.missing_datasets.join(","))}`);
|
||||
}
|
||||
if (eod.state === "cutoff_failed" && eod.missing_datasets) {
|
||||
eodExtra.push(`缺 ${esc(eod.missing_datasets.join(","))}`);
|
||||
}
|
||||
const revExtra = [];
|
||||
if (rev.detail) revExtra.push(esc(String(rev.detail)));
|
||||
if (rev.window) revExtra.push(esc(String(rev.window)));
|
||||
page.innerHTML = `
|
||||
<div class="cards">
|
||||
<div class="card"><div class="muted">交易日</div><strong>${esc(data.trade_date)}</strong></div>
|
||||
<div class="card"><div class="muted">阶段</div><strong>${esc(data.session_phase)}</strong></div>
|
||||
<div class="card"><div class="muted">今日发布</div><strong>${data.publications.length}</strong></div>
|
||||
<div class="card"><div class="muted">盘后补跑</div><strong>${esc(eodLabels[eod.state] || eod.state || "-")}</strong><div class="muted">${eodExtra.join(" · ")}</div></div>
|
||||
<div class="card"><div class="muted">估值复核</div><strong>${esc(revLabels[rev.state] || rev.state || "-")}</strong><div class="muted">${revExtra.join(" · ")}</div></div>
|
||||
<div class="card"><div class="muted">异常批次</div><strong class="${data.anomalies.length ? "fail" : "ok"}">${data.anomalies.length}</strong></div>
|
||||
</div>
|
||||
<h2>最近调用</h2>
|
||||
${table(["时间", "源", "端点", "结果", "耗时"], data.recent_calls.map((row) => [
|
||||
esc(row.created_at), esc(row.provider), esc(row.endpoint),
|
||||
row.ok ? '<span class="ok">成功</span>' : `<span class="fail">${esc(row.error)}</span>`,
|
||||
`${row.latency_ms ?? "-"} ms`,
|
||||
]))}
|
||||
`;
|
||||
return;
|
||||
}
|
||||
if (state.page === "sources") {
|
||||
const data = await api("/admin/api/sources");
|
||||
page.innerHTML = `<h2>数据源</h2>` + table(
|
||||
["源", "角色", "状态", "凭据", "操作"],
|
||||
data.items.map((item) => {
|
||||
const cred = item.credential || {};
|
||||
const credText = cred.configured ? `已配置 · ${esc(cred.last4 || "****")}` : "未配置";
|
||||
return [
|
||||
esc(item.provider),
|
||||
esc(item.role),
|
||||
esc((item.health && (item.health.state || item.health.status)) || "-"),
|
||||
credText,
|
||||
`<button data-probe="${esc(item.provider)}">探测一次</button>`,
|
||||
];
|
||||
}),
|
||||
);
|
||||
page.querySelectorAll("[data-probe]").forEach((btn) => {
|
||||
btn.addEventListener("click", async () => {
|
||||
const result = await api(`/admin/api/sources/${btn.dataset.probe}/probe`, { method: "POST", body: "{}" });
|
||||
alert(JSON.stringify(result));
|
||||
render();
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (state.page === "jobs") {
|
||||
const data = await api("/admin/api/jobs");
|
||||
page.innerHTML = `
|
||||
<h2>调度任务</h2>
|
||||
${table(["任务", "时刻", "操作"], data.jobs.map((job) => [
|
||||
`${esc(job.id)} · ${esc(job.title)}`, esc(job.at),
|
||||
`<button data-run="${esc(job.id)}">手动触发</button>`,
|
||||
]))}
|
||||
<h3>最近运行</h3>
|
||||
${table(["ID", "任务", "状态", "开始", "结束", "错误"], data.runs.map((row) => [
|
||||
row.id, esc(row.job_id), esc(row.state), esc(row.started_at), esc(row.finished_at), esc(row.error),
|
||||
]))}
|
||||
`;
|
||||
page.querySelectorAll("[data-run]").forEach((btn) => {
|
||||
btn.addEventListener("click", async () => {
|
||||
const date = prompt("交易日 YYYYMMDD（可留空=今天）", "") || "";
|
||||
await api(`/admin/api/jobs/${btn.dataset.run}/run`, { method: "POST", body: JSON.stringify({ trade_date: date }) });
|
||||
render();
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (state.page === "release") {
|
||||
const date = new Date().toISOString().slice(0, 10).replace(/-/g, "");
|
||||
const data = await api(`/admin/api/batches?date=${date}`);
|
||||
page.innerHTML = `
|
||||
<h2>盘后发布 ${esc(data.trade_date)}</h2>
|
||||
<div class="toolbar">
|
||||
<label>日期 <input id="rel-date" value="${esc(data.trade_date)}" /></label>
|
||||
<button type="button" id="rel-load">查看</button>
|
||||
<button type="button" id="rel-backfill">补数</button>
|
||||
</div>
|
||||
<h3>当前映射</h3>
|
||||
${table(["数据集", "活跃批次", "上一批次", "状态", "发布时间", "操作"], data.publications.map((row) => [
|
||||
esc(row.dataset), esc(row.active_batch), esc(row.prev_batch), esc(row.state), esc(row.published_at),
|
||||
row.prev_batch ? `<button class="danger" data-rollback="${esc(row.dataset)}">回滚</button>` : "-",
|
||||
]))}
|
||||
<h3>批次</h3>
|
||||
${table(["batch_id", "数据集", "状态", "行数", "错误"], data.batches.map((row) => [
|
||||
esc(row.batch_id), esc(row.dataset), esc(row.state), row.rows_out ?? "", esc(row.error),
|
||||
]))}
|
||||
`;
|
||||
$bindRelease(page);
|
||||
return;
|
||||
}
|
||||
if (state.page === "datasets") {
|
||||
const data = await api("/admin/api/datasets?date=");
|
||||
page.innerHTML = `
|
||||
<h2>数据集 / 质量 ${esc(data.trade_date)}</h2>
|
||||
${table(["数据集", "批次", "状态", "发布时间"], data.publications.map((row) => [
|
||||
esc(row.dataset), esc(row.active_batch), esc(row.state), esc(row.published_at),
|
||||
]))}
|
||||
<h3>源间差异</h3>
|
||||
${table(["指标", "左", "右", "偏差", "样本"], data.diff_reports.map((row) => [
|
||||
esc(row.metric), esc(row.left_value), esc(row.right_value), esc(row.deviation), row.sample_count ?? "",
|
||||
]))}
|
||||
`;
|
||||
return;
|
||||
}
|
||||
if (state.page === "audit") {
|
||||
const data = await api("/admin/api/audit");
|
||||
page.innerHTML = `<h2>审计</h2>` + table(
|
||||
["时间", "操作者", "动作", "对象", "详情"],
|
||||
data.items.map((row) => [esc(row.created_at), esc(row.actor), esc(row.action), esc(row.target), esc(row.detail)]),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function $bindRelease(page) {
|
||||
page.querySelector("#rel-load").addEventListener("click", async () => {
|
||||
const date = page.querySelector("#rel-date").value;
|
||||
const data = await api(`/admin/api/batches?date=${encodeURIComponent(date)}`);
|
||||
state.page = "release";
|
||||
// re-render with fetched date by writing location hash
|
||||
history.replaceState(null, "", `#release-${date}`);
|
||||
$("page").innerHTML = renderRelease(data);
|
||||
$bindRelease($("page"));
|
||||
});
|
||||
page.querySelector("#rel-backfill").addEventListener("click", () => dangerous("backfill"));
|
||||
page.querySelectorAll("[data-rollback]").forEach((btn) => {
|
||||
btn.addEventListener("click", () => dangerous("rollback", btn.dataset.rollback));
|
||||
});
|
||||
}
|
||||
|
||||
function renderRelease(data) {
|
||||
return `
|
||||
<h2>盘后发布 ${esc(data.trade_date)}</h2>
|
||||
<div class="toolbar">
|
||||
<label>日期 <input id="rel-date" value="${esc(data.trade_date)}" /></label>
|
||||
<button type="button" id="rel-load">查看</button>
|
||||
<button type="button" id="rel-backfill">补数</button>
|
||||
</div>
|
||||
<h3>当前映射</h3>
|
||||
${table(["数据集", "活跃批次", "上一批次", "状态", "发布时间", "操作"], data.publications.map((row) => [
|
||||
esc(row.dataset), esc(row.active_batch), esc(row.prev_batch), esc(row.state), esc(row.published_at),
|
||||
row.prev_batch ? `<button class="danger" data-rollback="${esc(row.dataset)}">回滚</button>` : "-",
|
||||
]))}
|
||||
<h3>批次</h3>
|
||||
${table(["batch_id", "数据集", "状态", "行数", "错误"], data.batches.map((row) => [
|
||||
esc(row.batch_id), esc(row.dataset), esc(row.state), row.rows_out ?? "", esc(row.error),
|
||||
]))}
|
||||
`;
|
||||
}
|
||||
|
||||
async function dangerous(kind, dataset) {
|
||||
const date = ($("rel-date") && $("rel-date").value) || "";
|
||||
const ds = dataset || prompt("数据集（daily/valuation/moneyflow/auction/stocks→A组整批；index_daily→B组；或 reference）", "daily");
|
||||
if (!ds) return;
|
||||
const password = prompt("二次确认：输入管理密码");
|
||||
if (!password) return;
|
||||
const confirmWord = `${ds}:${date}`;
|
||||
const typed = prompt(`请输入确认词：${confirmWord}`);
|
||||
const path = kind === "rollback" ? "/admin/api/rollback" : "/admin/api/backfill";
|
||||
await api(path, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ dataset: ds, trade_date: date, password, confirm: typed }),
|
||||
});
|
||||
render();
|
||||
}
|
||||
|
||||
boot();
|
||||
@@ -1,371 +0,0 @@
|
||||
"use strict";
|
||||
/* ==========================================================================
|
||||
xiaobai-datahub 管理后台 · 第九版核心引擎（A/B/C 三方向唯一共用来源）
|
||||
—— 会话/主题/轮询/真实事件识别/危险操作/路由，三个布局文件只消费这里的
|
||||
state、Bus 事件与 API，禁止各自再写一套请求或业务判断。
|
||||
========================================================================== */
|
||||
window.Core = (function () {
|
||||
function $(id) { return document.getElementById(id); }
|
||||
function esc(value) {
|
||||
return String(value ?? "").replace(/[&<>"]/g, (ch) => ({ "&": "&", "<": "<", ">": ">", '"': """ }[ch]));
|
||||
}
|
||||
function timeShort(value) {
|
||||
const s = String(value ?? "");
|
||||
const m = s.match(/(\d{2}:\d{2}:\d{2})/);
|
||||
return m ? m[1] : s.replace("T", " ").slice(0, 16);
|
||||
}
|
||||
function clamp(v, a, b) { return Math.max(a, Math.min(b, v)); }
|
||||
function table(headers, rows) {
|
||||
const thead = headers.map((h) => `<th>${esc(h)}</th>`).join("");
|
||||
const body = rows.length
|
||||
? rows.map((cols) => `<tr>${cols.map((c) => `<td>${c}</td>`).join("")}</tr>`).join("")
|
||||
: `<tr><td colspan="${headers.length}">暂无数据</td></tr>`;
|
||||
return `<table class="grid"><thead><tr>${thead}</tr></thead><tbody>${body}</tbody></table>`;
|
||||
}
|
||||
|
||||
const state = {
|
||||
csrf: "",
|
||||
layout: "flowline",
|
||||
releaseDate: "",
|
||||
reduced: matchMedia("(prefers-reduced-motion: reduce)").matches,
|
||||
visible: document.visibilityState === "visible",
|
||||
online: navigator.onLine,
|
||||
data: { overview: null, sources: null, jobs: null, batches: null, datasets: null, audit: null },
|
||||
};
|
||||
|
||||
async function api(path, options = {}) {
|
||||
const headers = Object.assign({ "Content-Type": "application/json" }, options.headers || {});
|
||||
if (state.csrf && (options.method || "GET") !== "GET") headers["X-CSRF-Token"] = state.csrf;
|
||||
const res = await fetch(path, Object.assign({}, options, { headers, credentials: "same-origin" }));
|
||||
const body = await res.json();
|
||||
if (!res.ok) {
|
||||
const msg = (body.error && body.error.message) || body.error || res.statusText;
|
||||
throw new Error(msg);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------- 事件总线
|
||||
'data' → 任一真实拉取完成，payload = key（overview/sources/...），布局重绘用
|
||||
'event' → 已经发生的真实事件（供三方向各自演绎接力/推进/贯穿动效）
|
||||
payload = { channel, kind, n, meta }
|
||||
channel: tushare|eastmoney|tencent|ifind|junction|tx|audit
|
||||
kind: ok|error|unconfigured|rollback|pub
|
||||
'theme' → 主题切换，payload = 'day'|'night'
|
||||
'reduced' → 减弱动效状态变化，payload = boolean
|
||||
*/
|
||||
const Bus = (() => {
|
||||
const listeners = {};
|
||||
return {
|
||||
on(type, fn) { (listeners[type] = listeners[type] || []).push(fn); return () => this.off(type, fn); },
|
||||
off(type, fn) { if (listeners[type]) listeners[type] = listeners[type].filter((f) => f !== fn); },
|
||||
emit(type, payload) { (listeners[type] || []).slice().forEach((fn) => { try { fn(payload); } catch (e) { console.error(e); } }); },
|
||||
};
|
||||
})();
|
||||
|
||||
/* ---------------------------------------------------------------- 常量与标签 */
|
||||
const FLOW_PROVIDERS = ["tushare", "eastmoney", "tencent", "ifind"];
|
||||
const FLOW_LABEL = {
|
||||
tushare: "Tushare", eastmoney: "东方财富", tencent: "腾讯行情", ifind: "iFinD",
|
||||
};
|
||||
const FLOW_ROLE = {
|
||||
tushare: "官方盘后", eastmoney: "盘中观察", tencent: "盘中观察", ifind: "授权实时",
|
||||
};
|
||||
const EOD_LABELS = {
|
||||
pending_first_attempt: "等待首次尝试", waiting_upstream: "等待上游", done: "已成功",
|
||||
cutoff_failed: "已截止失败", closed_day: "休市",
|
||||
};
|
||||
const REV_LABELS = {
|
||||
waiting_review: "等待复核", review_failed: "复核失败", aligned: "已追平",
|
||||
cutoff: "已截止", pending_publish: "待发布", closed_day: "休市",
|
||||
};
|
||||
const PHASE_LABELS = { pre: "盘前", intraday: "盘中", lunch: "午间", eod: "盘后", closed: "休市" };
|
||||
function todayYmd() {
|
||||
const d = new Date(); const pad = (n) => String(n).padStart(2, "0");
|
||||
return `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}`;
|
||||
}
|
||||
if (!state.releaseDate) state.releaseDate = todayYmd();
|
||||
|
||||
function chipClass(s) {
|
||||
const map = {
|
||||
ok: "ok", error: "err", warn: "warn", unknown: "unknown", unconfigured: "unconfigured",
|
||||
published: "ok", pending: "warn", missing: "err", building: "warn", staged: "warn", failed: "err",
|
||||
running: "warn", queued: "info", idle: "", success: "ok",
|
||||
};
|
||||
return map[s] ?? "";
|
||||
}
|
||||
function chipHtml(s, textOverride) {
|
||||
return `<span class="chip ${chipClass(s)}">${esc(textOverride ?? s)}</span>`;
|
||||
}
|
||||
|
||||
/* 真实健康态归一化：tushare 走断路器术语（closed=健康/half_open=试探/open=已跳闸），
|
||||
其余三路走 ok/empty/error/unconfigured；没有探测记录时是 unknown。
|
||||
四态必须原样区分显示，不得把 unknown 和 error 混为一谈，也不得把断路器
|
||||
"closed"(健康) 误读成红色错误。 */
|
||||
function linkState(health) {
|
||||
const s = (health && health.state) || "unknown";
|
||||
if (s === "ok" || s === "empty" || s === "closed") return "ok";
|
||||
if (s === "half_open") return "warn";
|
||||
if (s === "unconfigured" || s === "reserved") return "unconfigured";
|
||||
if (s === "unknown") return "unknown";
|
||||
return "error"; // open（断路器已跳闸）、error
|
||||
}
|
||||
function linkLampClass(link) {
|
||||
if (link === "ok") return "on";
|
||||
if (link === "error") return "err on";
|
||||
return "off"; // warn / unknown / unconfigured 灯位统一暗灭，状态文字由 chip 承载
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------- 真实事件识别
|
||||
只认已经发生的事实：recent_calls / probe 健康探测 / job runs / batch 状态变化 /
|
||||
publication 发布时间变化 / audit 新行。没有真实变化就不发事件，不臆造。 */
|
||||
const Seen = {
|
||||
callsMax: -1, jobRunsMax: -1, auditMax: -1,
|
||||
batchState: new Map(), pubPublishedAt: new Map(),
|
||||
firstOverview: true, firstJobs: true, firstAudit: true, firstBatches: true,
|
||||
};
|
||||
|
||||
async function pollOverview() {
|
||||
const data = await api("/admin/api/overview");
|
||||
const wasFirst = Seen.firstOverview;
|
||||
const calls = data.recent_calls || [];
|
||||
let maxId = Seen.callsMax;
|
||||
const fresh = [];
|
||||
for (const c of calls) { if (c.id > Seen.callsMax) fresh.push(c); if (c.id > maxId) maxId = c.id; }
|
||||
Seen.callsMax = maxId;
|
||||
Seen.firstOverview = false;
|
||||
if (!wasFirst && fresh.length) Bus.emit("event", { channel: "tushare", kind: fresh.some((c) => !c.ok) ? "error" : "ok", n: fresh.length, meta: { fresh } });
|
||||
state.data.overview = data;
|
||||
Bus.emit("data", "overview");
|
||||
}
|
||||
|
||||
async function pollSources() {
|
||||
const data = await api("/admin/api/sources");
|
||||
for (const item of data.items) {
|
||||
if (!FLOW_PROVIDERS.includes(item.provider)) continue;
|
||||
const health = item.health || {};
|
||||
const link = linkState(health);
|
||||
item._link = link;
|
||||
if (item.provider !== "tushare" && link !== "unconfigured" && link !== "unknown") {
|
||||
Bus.emit("event", { channel: item.provider, kind: link === "ok" ? "ok" : link === "warn" ? "warn" : "error", n: 1, meta: { item } });
|
||||
}
|
||||
}
|
||||
state.data.sources = data;
|
||||
Bus.emit("data", "sources");
|
||||
}
|
||||
|
||||
async function pollJobs() {
|
||||
const data = await api("/admin/api/jobs");
|
||||
const wasFirst = Seen.firstJobs;
|
||||
let maxId = Seen.jobRunsMax;
|
||||
const fresh = [];
|
||||
for (const r of data.runs || []) { if (r.id > Seen.jobRunsMax) fresh.push(r); if (r.id > maxId) maxId = r.id; }
|
||||
Seen.jobRunsMax = maxId;
|
||||
Seen.firstJobs = false;
|
||||
if (!wasFirst && fresh.length) {
|
||||
const anyFail = fresh.some((r) => r.state === "failed");
|
||||
Bus.emit("event", { channel: "junction", kind: anyFail ? "error" : "ok", n: fresh.length, meta: { fresh } });
|
||||
}
|
||||
state.data.jobs = data;
|
||||
state.freshJobRuns = fresh;
|
||||
Bus.emit("data", "jobs");
|
||||
}
|
||||
|
||||
async function pollBatches() {
|
||||
const date = state.releaseDate || "";
|
||||
const data = await api(`/admin/api/batches?date=${encodeURIComponent(date)}`);
|
||||
const wasFirst = Seen.firstBatches;
|
||||
let changed = false;
|
||||
for (const b of data.batches || []) {
|
||||
const prev = Seen.batchState.get(b.batch_id);
|
||||
if (prev !== b.state) { changed = true; Seen.batchState.set(b.batch_id, b.state); }
|
||||
}
|
||||
let published = false;
|
||||
for (const p of data.publications || []) {
|
||||
const key = `${p.dataset}:${p.trade_date}`;
|
||||
const prev = Seen.pubPublishedAt.get(key);
|
||||
if (prev !== p.published_at) { published = true; Seen.pubPublishedAt.set(key, p.published_at); }
|
||||
}
|
||||
Seen.firstBatches = false;
|
||||
if (!wasFirst && published) Bus.emit("event", { channel: "tx", kind: "pub", n: 1, meta: {} });
|
||||
else if (!wasFirst && changed) Bus.emit("event", { channel: "junction", kind: "ok", n: 1, meta: {} });
|
||||
state.data.batches = data;
|
||||
Bus.emit("data", "batches");
|
||||
}
|
||||
|
||||
async function pollDatasets() {
|
||||
const data = await api(`/admin/api/datasets?date=${encodeURIComponent("")}`);
|
||||
state.data.datasets = data;
|
||||
Bus.emit("data", "datasets");
|
||||
}
|
||||
|
||||
async function pollAudit() {
|
||||
const data = await api("/admin/api/audit");
|
||||
const wasFirst = Seen.firstAudit;
|
||||
let maxId = Seen.auditMax;
|
||||
const fresh = [];
|
||||
for (const a of data.items || []) { if (a.id > Seen.auditMax) fresh.push(a); if (a.id > maxId) maxId = a.id; }
|
||||
Seen.auditMax = maxId;
|
||||
Seen.firstAudit = false;
|
||||
if (!wasFirst && fresh.length) {
|
||||
const rollback = fresh.some((a) => String(a.action).includes("rollback"));
|
||||
Bus.emit("event", { channel: "audit", kind: rollback ? "rollback" : "ok", n: fresh.length, meta: { fresh } });
|
||||
}
|
||||
state.data.audit = data;
|
||||
Bus.emit("data", "audit");
|
||||
}
|
||||
|
||||
const Poller = (() => {
|
||||
const specs = [
|
||||
{ key: "overview", fn: pollOverview, every: 20000 },
|
||||
{ key: "sources", fn: pollSources, every: 45000 },
|
||||
{ key: "jobs", fn: pollJobs, every: 25000 },
|
||||
{ key: "batches", fn: pollBatches, every: 25000 },
|
||||
{ key: "datasets", fn: pollDatasets, every: 60000 },
|
||||
{ key: "audit", fn: pollAudit, every: 20000 },
|
||||
];
|
||||
const timers = new Map();
|
||||
function runtimeAvailable() { return state.visible && state.online; }
|
||||
function tickOne(spec) { spec.fn().catch((err) => console.error(`[hub] poll ${spec.key} failed`, err)); }
|
||||
function schedule(spec) {
|
||||
clearTimer(spec.key);
|
||||
const t = setInterval(() => { if (runtimeAvailable()) tickOne(spec); }, spec.every);
|
||||
timers.set(spec.key, t);
|
||||
}
|
||||
function clearTimer(key) { if (timers.has(key)) { clearInterval(timers.get(key)); timers.delete(key); } }
|
||||
function startAll() { specs.forEach((spec, i) => { setTimeout(() => { tickOne(spec); schedule(spec); }, i * 160); }); }
|
||||
function stopAll() { specs.forEach((s) => clearTimer(s.key)); }
|
||||
function pause() { stopAll(); }
|
||||
function resume() { specs.forEach((spec) => { tickOne(spec); schedule(spec); }); }
|
||||
return { startAll, stopAll, pause, resume, runtimeAvailable };
|
||||
})();
|
||||
|
||||
function onRuntimeAvailabilityChange() {
|
||||
if (Poller.runtimeAvailable()) Poller.resume();
|
||||
else Poller.pause();
|
||||
Bus.emit("runtime", Poller.runtimeAvailable());
|
||||
}
|
||||
document.addEventListener("visibilitychange", () => { state.visible = document.visibilityState === "visible"; onRuntimeAvailabilityChange(); });
|
||||
window.addEventListener("online", () => { state.online = true; onRuntimeAvailabilityChange(); });
|
||||
window.addEventListener("offline", () => { state.online = false; onRuntimeAvailabilityChange(); });
|
||||
|
||||
/* ---------------------------------------------------------------- 主题 */
|
||||
function applyTheme(theme) {
|
||||
const root = document.documentElement;
|
||||
if (theme === "night") root.setAttribute("data-theme", "night");
|
||||
else root.removeAttribute("data-theme");
|
||||
try { localStorage.setItem("hub_theme", theme); } catch { /* ignore */ }
|
||||
Bus.emit("theme", theme);
|
||||
}
|
||||
function currentTheme() { return document.documentElement.getAttribute("data-theme") === "night" ? "night" : "day"; }
|
||||
function toggleTheme() { applyTheme(currentTheme() === "night" ? "day" : "night"); }
|
||||
function bootTheme() {
|
||||
const saved = (() => { try { return localStorage.getItem("hub_theme"); } catch { return null; } })();
|
||||
applyTheme(saved === "night" ? "night" : "day");
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------- 减少动态效果 */
|
||||
const REDUCE_MQ = matchMedia("(prefers-reduced-motion: reduce)");
|
||||
function applyReduced(reduced) {
|
||||
state.reduced = reduced;
|
||||
document.documentElement.classList.toggle("reduced", reduced);
|
||||
Bus.emit("reduced", reduced);
|
||||
}
|
||||
REDUCE_MQ.addEventListener("change", (e) => applyReduced(e.matches));
|
||||
|
||||
/* ---------------------------------------------------------------- 危险操作确认 */
|
||||
async function dangerous(kind, dataset, onDone) {
|
||||
const date = state.releaseDate || "";
|
||||
const ds = dataset || prompt("数据集（daily/valuation/moneyflow/auction/stocks→A组整批；index_daily→B组；或 reference）", "daily");
|
||||
if (!ds) return;
|
||||
const password = prompt("二次确认：输入管理密码");
|
||||
if (!password) return;
|
||||
const confirmWord = `${ds}:${date}`;
|
||||
const typed = prompt(`请输入确认词：${confirmWord}`);
|
||||
if (!typed) return;
|
||||
const path = kind === "rollback" ? "/admin/api/rollback" : "/admin/api/backfill";
|
||||
try {
|
||||
const result = await api(path, { method: "POST", body: JSON.stringify({ dataset: ds, trade_date: date, password, confirm: typed }) });
|
||||
if (kind === "rollback") Bus.emit("event", { channel: "tx", kind: "rollback", n: 1, meta: {} });
|
||||
await pollBatches();
|
||||
await pollAudit();
|
||||
if (onDone) onDone(result);
|
||||
} catch (err) {
|
||||
alert(err.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function probeSource(provider) {
|
||||
const result = await api(`/admin/api/sources/${provider}/probe`, { method: "POST", body: "{}" });
|
||||
if (FLOW_PROVIDERS.includes(provider)) {
|
||||
const link = linkState(result);
|
||||
Bus.emit("event", { channel: provider, kind: link === "ok" ? "ok" : link === "warn" ? "warn" : "error", n: 1, meta: { manual: true } });
|
||||
}
|
||||
await pollSources();
|
||||
return result;
|
||||
}
|
||||
|
||||
async function runJob(jobId, tradeDate) {
|
||||
const result = await api(`/admin/api/jobs/${jobId}/run`, { method: "POST", body: JSON.stringify({ trade_date: tradeDate || "" }) });
|
||||
await pollJobs();
|
||||
return result;
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------- 路由：?layout=flowline|ledger|strata
|
||||
单一静态文件，只靠查询参数区分三页；不改后端路由。
|
||||
支持直接打开 / 刷新 / 浏览器前进后退。 */
|
||||
const LAYOUT_NAMES = ["flowline", "ledger", "strata"];
|
||||
const Router = (() => {
|
||||
let mounted = null;
|
||||
function parse() {
|
||||
const qs = new URLSearchParams(location.search);
|
||||
const l = qs.get("layout");
|
||||
return LAYOUT_NAMES.includes(l) ? l : "flowline";
|
||||
}
|
||||
function urlFor(name) {
|
||||
const qs = new URLSearchParams(location.search);
|
||||
qs.set("layout", name);
|
||||
return `${location.pathname}?${qs.toString()}${location.hash}`;
|
||||
}
|
||||
function navigate(name) {
|
||||
if (!LAYOUT_NAMES.includes(name)) return;
|
||||
if (parse() === name) return;
|
||||
history.pushState(null, "", urlFor(name));
|
||||
mount(name);
|
||||
}
|
||||
function mount(name) {
|
||||
if (mounted && window.HUB_LAYOUTS[mounted] && window.HUB_LAYOUTS[mounted].unmount) {
|
||||
try { window.HUB_LAYOUTS[mounted].unmount(); } catch (e) { console.error(e); }
|
||||
}
|
||||
state.layout = name;
|
||||
mounted = name;
|
||||
document.querySelectorAll("#layoutSwitch button[data-layout]").forEach((btn) => {
|
||||
const on = btn.dataset.layout === name;
|
||||
if (on) btn.setAttribute("aria-current", "page"); else btn.removeAttribute("aria-current");
|
||||
});
|
||||
const root = $("page-root");
|
||||
const impl = window.HUB_LAYOUTS[name];
|
||||
if (!impl) { root.innerHTML = `<div class="page-shell">布局未加载：${esc(name)}</div>`; return; }
|
||||
impl.mount(root);
|
||||
}
|
||||
function boot() {
|
||||
document.querySelectorAll("#layoutSwitch button[data-layout]").forEach((btn) => {
|
||||
btn.addEventListener("click", () => navigate(btn.dataset.layout));
|
||||
});
|
||||
window.addEventListener("popstate", () => mount(parse()));
|
||||
mount(parse());
|
||||
}
|
||||
return { boot, navigate, current: parse, urlFor };
|
||||
})();
|
||||
|
||||
return {
|
||||
$, esc, timeShort, clamp, table,
|
||||
state, api, Bus, Poller, Router,
|
||||
FLOW_PROVIDERS, FLOW_LABEL, FLOW_ROLE, EOD_LABELS, REV_LABELS, PHASE_LABELS,
|
||||
todayYmd, chipClass, chipHtml, linkState, linkLampClass,
|
||||
applyTheme, currentTheme, toggleTheme, bootTheme,
|
||||
applyReduced, REDUCE_MQ,
|
||||
dangerous, probeSource, runJob,
|
||||
pollOverview, pollSources, pollJobs, pollBatches, pollDatasets, pollAudit,
|
||||
};
|
||||
})();
|
||||
@@ -3,19 +3,14 @@
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>xiaobai-datahub 管理后台 · 数据中枢</title>
|
||||
<link rel="stylesheet" href="/admin/tokens.css" />
|
||||
<link rel="stylesheet" href="/admin/shared.css" />
|
||||
<link rel="stylesheet" href="/admin/layouts/flowline.css" />
|
||||
<link rel="stylesheet" href="/admin/layouts/ledger.css" />
|
||||
<link rel="stylesheet" href="/admin/layouts/strata.css" />
|
||||
<title>xiaobai-datahub 管理后台</title>
|
||||
<link rel="stylesheet" href="/admin/styles.css" />
|
||||
</head>
|
||||
<body>
|
||||
<div id="app">
|
||||
<section id="login-view" class="panel auth-panel">
|
||||
<span class="badge-sim">内网 · 8766</span>
|
||||
<h1>数据中枢</h1>
|
||||
<p class="muted">四路来源持续汇流、调度、发布与审计的运转空间。</p>
|
||||
<p class="muted">内网管理后台，用于查看源状态、调度和盘后发布批次。</p>
|
||||
<form id="login-form">
|
||||
<label>账号 <input name="username" value="hub_admin" autocomplete="username" /></label>
|
||||
<label>密码 <input name="password" type="password" autocomplete="current-password" /></label>
|
||||
@@ -34,31 +29,25 @@
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section id="shell" hidden style="display:flex; flex-direction:column; min-height:100vh;">
|
||||
<header id="topbar">
|
||||
<div class="title">小白复盘 <em>·</em> 数据中枢</div>
|
||||
<span class="vdiv"></span>
|
||||
<span class="tdate mono" id="crumb">8766 · 四源汇流 · 持续运转</span>
|
||||
<span class="spacer"></span>
|
||||
<nav id="layoutSwitch" aria-label="页面切换">
|
||||
<button type="button" data-layout="flowline"><span class="k">A</span>装配线</button>
|
||||
<button type="button" data-layout="ledger"><span class="k">B</span>值班台账</button>
|
||||
<button type="button" data-layout="strata"><span class="k">C</span>地层剖面</button>
|
||||
</nav>
|
||||
<span class="who muted" id="who"></span>
|
||||
<button type="button" id="theme-btn" class="btn ghost">夜间</button>
|
||||
<button type="button" id="logout-btn" class="btn ghost">退出</button>
|
||||
<section id="shell" hidden>
|
||||
<header class="top">
|
||||
<strong>xiaobai-datahub</strong>
|
||||
<span id="phase" class="pill"></span>
|
||||
<span id="who" class="muted"></span>
|
||||
<button type="button" id="theme-btn" class="ghost">夜间</button>
|
||||
<button type="button" id="logout-btn" class="ghost">退出</button>
|
||||
</header>
|
||||
|
||||
<main id="page-root" style="flex:1;"></main>
|
||||
<nav>
|
||||
<button data-page="overview" class="active">总览</button>
|
||||
<button data-page="sources">数据源</button>
|
||||
<button data-page="jobs">调度任务</button>
|
||||
<button data-page="release">盘后发布</button>
|
||||
<button data-page="datasets">数据集</button>
|
||||
<button data-page="audit">审计</button>
|
||||
</nav>
|
||||
<main id="page"></main>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<script src="/admin/core.js"></script>
|
||||
<script src="/admin/layouts/shared.js"></script>
|
||||
<script src="/admin/layouts/flowline.js"></script>
|
||||
<script src="/admin/layouts/ledger.js"></script>
|
||||
<script src="/admin/layouts/strata.js"></script>
|
||||
<script src="/admin/main.js"></script>
|
||||
<script src="/admin/app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
/* A · 装配线 Flowline —— 来源→加工→发布→审计 四工位水平因果链
|
||||
只写这个布局独有的排布；颜色/组件规则一律来自 tokens.css + shared.css。 */
|
||||
#fl-root { padding: var(--sp4); max-width: 1584px; margin: 0 auto; }
|
||||
|
||||
#fl-band { position: relative; padding: 14px 16px 18px; margin-bottom: 14px; overflow: hidden; }
|
||||
#fl-band .lane-labels { display: flex; }
|
||||
#fl-band .lane-labels span { flex: 1; }
|
||||
#fl-wire { position: relative; height: 1px; background: var(--line); margin: 22px 0 18px; }
|
||||
#fl-wire .chev { position: absolute; top: -8px; font-size: 13px; color: var(--t3); font-family: var(--font-mono); transform: translateX(-50%); }
|
||||
|
||||
#fl-lanes { display: grid; grid-template-columns: 2fr 1.15fr 1.05fr 0.85fr; gap: 14px; align-items: start; }
|
||||
/* 防止内部宽表格用 min-width:auto 撑爆 1fr 轨道，参见 ledger.css 同类注释。 */
|
||||
#fl-lanes > *, #fl-cols > * { min-width: 0; }
|
||||
#fl-srcCards { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; }
|
||||
.fl-src-card { position: relative; padding: 10px 12px; display: flex; flex-direction: column; gap: 6px; overflow: hidden; }
|
||||
.fl-src-card .row { display: flex; align-items: center; gap: 8px; }
|
||||
.fl-src-card .name { font-size: 14px; font-weight: 600; color: var(--t1); }
|
||||
.fl-src-card .role { font-size: 11px; color: var(--t3); }
|
||||
.fl-src-card .grow { flex: 1; }
|
||||
.fl-src-card .lat { font-size: 12px; color: var(--t2); }
|
||||
.fl-src-card .cred { font-size: 11px; color: var(--t3); }
|
||||
.fl-src-card .lamps { display: flex; align-items: center; gap: 4px; }
|
||||
.fl-src-card .edge {
|
||||
position: absolute; left: 0; top: 6px; bottom: 6px; width: 2px; background: var(--act);
|
||||
border-radius: 2px; opacity: 0; transition: opacity var(--dur-ui) var(--ease-out);
|
||||
}
|
||||
.fl-src-card .edge.err { background: var(--error); }
|
||||
|
||||
.fl-station { padding: 12px 14px; min-height: 150px; position: relative; transition: box-shadow var(--dur-ui) var(--ease-out); }
|
||||
.fl-station.pulse { box-shadow: 0 0 0 2px var(--action-soft) inset; }
|
||||
.fl-station .hd { display: flex; align-items: center; gap: 8px; margin-bottom: 8px; }
|
||||
.fl-station .hd .nm { font-size: 13px; font-weight: 600; color: var(--t1); }
|
||||
.fl-station .hd .grow { flex: 1; }
|
||||
.fl-station .task { font-size: 12px; color: var(--t2); margin-bottom: 8px; }
|
||||
.fl-station .task b { color: var(--t1); font-weight: 600; }
|
||||
.fl-pbar { position: relative; height: 4px; border-radius: 2px; background: var(--bg2); overflow: hidden; margin-bottom: 6px; }
|
||||
.fl-pbar > i { position: absolute; left: 0; top: 0; bottom: 0; background: var(--action); border-radius: 2px; transition: width var(--dur-ui) var(--ease-out); }
|
||||
.fl-station .queue { font-size: 12px; color: var(--t2); margin-bottom: 8px; display: flex; align-items: center; gap: 6px; }
|
||||
.fl-miniruns { border-top: 1px solid var(--line-soft); padding-top: 6px; }
|
||||
.fl-miniruns .mr { display: flex; gap: 8px; font-size: 11px; line-height: 1.7; }
|
||||
.fl-miniruns .mr .id { color: var(--t3); width: 34px; }
|
||||
.fl-miniruns .mr .why { color: var(--error); }
|
||||
|
||||
.fl-gate .bignum { display: flex; align-items: baseline; gap: 8px; margin-bottom: 8px; }
|
||||
.fl-gate .bignum .n { font-size: var(--fs-hero); font-weight: 600; color: var(--t1); display: inline-block; transition: transform var(--dur-ui) var(--ease-out); }
|
||||
.fl-gate .bignum .cap { font-size: 12px; color: var(--t3); }
|
||||
.fl-gate .latest { font-size: 12px; color: var(--t2); margin-bottom: 10px; }
|
||||
.fl-gate .valrow { display: flex; align-items: center; gap: 8px; font-size: 12px; color: var(--t2); }
|
||||
|
||||
.fl-audit .al { font-size: 11px; line-height: 1.75; color: var(--t2); white-space: normal; word-break: break-all; }
|
||||
.fl-audit .al .dim { color: var(--t3); }
|
||||
.fl-audit .adiv { border-top: 1px solid var(--line-soft); margin: 6px 0; }
|
||||
|
||||
#fl-packet {
|
||||
position: absolute; width: 34px; height: 3px; border-radius: 2px; background: var(--packet);
|
||||
opacity: 0; pointer-events: none; z-index: 5; top: 0; left: 0;
|
||||
}
|
||||
|
||||
#fl-cols { display: grid; grid-template-columns: 1.15fr 1.1fr 1fr; gap: 14px; }
|
||||
.fl-col { padding: 12px 14px; }
|
||||
.fl-col .sec-label { display: block; margin-bottom: 8px; }
|
||||
.fl-col .sub-label { display: block; margin: 12px 0 4px; }
|
||||
#fl-callsTable td.res-ok { color: var(--t2); }
|
||||
#fl-callsTable td.res-err { color: var(--error); }
|
||||
#fl-callsFoot { margin-top: 8px; font-size: 11px; color: var(--t3); border-top: 1px solid var(--line-soft); padding-top: 8px; }
|
||||
.fl-newrow { transition: background var(--dur-ui) var(--ease-out); }
|
||||
.fl-newrow.flashin { background: var(--alive-soft); }
|
||||
|
||||
.fl-jobrow { display: flex; align-items: center; gap: 10px; padding: 7px 0; border-bottom: 1px solid var(--line-soft); }
|
||||
.fl-jobrow .jid { font-size: 12px; color: var(--t1); }
|
||||
.fl-jobrow .jti { font-size: 12px; color: var(--t2); }
|
||||
.fl-jobrow .jat { font-size: 11px; color: var(--t3); }
|
||||
.fl-jobrow .grow { flex: 1; }
|
||||
|
||||
#fl-col3 .toolrow { display: flex; align-items: center; gap: 8px; margin-bottom: 10px; }
|
||||
#fl-col3 .toolrow input {
|
||||
font-family: var(--font-mono); font-size: 12px; color: var(--t1); background: var(--bg2);
|
||||
border: 1px solid var(--line); border-radius: var(--r-chip); padding: 5px 8px; width: 96px; outline: none;
|
||||
}
|
||||
.fl-anom { margin-top: 10px; }
|
||||
tr.fl-rowflash td { transition: background var(--dur-ui) var(--ease-out); }
|
||||
tr.fl-rowflash.on td { background: var(--alive-soft); }
|
||||
|
||||
/* ---------------- 1280 ---------------- */
|
||||
@media (max-width: 1280px) {
|
||||
#fl-lanes { grid-template-columns: 1fr; }
|
||||
#fl-srcCards { grid-template-columns: 1fr 1fr; }
|
||||
#fl-cols { grid-template-columns: 1fr 1fr; }
|
||||
#fl-col3 { grid-column: 1 / -1; }
|
||||
}
|
||||
/* ---------------- 1024 ---------------- */
|
||||
@media (max-width: 1100px) {
|
||||
#fl-root { padding: var(--sp3); }
|
||||
#fl-srcCards { grid-template-columns: 1fr; }
|
||||
#fl-cols { grid-template-columns: 1fr; }
|
||||
.fl-src-card .role, .fl-src-card .cred { display: none; }
|
||||
}
|
||||
@@ -1,300 +0,0 @@
|
||||
"use strict";
|
||||
/* ==========================================================================
|
||||
A · 装配线 Flowline —— 来源 → 加工 → 发布 → 审计 四工位水平因果链。
|
||||
数据卡是线上工件，真实事件沿发丝导线从触发的那一工位接力到下一工位。
|
||||
========================================================================== */
|
||||
(function () {
|
||||
const C = window.Core, S = window.HubShared;
|
||||
const { $, esc, timeShort } = C;
|
||||
let root = null;
|
||||
let unsubs = [];
|
||||
let stationFlashTimers = {};
|
||||
let mounted = false;
|
||||
let pendingTimers = [];
|
||||
/* 布局随时可能被切走：任何跨阶段延时回调都必须先确认还挂载着，
|
||||
否则 unmount 之后残留的 setTimeout 会在 root=null 时报错。 */
|
||||
function safeTimeout(fn, ms) {
|
||||
const id = setTimeout(() => { if (mounted) fn(); }, ms);
|
||||
pendingTimers.push(id);
|
||||
return id;
|
||||
}
|
||||
|
||||
const SKELETON = `
|
||||
<div id="fl-root" class="page-shell">
|
||||
<section id="fl-band" class="card">
|
||||
<div class="lane-labels">
|
||||
<span class="sec-label">来源 SOURCES</span>
|
||||
<span class="sec-label">加工 PROCESS</span>
|
||||
<span class="sec-label">发布 PUBLISH</span>
|
||||
<span class="sec-label">审计 AUDIT</span>
|
||||
</div>
|
||||
<div id="fl-wire"><span class="chev" style="left:26%">›</span><span class="chev" style="left:57%">›</span><span class="chev" style="left:80%">›</span></div>
|
||||
<div id="fl-lanes">
|
||||
<div id="fl-srcCards"></div>
|
||||
<div id="fl-process" class="card fl-station" data-station="process">
|
||||
<div class="hd"><span class="nm">中枢处理</span><span class="grow"></span><span data-pct class="mono">-</span></div>
|
||||
<div id="fl-processBody"></div>
|
||||
</div>
|
||||
<div id="fl-publish" class="card fl-station fl-gate" data-station="publish">
|
||||
<div class="hd"><span class="nm">发布闸口</span><span class="grow"></span><span class="sec-label">GATE</span></div>
|
||||
<div id="fl-publishBody"></div>
|
||||
</div>
|
||||
<div id="fl-audit" class="card fl-station fl-audit" data-station="audit">
|
||||
<div class="hd"><span class="nm">审计末端</span></div>
|
||||
<div id="fl-auditBody"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="fl-packet"></div>
|
||||
</section>
|
||||
|
||||
<div id="fl-cols">
|
||||
<section id="fl-col1" class="card fl-col">
|
||||
<span class="sec-label">最近调用 RECENT CALLS</span>
|
||||
<div id="fl-callsTable"></div>
|
||||
<div id="fl-callsFoot"></div>
|
||||
</section>
|
||||
<section id="fl-col2" class="card fl-col">
|
||||
<span class="sec-label">调度任务 JOBS</span>
|
||||
<div id="fl-jobRows"></div>
|
||||
<span class="sec-label sub-label">最近运行 RUNS</span>
|
||||
<div id="fl-runsTable"></div>
|
||||
</section>
|
||||
<section id="fl-col3" class="card fl-col">
|
||||
<span class="sec-label">盘后发布 RELEASE</span>
|
||||
<div class="toolrow">
|
||||
<input id="fl-rel-date" value="" spellcheck="false" />
|
||||
<button type="button" class="btn" id="fl-rel-load">查看</button>
|
||||
<button type="button" class="btn danger" id="fl-rel-backfill">补数</button>
|
||||
</div>
|
||||
<span class="sec-label sub-label">当前映射 MAPPING</span>
|
||||
<div id="fl-pubTable"></div>
|
||||
<div id="fl-anom" class="fl-anom"></div>
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
function srcCardHtml(item) {
|
||||
const health = item.health || {};
|
||||
const link = item._link || C.linkState(health);
|
||||
const cred = item.credential || {};
|
||||
const credText = cred.configured ? `已配置 · ${esc(cred.last4 || "****")}` : "未配置";
|
||||
return `
|
||||
<div class="fl-src-card card" data-provider="${esc(item.provider)}">
|
||||
<div class="edge"></div>
|
||||
<div class="row">
|
||||
<span class="name">${esc(C.FLOW_LABEL[item.provider] || item.provider)}</span>
|
||||
<span class="role">${esc(C.FLOW_ROLE[item.provider] || item.role)}</span>
|
||||
<span class="grow"></span>
|
||||
${C.chipHtml(link, health.state || link)}
|
||||
</div>
|
||||
<div class="row">
|
||||
<span class="lamps"><i class="lamp link ${C.linkLampClass(link)}" data-link></i><span class="lamp-tag">LINK</span></span>
|
||||
<span class="lamps"><i class="lamp act" data-act></i><span class="lamp-tag">ACT</span></span>
|
||||
<span class="grow"></span>
|
||||
<span class="lat mono">${health.latency_ms == null ? "-" : health.latency_ms + "ms"}</span>
|
||||
</div>
|
||||
<div class="row">
|
||||
<span class="cred mono">${credText}</span><span class="grow"></span>
|
||||
<button class="btn ghost mini" data-probe="${esc(item.provider)}">探测一次</button>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderSources() {
|
||||
const v = S.sourcesView();
|
||||
const host = $("fl-srcCards");
|
||||
if (!v) { host.innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
host.innerHTML = v.live.map(srcCardHtml).join("") +
|
||||
(v.reserved.length ? `<div class="fl-src-card card muted" style="align-items:center;justify-content:center;font-size:12px;grid-column: 1 / -1">预留源 ${v.reserved.length} 个 · 未接入 · 无真实调用</div>` : "");
|
||||
S.bindProbeButtons(host);
|
||||
}
|
||||
|
||||
function renderProcess() {
|
||||
const jv = S.jobsView();
|
||||
const body = $("fl-processBody");
|
||||
const pctEl = root.querySelector('[data-pct]');
|
||||
if (!jv || !jv.latest) { body.innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
const latest = jv.latest;
|
||||
const running = jv.runs.filter((r) => r.state === "running").length;
|
||||
const pct = latest.state === "running" ? 62 : latest.state === "success" ? 100 : latest.state === "failed" ? 100 : 0;
|
||||
pctEl.textContent = latest.state === "running" ? `${pct}%` : "空闲";
|
||||
body.innerHTML = `
|
||||
<div class="task mono"><b>${esc(latest.job_id)}</b> · ${esc((jv.jobs.find((j) => j.id === latest.job_id) || {}).title || "")}</div>
|
||||
<div class="fl-pbar"><i style="width:${pct}%;background:${latest.state === "failed" ? "var(--error)" : "var(--action)"}"></i></div>
|
||||
<div class="queue">队列 <b class="mono">${running}</b> · 最近 20 次失败 <span class="chip ${jv.failedRecent ? "err" : "ok"}">${jv.failedRecent}</span></div>
|
||||
<div class="fl-miniruns">
|
||||
${jv.runs.slice(0, 3).map((r) => `<div class="mr mono"><span class="id">${r.id}</span><span class="${r.state === "failed" ? "fail" : r.state === "running" ? "warn" : "ok"}">${esc(r.state)}</span><span class="${r.error ? "why" : "muted"}">${esc(r.error || (r.finished_at !== "-" && r.finished_at ? timeShort(r.finished_at) : "—"))}</span></div>`).join("")}
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function renderPublish() {
|
||||
const rv = S.releaseView();
|
||||
const body = $("fl-publishBody");
|
||||
if (!rv) { body.innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
const valuation = rv.pubs.find((p) => p.dataset === "valuation");
|
||||
const latestPub = rv.pubs.slice().sort((a, b) => String(a.published_at).localeCompare(String(b.published_at))).pop();
|
||||
body.innerHTML = `
|
||||
<div class="bignum"><span class="n mono" data-pubnum>${rv.pubs.length}</span><span class="cap">今日发布</span></div>
|
||||
<div class="latest mono">最新批次 ${esc(latestPub ? `${latestPub.active_batch} · ${latestPub.published_at}` : "暂无")}</div>
|
||||
${valuation ? `<div class="valrow"><span class="mono">valuation</span>${C.chipHtml(valuation.state)}</div>` : ""}`;
|
||||
}
|
||||
|
||||
function renderAudit() {
|
||||
const av = S.auditView();
|
||||
const body = $("fl-auditBody");
|
||||
if (!av || !av.items.length) { body.innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
const a0 = av.items[0];
|
||||
body.innerHTML = `
|
||||
<div class="al mono">${esc(timeShort(a0.created_at))} ${esc(a0.actor)} · ${esc(a0.action)}</div>
|
||||
<div class="al mono dim">${esc(a0.target)}${a0.detail ? " · " + esc(a0.detail) : ""}</div>
|
||||
<div class="adiv"></div>
|
||||
${av.items.slice(1, 3).map((a) => `<div class="al mono dim">${esc(timeShort(a.created_at))} ${esc(a.actor)} · ${esc(a.action)}</div>`).join("")}`;
|
||||
}
|
||||
|
||||
function renderCalls() {
|
||||
const ov = S.overviewView();
|
||||
const sv = S.sourcesView();
|
||||
if (!ov) { $("fl-callsTable").innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
$("fl-callsTable").innerHTML = C.table(["时间", "源", "端点", "结果", "耗时"], ov.recentCalls.map((row) => [
|
||||
`<span class="mono">${esc(timeShort(row.created_at))}</span>`,
|
||||
`<span class="mono k">${esc(row.provider)}</span>`,
|
||||
`<span class="mono">${esc(row.endpoint)}</span>`,
|
||||
row.ok ? '<span class="res-ok">成功</span>' : `<span class="res-err mono">${esc(row.error)}</span>`,
|
||||
`<span class="mono">${row.latency_ms ?? "-"} ms</span>`,
|
||||
]));
|
||||
if (sv) {
|
||||
$("fl-callsFoot").textContent = "今日调用 · " + sv.live.map((s) => `${s.provider} ${s.calls_today ?? 0}`).join(" · ");
|
||||
}
|
||||
}
|
||||
|
||||
function renderJobsCol() {
|
||||
const jv = S.jobsView();
|
||||
const jobRows = $("fl-jobRows");
|
||||
if (!jv) { jobRows.innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
jobRows.innerHTML = jv.jobs.map((job) => `
|
||||
<div class="fl-jobrow">
|
||||
<span class="jid mono">${esc(job.id)}</span><span class="jti">${esc(job.title)}</span>
|
||||
<span class="jat mono">${esc(job.at)}</span>${C.chipHtml(jv.latestByJob.get(job.id) ? jv.latestByJob.get(job.id).state : "idle")}
|
||||
<span class="grow"></span><button class="btn ghost mini" data-run="${esc(job.id)}">手动触发</button>
|
||||
</div>`).join("");
|
||||
S.bindRunButtons(jobRows);
|
||||
$("fl-runsTable").innerHTML = C.table(["ID", "任务", "状态", "开始", "结束", "错误"], jv.runs.map((r) => [
|
||||
`<span class="mono">${r.id}</span>`, `<span class="mono k">${esc(r.job_id)}</span>`,
|
||||
`<span class="mono ${r.state === "failed" ? "fail" : r.state === "running" ? "warn" : "ok"}">${esc(r.state)}</span>`,
|
||||
`<span class="mono">${esc(timeShort(r.started_at))}</span>`, `<span class="mono">${esc(timeShort(r.finished_at))}</span>`,
|
||||
`<span class="fail">${esc(r.error || "")}</span>`,
|
||||
]));
|
||||
}
|
||||
|
||||
function renderReleaseCol() {
|
||||
const rv = S.releaseView();
|
||||
const dateInput = $("fl-rel-date");
|
||||
if (!dateInput.value) dateInput.value = C.state.releaseDate;
|
||||
if (!rv) { $("fl-pubTable").innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
$("fl-pubTable").innerHTML = C.table(["数据集", "活跃批次", "上一批次", "状态", "发布时间", "操作"], rv.pubs.map((p) => [
|
||||
`<span class="mono k">${esc(p.dataset)}</span>`, `<span class="mono">${esc(p.active_batch)}</span>`,
|
||||
`<span class="mono">${esc(p.prev_batch || "-")}</span>`, C.chipHtml(p.state),
|
||||
`<span class="mono">${esc(p.published_at)}</span>`,
|
||||
p.prev_batch ? `<button class="btn danger mini" data-rollback="${esc(p.dataset)}">回滚</button>` : "-",
|
||||
]));
|
||||
S.bindRollbackButtons($("fl-pubTable"), () => renderReleaseCol());
|
||||
const failed = rv.batches.filter((b) => b.state === "failed" || b.state === "error");
|
||||
$("fl-anom").innerHTML = failed.length
|
||||
? failed.map((b) => `<span class="chip err mono">${esc(b.batch_id)} · ${esc(b.dataset)} · ${esc(b.error || "异常")}</span>`).join(" ")
|
||||
: `<span class="chip ok">批次全部正常</span>`;
|
||||
}
|
||||
|
||||
function refresh() {
|
||||
if (!root) return;
|
||||
renderSources(); renderProcess(); renderPublish(); renderAudit();
|
||||
renderCalls(); renderJobsCol(); renderReleaseCol();
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------- 真实事件动效
|
||||
水平接力：source ACT 闪 → 沿导线光梭滑到 process → process 高亮 → 光梭滑到
|
||||
publish → publish 高亮 → 光梭滑到 audit → audit 高亮。reduced 时只做状态高亮，
|
||||
不做位移。 */
|
||||
function flashSource(provider, kind) {
|
||||
const card = root.querySelector(`.fl-src-card[data-provider="${provider}"]`);
|
||||
if (!card) return;
|
||||
const act = card.querySelector("[data-act]");
|
||||
act.classList.remove("on", "err");
|
||||
act.classList.add("on"); if (kind === "error" || kind === "rollback") act.classList.add("err");
|
||||
const edge = card.querySelector(".edge");
|
||||
edge.classList.toggle("err", kind === "error");
|
||||
edge.style.opacity = "1";
|
||||
clearTimeout(card._flashTimer);
|
||||
card._flashTimer = setTimeout(() => { act.classList.remove("on", "err"); edge.style.opacity = "0"; }, 820);
|
||||
return card;
|
||||
}
|
||||
|
||||
function pulseStation(key, ms) {
|
||||
const el = root.querySelector(`.fl-station[data-station="${key}"]`);
|
||||
if (!el) return;
|
||||
el.classList.add("pulse");
|
||||
clearTimeout(stationFlashTimers[key]);
|
||||
stationFlashTimers[key] = setTimeout(() => el.classList.remove("pulse"), ms || 640);
|
||||
}
|
||||
|
||||
function travelPacket(fromEl, toEl, color) {
|
||||
if (!fromEl || !toEl || C.state.reduced) return;
|
||||
const pk = $("fl-packet");
|
||||
const band = $("fl-band").getBoundingClientRect();
|
||||
const a = fromEl.getBoundingClientRect(), b = toEl.getBoundingClientRect();
|
||||
const x0 = a.left + a.width / 2 - band.left, y0 = a.top + a.height / 2 - band.top;
|
||||
const x1 = b.left + b.width / 2 - band.left, y1 = b.top + b.height / 2 - band.top;
|
||||
pk.style.background = color || "var(--packet)";
|
||||
pk.style.top = y0 - 1.5 + "px"; pk.style.left = x0 - 17 + "px"; pk.style.opacity = "0";
|
||||
if (pk._anim) pk._anim.cancel();
|
||||
pk._anim = pk.animate([
|
||||
{ transform: "translate(0,0)", opacity: 0 },
|
||||
{ transform: "translate(0,0)", opacity: 1, offset: .08 },
|
||||
{ transform: `translate(${x1 - x0}px, ${y1 - y0}px)`, opacity: 1, offset: .92 },
|
||||
{ transform: `translate(${x1 - x0}px, ${y1 - y0}px)`, opacity: 0 },
|
||||
], { duration: 640, easing: "linear" });
|
||||
pk._anim.onfinish = () => { pk.style.opacity = "0"; };
|
||||
}
|
||||
|
||||
function onEvent(evt) {
|
||||
if (!root || !mounted) return;
|
||||
const process = root.querySelector('.fl-station[data-station="process"]');
|
||||
const publish = root.querySelector('.fl-station[data-station="publish"]');
|
||||
const audit = root.querySelector('.fl-station[data-station="audit"]');
|
||||
if (C.FLOW_PROVIDERS.includes(evt.channel)) {
|
||||
const card = flashSource(evt.channel, evt.kind);
|
||||
safeTimeout(() => { travelPacket(card, process, evt.kind === "error" ? "var(--error)" : "var(--packet)"); pulseStation("process", 700); }, 260);
|
||||
renderCalls();
|
||||
} else if (evt.channel === "junction") {
|
||||
pulseStation("process", 700);
|
||||
safeTimeout(() => { travelPacket(process, publish, "var(--packet)"); pulseStation("publish", 700); renderProcess(); }, 260);
|
||||
} else if (evt.channel === "tx") {
|
||||
pulseStation("publish", 700);
|
||||
safeTimeout(() => { travelPacket(publish, audit, evt.kind === "rollback" ? "var(--error)" : "var(--packet)"); pulseStation("audit", 700); renderPublish(); renderReleaseCol(); }, 260);
|
||||
} else if (evt.channel === "audit") {
|
||||
pulseStation("audit", 700);
|
||||
renderAudit();
|
||||
}
|
||||
}
|
||||
|
||||
function mount(el) {
|
||||
root = el;
|
||||
mounted = true;
|
||||
root.innerHTML = SKELETON;
|
||||
refresh();
|
||||
S.bindReleaseDateReload(root, "#fl-rel-date", "#fl-rel-load", () => renderReleaseCol());
|
||||
S.bindBackfillButton(root, "#fl-rel-backfill", () => renderReleaseCol());
|
||||
unsubs.push(C.Bus.on("data", refresh));
|
||||
unsubs.push(C.Bus.on("event", onEvent));
|
||||
}
|
||||
function unmount() {
|
||||
mounted = false;
|
||||
unsubs.forEach((fn) => fn()); unsubs = [];
|
||||
Object.values(stationFlashTimers).forEach(clearTimeout); stationFlashTimers = {};
|
||||
pendingTimers.forEach(clearTimeout); pendingTimers = [];
|
||||
root = null;
|
||||
}
|
||||
|
||||
window.HUB_LAYOUTS = window.HUB_LAYOUTS || {};
|
||||
window.HUB_LAYOUTS.flowline = { mount, unmount };
|
||||
})();
|
||||
@@ -1,81 +0,0 @@
|
||||
/* B · 值班台账 Ledger Desk —— 表格本身就是主舞台，每条真实事件一行。
|
||||
高密度、精密仪器感；按钮/状态/异常一眼可读。 */
|
||||
#lg-root { padding: var(--sp3) var(--sp4); max-width: 1680px; margin: 0 auto; }
|
||||
#lg-grid { display: grid; grid-template-columns: 250px 1fr 300px; gap: 12px; align-items: start; }
|
||||
/* CSS Grid 子项默认 min-width:auto，会被内部宽表格的内在尺寸撑爆整条 1fr 轨道；
|
||||
显式清零，让表格自己用 overflow 滚动，而不是撑破布局。 */
|
||||
#lg-grid > * { min-width: 0; }
|
||||
|
||||
#lg-side, #lg-side-r { display: flex; flex-direction: column; gap: 10px; min-width: 0; }
|
||||
.lg-panel { padding: 10px 12px; min-width: 0; }
|
||||
.lg-panel .sec-label { display: block; margin-bottom: 8px; }
|
||||
|
||||
.lg-src-row { display: flex; align-items: center; gap: 7px; padding: 5px 0; border-bottom: 1px solid var(--line-soft); font-size: 12px; }
|
||||
.lg-src-row:last-child { border-bottom: 0; }
|
||||
.lg-src-row .nm { flex: 1; color: var(--t1); }
|
||||
.lg-src-row .lat { color: var(--t3); font-size: 11px; }
|
||||
.lg-src-row .lamps { display: flex; gap: 3px; }
|
||||
|
||||
.lg-jobline { display: flex; align-items: center; gap: 6px; padding: 5px 0; border-bottom: 1px solid var(--line-soft); font-size: 12px; }
|
||||
.lg-jobline:last-child { border-bottom: 0; }
|
||||
.lg-jobline .nm { flex: 1; color: var(--t1); }
|
||||
.lg-jobline .at { color: var(--t3); font-size: 11px; }
|
||||
.lg-jobline button { margin-left: 4px; }
|
||||
|
||||
/* -------- 主舞台：事件台账表 -------- */
|
||||
#lg-ledger { padding: 10px 12px; }
|
||||
#lg-ledger .lg-ledger-hd { display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 8px; }
|
||||
#lg-ledger .lg-ledger-hd .hint { font-size: 11px; color: var(--t3); }
|
||||
table.lg-ledger { width: 100%; border-collapse: collapse; font-size: 12px; }
|
||||
table.lg-ledger th {
|
||||
position: sticky; top: 0; background: var(--bg1); text-align: left; font-weight: 500; color: var(--t3);
|
||||
font-size: 11px; padding: 6px 8px; border-bottom: 1px solid var(--line); white-space: nowrap; z-index: 1;
|
||||
}
|
||||
table.lg-ledger th.stage { text-align: center; }
|
||||
table.lg-ledger td { padding: 7px 8px; border-bottom: 1px solid var(--line-soft); vertical-align: middle; }
|
||||
table.lg-ledger td.ev { min-width: 150px; }
|
||||
table.lg-ledger td.ev .nm { color: var(--t1); font-weight: 600; font-size: 12.5px; }
|
||||
table.lg-ledger td.ev .sub { color: var(--t3); font-size: 10.5px; }
|
||||
table.lg-ledger td.src { color: var(--t2); font-size: 11.5px; white-space: nowrap; }
|
||||
table.lg-ledger td.stage { text-align: center; white-space: nowrap; }
|
||||
.lg-cell {
|
||||
display: inline-flex; align-items: center; justify-content: center; gap: 4px; min-width: 58px;
|
||||
padding: 3px 7px; border-radius: var(--r-chip); font-family: var(--font-mono); font-size: 11px;
|
||||
background: var(--bg2); color: var(--t3); transition: background var(--dur-ui) var(--ease-out), color var(--dur-ui) var(--ease-out);
|
||||
}
|
||||
.lg-cell.ok { background: var(--alive-soft); color: var(--alive); }
|
||||
.lg-cell.err { background: var(--error-soft); color: var(--error); }
|
||||
.lg-cell.warn { background: var(--act-soft); color: var(--act); }
|
||||
.lg-cell.dash { opacity: .4; }
|
||||
tr.lg-row { transition: background var(--dur-ui) var(--ease-out); }
|
||||
tr.lg-row.flashnew td { background: var(--action-soft); }
|
||||
tr.lg-row.flashnew.err td { background: var(--error-soft); }
|
||||
|
||||
/* -------- 调用带（持续，只滚真实数据） -------- */
|
||||
#lg-tape { margin-top: 10px; padding: 8px 0 0; border-top: 1px solid var(--line-soft); }
|
||||
#lg-tape .sec-label { display: block; margin-bottom: 6px; }
|
||||
#lg-tape-track { overflow: hidden; white-space: nowrap; position: relative; height: 24px; }
|
||||
#lg-tape-inner { display: inline-flex; gap: 22px; will-change: transform; }
|
||||
.lg-tape-item { font-family: var(--font-mono); font-size: 11.5px; color: var(--t2); white-space: nowrap; }
|
||||
.lg-tape-item b { color: var(--t1); }
|
||||
.lg-tape-item.err { color: var(--error); }
|
||||
#lg-tape.paused #lg-tape-inner { animation-play-state: paused; }
|
||||
|
||||
/* -------- 右侧发布/审计 -------- */
|
||||
#lg-release table.grid td, #lg-release table.grid th { padding: 5px 6px; font-size: 11px; }
|
||||
#lg-release .toolrow { display: flex; gap: 6px; margin-bottom: 8px; }
|
||||
#lg-release .toolrow input { width: 84px; font-family: var(--font-mono); font-size: 11px; background: var(--bg2); border: 1px solid var(--line); border-radius: var(--r-chip); padding: 4px 6px; color: var(--t1); }
|
||||
.lg-auditline { font-size: 11px; color: var(--t2); padding: 5px 0; border-bottom: 1px solid var(--line-soft); line-height: 1.5; }
|
||||
.lg-auditline:last-child { border-bottom: 0; }
|
||||
.lg-auditline .dim { color: var(--t3); }
|
||||
|
||||
@media (max-width: 1280px) {
|
||||
#lg-grid { grid-template-columns: 1fr; }
|
||||
#lg-side, #lg-side-r { flex-direction: row; flex-wrap: wrap; }
|
||||
.lg-panel { flex: 1 1 260px; }
|
||||
}
|
||||
@media (max-width: 1100px) {
|
||||
#lg-root { padding: var(--sp2) var(--sp3); }
|
||||
table.lg-ledger { font-size: 11px; }
|
||||
.lg-cell { min-width: 46px; font-size: 10px; padding: 2px 5px; }
|
||||
}
|
||||
@@ -1,291 +0,0 @@
|
||||
"use strict";
|
||||
/* ==========================================================================
|
||||
B · 值班台账 Ledger Desk —— 表格本身就是主舞台，每条真实事件一行。
|
||||
响应/汇入/处理/发布/审计五列只填该行真实具备的阶段，没有发生的阶段留空，
|
||||
不做跨行编造关联。底部调用带只滚真实 recent_calls。
|
||||
========================================================================== */
|
||||
(function () {
|
||||
const C = window.Core, S = window.HubShared;
|
||||
const { $, esc, timeShort } = C;
|
||||
let root = null;
|
||||
let unsubs = [];
|
||||
let seenRowKeys = new Set();
|
||||
let mounted = false;
|
||||
let pendingTimers = [];
|
||||
function safeTimeout(fn, ms) {
|
||||
const id = setTimeout(() => { if (mounted) fn(); }, ms);
|
||||
pendingTimers.push(id);
|
||||
return id;
|
||||
}
|
||||
|
||||
const SKELETON = `
|
||||
<div id="lg-root" class="page-shell">
|
||||
<div id="lg-grid">
|
||||
<div id="lg-side">
|
||||
<section class="card lg-panel">
|
||||
<span class="sec-label">来源健康 SOURCES</span>
|
||||
<div id="lg-srcList"></div>
|
||||
</section>
|
||||
<section class="card lg-panel">
|
||||
<span class="sec-label">调度任务 JOBS</span>
|
||||
<div id="lg-jobList"></div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<section class="card" id="lg-ledger">
|
||||
<div class="lg-ledger-hd">
|
||||
<span class="sec-label">事件台账 EVENT LEDGER</span>
|
||||
<span class="hint">每条事件依次经过 响应→汇入→处理→发布→审计</span>
|
||||
</div>
|
||||
<div style="overflow:auto; max-height: 62vh;">
|
||||
<table class="lg-ledger">
|
||||
<thead><tr>
|
||||
<th>时间</th><th>事件</th><th>来源</th>
|
||||
<th class="stage">响应<br/><span style="opacity:.6">RESP</span></th>
|
||||
<th class="stage">汇入<br/><span style="opacity:.6">INGEST</span></th>
|
||||
<th class="stage">处理<br/><span style="opacity:.6">PROC</span></th>
|
||||
<th class="stage">发布<br/><span style="opacity:.6">PUB</span></th>
|
||||
<th class="stage">审计<br/><span style="opacity:.6">AUDIT</span></th>
|
||||
</tr></thead>
|
||||
<tbody id="lg-ledgerBody"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div id="lg-tape">
|
||||
<span class="sec-label">调用带 CALL TAPE</span>
|
||||
<div id="lg-tape-track"><div id="lg-tape-inner"></div></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div id="lg-side-r">
|
||||
<section class="card lg-panel" id="lg-release">
|
||||
<span class="sec-label">盘后发布 RELEASE</span>
|
||||
<div class="toolrow">
|
||||
<input id="lg-rel-date" value="" />
|
||||
<button type="button" class="btn mini" id="lg-rel-load">查看</button>
|
||||
<button type="button" class="btn danger mini" id="lg-rel-backfill">补数</button>
|
||||
</div>
|
||||
<div id="lg-pubTable"></div>
|
||||
</section>
|
||||
<section class="card lg-panel">
|
||||
<span class="sec-label">审计 AUDIT</span>
|
||||
<div id="lg-auditList"></div>
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
function renderSources() {
|
||||
const v = S.sourcesView();
|
||||
const host = $("lg-srcList");
|
||||
if (!v) { host.innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
host.innerHTML = v.live.map((it) => {
|
||||
const health = it.health || {};
|
||||
const link = it._link || C.linkState(health);
|
||||
return `<div class="lg-src-row" data-provider="${esc(it.provider)}">
|
||||
<span class="lamps"><i class="lamp link ${C.linkLampClass(link)}"></i><i class="lamp act" data-act></i></span>
|
||||
<span class="nm">${esc(C.FLOW_LABEL[it.provider] || it.provider)}</span>
|
||||
${C.chipHtml(link, health.state || link)}
|
||||
<span class="lat mono">${health.latency_ms == null ? "-" : health.latency_ms + "ms"}</span>
|
||||
<button class="btn ghost mini" data-probe="${esc(it.provider)}">探测</button>
|
||||
</div>`;
|
||||
}).join("");
|
||||
S.bindProbeButtons(host, () => renderSources());
|
||||
}
|
||||
|
||||
function renderJobs() {
|
||||
const v = S.jobsView();
|
||||
const host = $("lg-jobList");
|
||||
if (!v) { host.innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
host.innerHTML = v.jobs.map((job) => {
|
||||
const latest = v.latestByJob.get(job.id);
|
||||
return `<div class="lg-jobline">
|
||||
${C.chipHtml(latest ? latest.state : "idle")}
|
||||
<span class="nm mono">${esc(job.id)}</span>
|
||||
<span class="at mono">${esc(job.at)}</span>
|
||||
<button class="btn ghost mini" data-run="${esc(job.id)}">触发</button>
|
||||
</div>`;
|
||||
}).join("");
|
||||
S.bindRunButtons(host, () => renderJobs());
|
||||
}
|
||||
|
||||
function renderRelease() {
|
||||
const v = S.releaseView();
|
||||
const dateInput = $("lg-rel-date");
|
||||
if (!dateInput.value) dateInput.value = C.state.releaseDate;
|
||||
if (!v) { $("lg-pubTable").innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
$("lg-pubTable").innerHTML = C.table(["数据集", "批次", "状态", "时间", ""], v.pubs.map((p) => [
|
||||
`<span class="mono k">${esc(p.dataset)}</span>`, `<span class="mono">${esc(p.active_batch)}</span>`,
|
||||
C.chipHtml(p.state), `<span class="mono">${esc(p.published_at)}</span>`,
|
||||
p.prev_batch ? `<button class="btn danger mini" data-rollback="${esc(p.dataset)}">回滚</button>` : "-",
|
||||
]));
|
||||
S.bindRollbackButtons($("lg-pubTable"), () => renderRelease());
|
||||
}
|
||||
|
||||
function renderAuditList() {
|
||||
const v = S.auditView();
|
||||
const host = $("lg-auditList");
|
||||
if (!v) { host.innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
host.innerHTML = v.items.slice(0, 10).map((a) => `
|
||||
<div class="lg-auditline mono">
|
||||
<div>${esc(timeShort(a.created_at))} <b>${esc(a.actor)}</b> <span class="${String(a.action).includes("rollback") ? "warn" : ""}">${esc(a.action)}</span></div>
|
||||
<div class="dim">${esc(a.target)}${a.detail ? " · " + esc(a.detail) : ""}</div>
|
||||
</div>`).join("");
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------- 台账行合成
|
||||
每行只填该事件类型真实具备的阶段；不同类型互不编造对方的字段。 */
|
||||
function cell(text, kind) {
|
||||
if (text == null || text === "") return `<span class="lg-cell dash">—</span>`;
|
||||
return `<span class="lg-cell ${kind || ""}">${esc(text)}</span>`;
|
||||
}
|
||||
|
||||
function buildLedgerRows() {
|
||||
const ov = S.overviewView(), jv = S.jobsView(), rv = S.releaseView(), av = S.auditView();
|
||||
const rows = [];
|
||||
if (ov) {
|
||||
for (const c of ov.recentCalls) {
|
||||
rows.push({
|
||||
key: `call:${c.id}`, t: c.created_at, name: c.endpoint, sub: "来源响应", source: c.provider,
|
||||
resp: cell(timeShort(c.created_at), c.ok ? "ok" : "err"),
|
||||
ingest: c.ok ? cell(timeShort(c.created_at), "ok") : cell(),
|
||||
proc: cell(), pub: cell(), audit: cell(),
|
||||
err: !c.ok,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (jv) {
|
||||
for (const r of jv.runs.slice(0, 30)) {
|
||||
const job = jv.jobs.find((j) => j.id === r.job_id);
|
||||
rows.push({
|
||||
key: `run:${r.id}`, t: r.started_at, name: (job && job.title) || r.job_id, sub: r.job_id, source: "system",
|
||||
resp: cell(), ingest: cell(timeShort(r.started_at), "ok"),
|
||||
proc: r.state === "running" ? cell("running", "warn") : r.state === "failed" ? cell(r.error || "failed", "err") : cell(timeShort(r.finished_at), "ok"),
|
||||
pub: cell(), audit: cell(),
|
||||
err: r.state === "failed",
|
||||
});
|
||||
}
|
||||
}
|
||||
if (rv) {
|
||||
for (const p of rv.pubs) {
|
||||
if (!p.published_at || p.published_at === "-") continue;
|
||||
rows.push({
|
||||
key: `pub:${p.dataset}:${p.active_batch}`, t: p.published_at, name: `批次发布 · ${p.dataset}`, sub: p.active_batch, source: "system",
|
||||
resp: cell(), ingest: cell(), proc: cell(),
|
||||
pub: cell(timeShort(p.published_at), p.state === "published" ? "ok" : "warn"),
|
||||
audit: cell(),
|
||||
err: p.state === "missing" || p.state === "error",
|
||||
});
|
||||
}
|
||||
}
|
||||
if (av) {
|
||||
for (const a of av.items.slice(0, 30)) {
|
||||
const isRollback = String(a.action).includes("rollback");
|
||||
const isProbe = a.action === "probe";
|
||||
rows.push({
|
||||
key: `audit:${a.id}`, t: a.created_at, name: a.action, sub: a.target, source: a.actor,
|
||||
resp: isProbe ? cell(timeShort(a.created_at), String(a.detail).includes("失败") ? "err" : "ok") : cell(),
|
||||
ingest: cell(), proc: cell(),
|
||||
pub: isRollback ? cell(timeShort(a.created_at), "warn") : cell(),
|
||||
audit: cell(timeShort(a.created_at), isRollback ? "warn" : "ok"),
|
||||
err: isRollback,
|
||||
});
|
||||
}
|
||||
}
|
||||
rows.sort((a, b) => String(b.t).localeCompare(String(a.t)));
|
||||
return rows.slice(0, 60);
|
||||
}
|
||||
|
||||
function renderLedger() {
|
||||
const rows = buildLedgerRows();
|
||||
const body = $("lg-ledgerBody");
|
||||
body.innerHTML = rows.map((r) => `
|
||||
<tr class="lg-row ${seenRowKeys.has(r.key) ? "" : "flashnew"} ${r.err ? "err" : ""}" data-key="${esc(r.key)}">
|
||||
<td class="mono">${esc(timeShort(r.t))}</td>
|
||||
<td class="ev"><div class="nm">${esc(r.name)}</div><div class="sub">${esc(r.sub || "")}</div></td>
|
||||
<td class="src mono">${esc(r.source)}</td>
|
||||
<td class="stage">${r.resp}</td>
|
||||
<td class="stage">${r.ingest}</td>
|
||||
<td class="stage">${r.proc}</td>
|
||||
<td class="stage">${r.pub}</td>
|
||||
<td class="stage">${r.audit}</td>
|
||||
</tr>`).join("");
|
||||
rows.forEach((r) => seenRowKeys.add(r.key));
|
||||
if (!C.state.reduced) {
|
||||
body.querySelectorAll("tr.flashnew").forEach((tr) => {
|
||||
setTimeout(() => tr.classList.remove("flashnew"), 1000);
|
||||
});
|
||||
} else {
|
||||
body.querySelectorAll("tr.flashnew").forEach((tr) => tr.classList.remove("flashnew"));
|
||||
}
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------- 调用带：只滚真实 recent_calls */
|
||||
function renderTape() {
|
||||
const ov = S.overviewView();
|
||||
const inner = $("lg-tape-inner");
|
||||
if (!ov || !ov.recentCalls.length) { inner.innerHTML = `<span class="lg-tape-item muted">暂无真实调用</span>`; inner.style.animation = "none"; return; }
|
||||
const items = ov.recentCalls.slice(0, 16).map((c) => `<span class="lg-tape-item ${c.ok ? "" : "err"}">${esc(timeShort(c.created_at))} <b>${esc(c.provider)}</b> ${esc(c.endpoint)} ${c.ok ? "✓" : "× " + esc(c.error)}</span>`).join("");
|
||||
inner.innerHTML = items + items; // 首尾拼接形成无缝循环，内容仍全部来自真实调用
|
||||
if (C.state.reduced) { inner.style.animation = "none"; return; }
|
||||
const width = inner.scrollWidth / 2;
|
||||
inner.style.animation = "none";
|
||||
void inner.offsetWidth;
|
||||
inner.style.setProperty("--tape-w", `-${width}px`);
|
||||
inner.style.animation = `lg-tape-scroll ${Math.max(12, width / 40)}s linear infinite`;
|
||||
}
|
||||
if (!document.getElementById("lg-tape-keyframes")) {
|
||||
const style = document.createElement("style");
|
||||
style.id = "lg-tape-keyframes";
|
||||
style.textContent = `@keyframes lg-tape-scroll { from { transform: translateX(0); } to { transform: translateX(var(--tape-w, -800px)); } }`;
|
||||
document.head.appendChild(style);
|
||||
}
|
||||
|
||||
function updateTapePauseState() {
|
||||
const el = $("lg-tape");
|
||||
if (!el) return;
|
||||
el.classList.toggle("paused", !C.Poller.runtimeAvailable() || C.state.reduced);
|
||||
}
|
||||
|
||||
function refresh() {
|
||||
if (!root) return;
|
||||
renderSources(); renderJobs(); renderRelease(); renderAuditList(); renderLedger(); renderTape(); updateTapePauseState();
|
||||
}
|
||||
|
||||
function onEvent(evt) {
|
||||
if (!root || !mounted) return;
|
||||
if (C.FLOW_PROVIDERS.includes(evt.channel)) {
|
||||
const row = root.querySelector(`.lg-src-row[data-provider="${evt.channel}"] [data-act]`);
|
||||
if (row) {
|
||||
row.classList.remove("on", "err");
|
||||
row.classList.add("on"); if (evt.kind === "error") row.classList.add("err");
|
||||
setTimeout(() => row.classList.remove("on", "err"), 800);
|
||||
}
|
||||
}
|
||||
renderLedger();
|
||||
renderTape();
|
||||
}
|
||||
|
||||
function mount(el) {
|
||||
root = el;
|
||||
mounted = true;
|
||||
seenRowKeys = new Set();
|
||||
root.innerHTML = SKELETON;
|
||||
refresh();
|
||||
S.bindReleaseDateReload(root, "#lg-rel-date", "#lg-rel-load", () => renderRelease());
|
||||
S.bindBackfillButton(root, "#lg-rel-backfill", () => renderRelease());
|
||||
unsubs.push(C.Bus.on("data", refresh));
|
||||
unsubs.push(C.Bus.on("event", onEvent));
|
||||
unsubs.push(C.Bus.on("runtime", updateTapePauseState));
|
||||
unsubs.push(C.Bus.on("reduced", () => { renderTape(); updateTapePauseState(); }));
|
||||
}
|
||||
function unmount() {
|
||||
mounted = false;
|
||||
unsubs.forEach((fn) => fn()); unsubs = [];
|
||||
pendingTimers.forEach(clearTimeout); pendingTimers = [];
|
||||
root = null;
|
||||
}
|
||||
|
||||
window.HUB_LAYOUTS = window.HUB_LAYOUTS || {};
|
||||
window.HUB_LAYOUTS.ledger = { mount, unmount };
|
||||
})();
|
||||
@@ -1,136 +0,0 @@
|
||||
"use strict";
|
||||
/* ==========================================================================
|
||||
三方向共用的数据整形 + 操作绑定。A/B/C 各自的 mount() 只管 DOM 结构和动效编排，
|
||||
"字段从哪来、按钮点了调什么接口" 统一走这里，禁止在布局文件里重写业务判断。
|
||||
========================================================================== */
|
||||
window.HubShared = (function () {
|
||||
const C = window.Core;
|
||||
const { esc, timeShort } = C;
|
||||
|
||||
function overviewView() {
|
||||
const d = C.state.data.overview;
|
||||
if (!d) return null;
|
||||
const eod = d.eod_status || {}, rev = d.revision_status || {};
|
||||
return {
|
||||
tradeDate: d.trade_date,
|
||||
phase: C.PHASE_LABELS[d.session_phase] || d.session_phase,
|
||||
pubCount: d.publications.length,
|
||||
eodState: eod.state, eodLabel: C.EOD_LABELS[eod.state] || eod.state || "-", eod,
|
||||
revState: rev.state, revLabel: C.REV_LABELS[rev.state] || rev.state || "-", rev,
|
||||
anomalies: d.anomalies.length,
|
||||
recentCalls: d.recent_calls || [],
|
||||
sourceCount: d.source_count,
|
||||
};
|
||||
}
|
||||
|
||||
function sourcesView() {
|
||||
const d = C.state.data.sources;
|
||||
if (!d) return null;
|
||||
const live = d.items.filter((it) => C.FLOW_PROVIDERS.includes(it.provider));
|
||||
const reserved = d.items.filter((it) => !C.FLOW_PROVIDERS.includes(it.provider));
|
||||
return { live, reserved, all: d.items };
|
||||
}
|
||||
|
||||
function jobsView() {
|
||||
const d = C.state.data.jobs;
|
||||
if (!d) return null;
|
||||
const runs = d.runs || [];
|
||||
const latestByJob = new Map();
|
||||
for (const r of runs) if (!latestByJob.has(r.job_id)) latestByJob.set(r.job_id, r);
|
||||
return {
|
||||
jobs: d.jobs || [], runs, latest: runs[0], latestByJob,
|
||||
failedRecent: runs.slice(0, 20).filter((r) => r.state === "failed").length,
|
||||
fresh: C.state.freshJobRuns || [],
|
||||
};
|
||||
}
|
||||
|
||||
function releaseView() {
|
||||
const d = C.state.data.batches;
|
||||
if (!d) return null;
|
||||
const pubs = d.publications || [];
|
||||
const batches = d.batches || [];
|
||||
return {
|
||||
tradeDate: d.trade_date, pubs, batches,
|
||||
rollbacks: pubs.filter((p) => p.prev_batch).length,
|
||||
failedBatches: batches.filter((b) => b.state === "failed").length,
|
||||
};
|
||||
}
|
||||
|
||||
function datasetsView() {
|
||||
const d = C.state.data.datasets;
|
||||
if (!d) return null;
|
||||
return { tradeDate: d.trade_date, pubs: d.publications || [], diffs: d.diff_reports || [] };
|
||||
}
|
||||
|
||||
function auditView() {
|
||||
const d = C.state.data.audit;
|
||||
if (!d) return null;
|
||||
return { items: d.items || [] };
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------- 操作绑定 */
|
||||
function bindProbeButtons(root, onDone) {
|
||||
root.querySelectorAll("[data-probe]").forEach((btn) => {
|
||||
btn.addEventListener("click", async () => {
|
||||
const provider = btn.dataset.probe;
|
||||
btn.disabled = true;
|
||||
const original = btn.textContent;
|
||||
btn.textContent = "探测中…";
|
||||
try {
|
||||
const result = await C.probeSource(provider);
|
||||
btn.textContent = "已探测";
|
||||
setTimeout(() => { btn.textContent = original; }, 1200);
|
||||
if (onDone) onDone(result, provider);
|
||||
} catch (err) {
|
||||
alert(err.message);
|
||||
btn.textContent = original;
|
||||
} finally {
|
||||
setTimeout(() => { btn.disabled = false; }, 400);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function bindRunButtons(root, onDone) {
|
||||
root.querySelectorAll("[data-run]").forEach((btn) => {
|
||||
btn.addEventListener("click", async () => {
|
||||
const date = prompt("交易日 YYYYMMDD（可留空=今天）", "") || "";
|
||||
btn.disabled = true;
|
||||
try {
|
||||
const result = await C.runJob(btn.dataset.run, date);
|
||||
if (onDone) onDone(result);
|
||||
} catch (err) {
|
||||
alert(err.message);
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function bindRollbackButtons(root, onDone) {
|
||||
root.querySelectorAll("[data-rollback]").forEach((btn) => {
|
||||
btn.addEventListener("click", () => C.dangerous("rollback", btn.dataset.rollback, onDone));
|
||||
});
|
||||
}
|
||||
|
||||
function bindBackfillButton(root, selector, onDone) {
|
||||
const el = root.querySelector(selector);
|
||||
if (el) el.addEventListener("click", () => C.dangerous("backfill", null, onDone));
|
||||
}
|
||||
|
||||
function bindReleaseDateReload(root, inputSel, btnSel, onDone) {
|
||||
const btn = root.querySelector(btnSel);
|
||||
if (!btn) return;
|
||||
btn.addEventListener("click", async () => {
|
||||
C.state.releaseDate = root.querySelector(inputSel).value.trim();
|
||||
await C.pollBatches();
|
||||
if (onDone) onDone();
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
overviewView, sourcesView, jobsView, releaseView, datasetsView, auditView,
|
||||
bindProbeButtons, bindRunButtons, bindRollbackButtons, bindBackfillButton, bindReleaseDateReload,
|
||||
};
|
||||
})();
|
||||
@@ -1,60 +0,0 @@
|
||||
/* C · 地层剖面 Strata —— 来源层→加工层→发布层→审计沉积层 四条全宽横带。
|
||||
真实事件自上而下贯穿；整齐、纵向追踪；选中/关联状态解决跨层追踪费眼问题。 */
|
||||
#st-root { padding: var(--sp3) var(--sp4); max-width: 1680px; margin: 0 auto; position: relative; }
|
||||
#st-spine { position: absolute; left: 46px; top: 56px; bottom: 12px; width: 2px; background: var(--line); z-index: 0; }
|
||||
#st-dot { position: absolute; left: 41px; width: 12px; height: 12px; border-radius: 50%; background: var(--packet); opacity: 0; z-index: 3; pointer-events: none; box-shadow: 0 0 0 4px var(--bg0); }
|
||||
|
||||
.st-band { position: relative; display: flex; gap: 16px; padding: 14px 16px 16px 62px; margin-bottom: 10px; z-index: 1; border-left: 3px solid transparent; transition: border-color var(--dur-ui) var(--ease-out), background var(--dur-ui) var(--ease-out); }
|
||||
.st-band > div { min-width: 0; }
|
||||
.st-band.pulse { border-left-color: var(--action); background: var(--action-soft); }
|
||||
.st-band.pulse.err { border-left-color: var(--error); background: var(--error-soft); }
|
||||
.st-band .st-idx { position: absolute; left: 6px; top: 14px; width: 36px; text-align: center; }
|
||||
.st-band .st-idx .n { font-family: var(--font-mono); font-size: 18px; font-weight: 700; color: var(--t3); display: block; }
|
||||
.st-band .st-idx .lb { font-size: 9.5px; color: var(--t3); letter-spacing: .12em; display: block; margin-top: 2px; }
|
||||
|
||||
#st-sources { display: grid; grid-template-columns: repeat(4, 1fr) auto; gap: 12px; flex: 1; min-width: 0; }
|
||||
/* 防止内部宽表格用 min-width:auto 撑爆 1fr 轨道，参见 ledger.css 同类注释。 */
|
||||
#st-sources > *, #st-process > *, #st-publish > * { min-width: 0; }
|
||||
.st-src { padding: 8px 10px; cursor: pointer; border: 1px solid var(--line); border-radius: var(--r-card); transition: border-color var(--dur-ui) var(--ease-out); }
|
||||
.st-src.sel { border-color: var(--action); box-shadow: 0 0 0 1px var(--action) inset; }
|
||||
.st-src .row { display: flex; align-items: center; gap: 6px; }
|
||||
.st-src .nm { font-size: 13px; font-weight: 600; color: var(--t1); flex: 1; }
|
||||
.st-src .sub { font-size: 10.5px; color: var(--t3); margin-top: 2px; }
|
||||
.st-src .lat { font-size: 11px; color: var(--t2); }
|
||||
.st-reserved { align-self: center; font-size: 11px; color: var(--t3); white-space: nowrap; padding: 0 8px; }
|
||||
|
||||
#st-process { flex: 1; display: grid; grid-template-columns: 1.3fr 1fr; gap: 14px; }
|
||||
.st-jobs .st-job { display: flex; align-items: center; gap: 8px; padding: 6px 0; border-bottom: 1px solid var(--line-soft); font-size: 12px; cursor: pointer; border-radius: 4px; }
|
||||
.st-jobs .st-job:last-child { border-bottom: 0; }
|
||||
.st-jobs .st-job.sel { background: var(--action-soft); }
|
||||
.st-jobs .st-job .nm { flex: 1; color: var(--t1); }
|
||||
.st-jobs .st-job .at { color: var(--t3); font-size: 11px; }
|
||||
.st-runs table.grid td, .st-runs table.grid th { padding: 4px 6px; font-size: 11px; }
|
||||
.st-runs tr.hi td { background: var(--action-soft); }
|
||||
|
||||
#st-publish { flex: 1; display: grid; grid-template-columns: 220px 1fr; gap: 16px; }
|
||||
.st-gate .bignum { display: flex; align-items: baseline; gap: 8px; }
|
||||
.st-gate .bignum .n { font-size: var(--fs-hero); font-weight: 700; color: var(--t1); }
|
||||
.st-gate .bignum .cap { font-size: 11px; color: var(--t3); }
|
||||
.st-gate .win { font-size: 11px; color: var(--t2); margin-top: 8px; line-height: 1.7; }
|
||||
.st-datasets table.grid td, .st-datasets table.grid th { padding: 4px 6px; font-size: 11px; }
|
||||
.st-datasets tr.hi td { background: var(--action-soft); }
|
||||
|
||||
#st-audit { flex: 1; display: flex; flex-direction: column; gap: 6px; }
|
||||
.st-audit-line { font-size: 11.5px; color: var(--t2); font-family: var(--font-mono); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||
.st-audit-line b { color: var(--t1); }
|
||||
.st-audit-line.warn { color: var(--act); }
|
||||
|
||||
@media (max-width: 1280px) {
|
||||
#st-sources { grid-template-columns: repeat(2, 1fr); }
|
||||
.st-reserved { grid-column: 1 / -1; }
|
||||
#st-process, #st-publish { grid-template-columns: 1fr; }
|
||||
}
|
||||
@media (max-width: 1100px) {
|
||||
#st-root { padding: var(--sp2) var(--sp3); }
|
||||
#st-spine { left: 30px; }
|
||||
#st-dot { left: 25px; }
|
||||
.st-band { padding-left: 46px; gap: 10px; }
|
||||
.st-band .st-idx { width: 26px; }
|
||||
.st-band .st-idx .n { font-size: 14px; }
|
||||
}
|
||||
@@ -1,248 +0,0 @@
|
||||
"use strict";
|
||||
/* ==========================================================================
|
||||
C · 地层剖面 Strata —— 来源层 → 加工层 → 发布层 → 审计沉积层 四条全宽横带。
|
||||
真实事件自上而下贯穿（脊线上的光点从上一层滑到下一层）；点击任一层的项目
|
||||
进入"选中"态，用同一颜色在其它层高亮可关联的真实行，解决跨层追踪费眼问题。
|
||||
========================================================================== */
|
||||
(function () {
|
||||
const C = window.Core, S = window.HubShared;
|
||||
const { $, esc, timeShort } = C;
|
||||
let root = null;
|
||||
let mounted = false;
|
||||
let pendingTimers = [];
|
||||
function safeTimeout(fn, ms) {
|
||||
const id = setTimeout(() => { if (mounted) fn(); }, ms);
|
||||
pendingTimers.push(id);
|
||||
return id;
|
||||
}
|
||||
let unsubs = [];
|
||||
let selection = null; // { type: 'provider'|'job'|'dataset', value }
|
||||
|
||||
const SKELETON = `
|
||||
<div id="st-root" class="page-shell">
|
||||
<div id="st-spine"></div>
|
||||
<div id="st-dot"></div>
|
||||
|
||||
<section class="st-band card" data-band="sources">
|
||||
<div class="st-idx"><span class="n">01</span><span class="lb">SOURCES<br/>来源层</span></div>
|
||||
<div id="st-sources"></div>
|
||||
</section>
|
||||
|
||||
<section class="st-band card" data-band="process">
|
||||
<div class="st-idx"><span class="n">02</span><span class="lb">PROCESS<br/>加工层</span></div>
|
||||
<div id="st-process">
|
||||
<div class="st-jobs">
|
||||
<span class="sec-label">调度任务 JOBS</span>
|
||||
<div id="st-jobList"></div>
|
||||
</div>
|
||||
<div class="st-runs">
|
||||
<span class="sec-label">最近运行 RUNS</span>
|
||||
<div id="st-runsTable"></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="st-band card" data-band="publish">
|
||||
<div class="st-idx"><span class="n">03</span><span class="lb">PUBLISH<br/>发布层</span></div>
|
||||
<div id="st-publish">
|
||||
<div class="st-gate">
|
||||
<span class="sec-label">盘后发布 RELEASE</span>
|
||||
<div class="bignum"><span class="n mono" id="st-pubnum">-</span><span class="cap">今日发布</span></div>
|
||||
<div class="win mono" id="st-pubwin"></div>
|
||||
<div class="toolrow" style="margin-top:8px;display:flex;gap:6px;">
|
||||
<input id="st-rel-date" style="width:88px;font-family:var(--font-mono);font-size:11px;background:var(--bg2);border:1px solid var(--line);border-radius:5px;padding:4px 6px;color:var(--t1)" />
|
||||
<button type="button" class="btn mini" id="st-rel-load">查看</button>
|
||||
<button type="button" class="btn danger mini" id="st-rel-backfill">补数</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="st-datasets">
|
||||
<span class="sec-label">数据集 DATASETS</span>
|
||||
<div id="st-datasetsTable"></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="st-band card" data-band="audit">
|
||||
<div class="st-idx"><span class="n">04</span><span class="lb">AUDIT<br/>审计层</span></div>
|
||||
<div id="st-audit"></div>
|
||||
</section>
|
||||
</div>
|
||||
`;
|
||||
|
||||
function applySelectionClasses() {
|
||||
root.querySelectorAll("[data-sel-provider]").forEach((el) => el.classList.toggle("sel", selection && selection.type === "provider" && el.dataset.selProvider === selection.value));
|
||||
root.querySelectorAll("[data-sel-job]").forEach((el) => el.classList.toggle("sel", selection && selection.type === "job" && el.dataset.selJob === selection.value));
|
||||
root.querySelectorAll("[data-rel-job]").forEach((el) => el.classList.toggle("hi", selection && selection.type === "job" && el.dataset.relJob === selection.value));
|
||||
root.querySelectorAll("[data-rel-dataset]").forEach((el) => el.classList.toggle("hi", selection && selection.type === "dataset" && el.dataset.relDataset === selection.value));
|
||||
}
|
||||
function setSelection(type, value) {
|
||||
selection = (selection && selection.type === type && selection.value === value) ? null : { type, value };
|
||||
applySelectionClasses();
|
||||
}
|
||||
|
||||
function renderSources() {
|
||||
const v = S.sourcesView();
|
||||
const host = $("st-sources");
|
||||
if (!v) { host.innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
host.innerHTML = v.live.map((it) => {
|
||||
const health = it.health || {};
|
||||
const link = it._link || C.linkState(health);
|
||||
return `<div class="st-src card" data-sel-provider="${esc(it.provider)}" data-provider="${esc(it.provider)}">
|
||||
<div class="row">
|
||||
<span class="lamp link ${C.linkLampClass(link)}"></span>
|
||||
<span class="lamp act" data-act></span>
|
||||
<span class="nm">${esc(C.FLOW_LABEL[it.provider] || it.provider)}</span>
|
||||
${C.chipHtml(link, health.state || link)}
|
||||
</div>
|
||||
<div class="sub">${esc(C.FLOW_ROLE[it.provider] || it.role)} · 配置 ${(it.credential || {}).configured ? "已配置" : "未配置"} · ${it.calls_today ?? 0} 次</div>
|
||||
<div class="row" style="margin-top:6px;justify-content:space-between">
|
||||
<span class="lat mono">${health.latency_ms == null ? "-" : health.latency_ms + "ms"}</span>
|
||||
<button class="btn ghost mini" data-probe="${esc(it.provider)}">探测一次</button>
|
||||
</div>
|
||||
</div>`;
|
||||
}).join("") + (v.reserved.length ? `<div class="st-reserved">预留 ${v.reserved.length} 个<br/>未接入</div>` : "");
|
||||
S.bindProbeButtons(host, () => renderSources());
|
||||
host.querySelectorAll(".st-src").forEach((el) => el.addEventListener("click", (e) => {
|
||||
if (e.target.closest("button")) return;
|
||||
setSelection("provider", el.dataset.provider);
|
||||
}));
|
||||
applySelectionClasses();
|
||||
}
|
||||
|
||||
function renderProcess() {
|
||||
const v = S.jobsView();
|
||||
const jobHost = $("st-jobList");
|
||||
if (!v) { jobHost.innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
jobHost.innerHTML = v.jobs.map((job) => {
|
||||
const latest = v.latestByJob.get(job.id);
|
||||
return `<div class="st-job" data-sel-job="${esc(job.id)}" data-job="${esc(job.id)}">
|
||||
${C.chipHtml(latest ? latest.state : "idle")}
|
||||
<span class="nm mono">${esc(job.id)}</span><span class="at mono">${esc(job.at)}</span>
|
||||
<button class="btn ghost mini" data-run="${esc(job.id)}">触发</button>
|
||||
</div>`;
|
||||
}).join("");
|
||||
S.bindRunButtons(jobHost, () => renderProcess());
|
||||
jobHost.querySelectorAll(".st-job").forEach((el) => el.addEventListener("click", (e) => {
|
||||
if (e.target.closest("button")) return;
|
||||
setSelection("job", el.dataset.job);
|
||||
}));
|
||||
$("st-runsTable").innerHTML = C.table(["ID", "任务", "状态", "开始", "结束", "错误"], v.runs.map((r) => [
|
||||
`<span class="mono" data-rel-job="${esc(r.job_id)}">${r.id}</span>`,
|
||||
`<span class="mono k">${esc(r.job_id)}</span>`,
|
||||
`<span class="mono ${r.state === "failed" ? "fail" : r.state === "running" ? "warn" : "ok"}">${esc(r.state)}</span>`,
|
||||
`<span class="mono">${esc(timeShort(r.started_at))}</span>`, `<span class="mono">${esc(timeShort(r.finished_at))}</span>`,
|
||||
`<span class="fail">${esc(r.error || "")}</span>`,
|
||||
]));
|
||||
root.querySelectorAll("#st-runsTable tr").forEach((tr, i) => {
|
||||
if (i === 0) return;
|
||||
const run = v.runs[i - 1]; if (run) tr.dataset.relJob = run.job_id;
|
||||
});
|
||||
applySelectionClasses();
|
||||
}
|
||||
|
||||
function renderPublish() {
|
||||
const v = S.releaseView();
|
||||
const dateInput = $("st-rel-date");
|
||||
if (!dateInput.value) dateInput.value = C.state.releaseDate;
|
||||
if (!v) { $("st-pubnum").textContent = "-"; $("st-datasetsTable").innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
$("st-pubnum").textContent = v.pubs.length;
|
||||
const eod = (S.overviewView() || {}).eod || {};
|
||||
$("st-pubwin").textContent = eod.state === "waiting_upstream" ? `等待上游 · 已试 ${eod.attempts ?? "-"} 次` : "";
|
||||
$("st-datasetsTable").innerHTML = C.table(["数据集", "活跃批次", "上一批次", "状态", "发布时间", "操作"], v.pubs.map((p) => [
|
||||
`<span class="mono k" data-rel-dataset="${esc(p.dataset)}">${esc(p.dataset)}</span>`,
|
||||
`<span class="mono">${esc(p.active_batch)}</span>`, `<span class="mono">${esc(p.prev_batch || "-")}</span>`,
|
||||
C.chipHtml(p.state), `<span class="mono">${esc(p.published_at)}</span>`,
|
||||
p.prev_batch ? `<button class="btn danger mini" data-rollback="${esc(p.dataset)}">回滚</button>` : "-",
|
||||
]));
|
||||
root.querySelectorAll("#st-datasetsTable tr").forEach((tr, i) => {
|
||||
if (i === 0) return;
|
||||
const p = v.pubs[i - 1]; if (p) tr.dataset.relDataset = p.dataset;
|
||||
});
|
||||
S.bindRollbackButtons($("st-datasetsTable"), () => renderPublish());
|
||||
applySelectionClasses();
|
||||
}
|
||||
|
||||
function renderAudit() {
|
||||
const v = S.auditView();
|
||||
const host = $("st-audit");
|
||||
if (!v) { host.innerHTML = `<div class="muted">正在加载…</div>`; return; }
|
||||
host.innerHTML = v.items.slice(0, 8).map((a) => `
|
||||
<div class="st-audit-line ${String(a.action).includes("rollback") ? "warn" : ""}">
|
||||
${esc(timeShort(a.created_at))} <b>${esc(a.actor)}</b> ${esc(a.action)} · ${esc(a.target)}${a.detail ? " · " + esc(a.detail) : ""}
|
||||
</div>`).join("");
|
||||
}
|
||||
|
||||
function refresh() { if (!root) return; renderSources(); renderProcess(); renderPublish(); renderAudit(); }
|
||||
|
||||
/* ---------------------------------------------------------------- 纵向贯穿动效 */
|
||||
const BAND_ORDER = ["sources", "process", "publish", "audit"];
|
||||
function pulseBand(key, err) {
|
||||
if (!root) return;
|
||||
const el = root.querySelector(`.st-band[data-band="${key}"]`);
|
||||
if (!el) return;
|
||||
el.classList.add("pulse"); el.classList.toggle("err", !!err);
|
||||
clearTimeout(el._t);
|
||||
el._t = setTimeout(() => el.classList.remove("pulse", "err"), 700);
|
||||
}
|
||||
function travelSpine(fromKey, toKey) {
|
||||
if (C.state.reduced || !root) return;
|
||||
const dot = $("st-dot");
|
||||
const a = root.querySelector(`.st-band[data-band="${fromKey}"]`);
|
||||
const b = root.querySelector(`.st-band[data-band="${toKey}"]`);
|
||||
if (!a || !b) return;
|
||||
const rootRect = $("st-root").getBoundingClientRect();
|
||||
const ar = a.getBoundingClientRect(), br = b.getBoundingClientRect();
|
||||
const y0 = ar.top - rootRect.top + 20, y1 = br.top - rootRect.top + 20;
|
||||
dot.style.top = y0 + "px"; dot.style.opacity = "0";
|
||||
if (dot._anim) dot._anim.cancel();
|
||||
dot._anim = dot.animate([
|
||||
{ transform: "translateY(0)", opacity: 0 },
|
||||
{ transform: "translateY(0)", opacity: 1, offset: .1 },
|
||||
{ transform: `translateY(${y1 - y0}px)`, opacity: 1, offset: .9 },
|
||||
{ transform: `translateY(${y1 - y0}px)`, opacity: 0 },
|
||||
], { duration: 620, easing: "linear" });
|
||||
}
|
||||
|
||||
function onEvent(evt) {
|
||||
if (!root || !mounted) return;
|
||||
if (C.FLOW_PROVIDERS.includes(evt.channel)) {
|
||||
const act = root.querySelector(`.st-src[data-provider="${evt.channel}"] [data-act]`);
|
||||
if (act) {
|
||||
act.classList.remove("on", "err"); act.classList.add("on"); if (evt.kind === "error") act.classList.add("err");
|
||||
setTimeout(() => act.classList.remove("on", "err"), 800);
|
||||
}
|
||||
pulseBand("sources", evt.kind === "error");
|
||||
safeTimeout(() => { travelSpine("sources", "process"); pulseBand("process"); renderSources(); }, 200);
|
||||
} else if (evt.channel === "junction") {
|
||||
pulseBand("process", evt.kind === "error");
|
||||
safeTimeout(() => { travelSpine("process", "publish"); pulseBand("publish"); renderProcess(); }, 200);
|
||||
} else if (evt.channel === "tx") {
|
||||
pulseBand("publish", evt.kind === "rollback");
|
||||
safeTimeout(() => { travelSpine("publish", "audit"); pulseBand("audit", evt.kind === "rollback"); renderPublish(); }, 200);
|
||||
} else if (evt.channel === "audit") {
|
||||
pulseBand("audit", evt.kind === "rollback");
|
||||
renderAudit();
|
||||
}
|
||||
}
|
||||
|
||||
function mount(el) {
|
||||
root = el;
|
||||
mounted = true;
|
||||
selection = null;
|
||||
root.innerHTML = SKELETON;
|
||||
refresh();
|
||||
S.bindReleaseDateReload(root, "#st-rel-date", "#st-rel-load", () => renderPublish());
|
||||
S.bindBackfillButton(root, "#st-rel-backfill", () => renderPublish());
|
||||
unsubs.push(C.Bus.on("data", refresh));
|
||||
unsubs.push(C.Bus.on("event", onEvent));
|
||||
}
|
||||
function unmount() {
|
||||
mounted = false;
|
||||
unsubs.forEach((fn) => fn()); unsubs = [];
|
||||
pendingTimers.forEach(clearTimeout); pendingTimers = [];
|
||||
root = null;
|
||||
}
|
||||
|
||||
window.HUB_LAYOUTS = window.HUB_LAYOUTS || {};
|
||||
window.HUB_LAYOUTS.strata = { mount, unmount };
|
||||
})();
|
||||
@@ -1,91 +0,0 @@
|
||||
"use strict";
|
||||
/* ==========================================================================
|
||||
xiaobai-datahub 管理后台 · 启动壳
|
||||
会话/登录/改密/登出/主题按钮/退出确认 —— 与具体布局无关，三页共用同一份。
|
||||
========================================================================== */
|
||||
(function () {
|
||||
const C = window.Core;
|
||||
const { $ } = C;
|
||||
|
||||
function show(id) {
|
||||
["login-view", "change-view", "shell"].forEach((key) => { $(key).hidden = key !== id; });
|
||||
}
|
||||
|
||||
async function boot() {
|
||||
try {
|
||||
const session = await C.api("/admin/api/session");
|
||||
C.state.csrf = session.csrf;
|
||||
$("who").textContent = session.username;
|
||||
if (session.must_change) { show("change-view"); return; }
|
||||
show("shell");
|
||||
enterShell();
|
||||
} catch {
|
||||
show("login-view");
|
||||
}
|
||||
}
|
||||
|
||||
$("login-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const form = new FormData(event.target);
|
||||
$("login-error").hidden = true;
|
||||
try {
|
||||
const result = await C.api("/admin/api/login", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ username: form.get("username"), password: form.get("password") }),
|
||||
});
|
||||
C.state.csrf = result.csrf;
|
||||
if (result.must_change) show("change-view");
|
||||
else { show("shell"); enterShell(); }
|
||||
} catch (err) {
|
||||
$("login-error").hidden = false;
|
||||
$("login-error").textContent = err.message;
|
||||
}
|
||||
});
|
||||
|
||||
$("change-form").addEventListener("submit", async (event) => {
|
||||
event.preventDefault();
|
||||
const form = new FormData(event.target);
|
||||
try {
|
||||
await C.api("/admin/api/change-password", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ current: form.get("current"), new_password: form.get("new_password") }),
|
||||
});
|
||||
show("shell");
|
||||
enterShell();
|
||||
} catch (err) {
|
||||
$("change-error").hidden = false;
|
||||
$("change-error").textContent = err.message;
|
||||
}
|
||||
});
|
||||
|
||||
$("logout-btn").addEventListener("click", async () => {
|
||||
await C.api("/admin/api/logout", { method: "POST", body: "{}" });
|
||||
C.Poller.stopAll();
|
||||
Object.values(window.HUB_LAYOUTS || {}).forEach((impl) => { try { impl.unmount && impl.unmount(); } catch { /* ignore */ } });
|
||||
show("login-view");
|
||||
});
|
||||
|
||||
$("theme-btn").addEventListener("click", () => C.toggleTheme());
|
||||
C.Bus.on("theme", (theme) => { $("theme-btn").textContent = theme === "night" ? "日间" : "夜间"; });
|
||||
|
||||
function updateCrumb() {
|
||||
const el = $("crumb");
|
||||
if (!el) return;
|
||||
if (!C.state.online) { el.textContent = "网络已断开 · 已暂停实时"; return; }
|
||||
if (!C.state.visible) { el.textContent = "已切至后台 · 已暂停动效"; return; }
|
||||
const d = C.state.data.overview;
|
||||
el.textContent = d ? `8766 · ${d.trade_date} · 持续运转` : "8766 · 四源汇流 · 持续运转";
|
||||
}
|
||||
C.Bus.on("runtime", updateCrumb);
|
||||
C.Bus.on("data", updateCrumb);
|
||||
|
||||
function enterShell() {
|
||||
C.bootTheme();
|
||||
C.applyReduced(C.REDUCE_MQ.matches);
|
||||
C.Poller.startAll();
|
||||
updateCrumb();
|
||||
C.Router.boot();
|
||||
}
|
||||
|
||||
boot();
|
||||
})();
|
||||
@@ -1,136 +0,0 @@
|
||||
/* xiaobai-datahub 数据中枢后台 —— A/B/C 三方向共用组件层
|
||||
只依赖 tokens.css 的变量；三个布局文件（flowline.css / ledger.css / strata.css）
|
||||
只写各自独有的排布，不得重复定义这里已有的颜色/组件规则。 */
|
||||
* { box-sizing: border-box; }
|
||||
html, body { margin: 0; height: 100%; }
|
||||
body {
|
||||
background: var(--bg0); color: var(--t1); font-family: var(--font-cn);
|
||||
font-size: var(--fs-body); -webkit-font-smoothing: antialiased;
|
||||
}
|
||||
.num, .mono { font-variant-numeric: tabular-nums; font-feature-settings: "tnum"; }
|
||||
.mono { font-family: var(--font-mono); }
|
||||
.muted { color: var(--t3); }
|
||||
.error, .fail { color: var(--error); }
|
||||
.ok { color: var(--alive); }
|
||||
.warn { color: var(--act); }
|
||||
button { font-family: var(--font-cn); cursor: pointer; }
|
||||
|
||||
/* ---------- 区段标签：mono 小字大写，是整套系统的刻度尺 ---------- */
|
||||
.sec-label {
|
||||
font-family: var(--font-mono); font-size: var(--fs-label); letter-spacing: .14em;
|
||||
text-transform: uppercase; color: var(--t3);
|
||||
}
|
||||
|
||||
/* ---------- 卡片：只靠底色阶梯和 1px 发丝线分层 ---------- */
|
||||
.card { background: var(--bg1); border: 1px solid var(--line); border-radius: var(--r-card); }
|
||||
|
||||
/* ---------- 状态 chip ---------- */
|
||||
.chip {
|
||||
display: inline-flex; align-items: center; gap: 5px; font-size: var(--fs-small);
|
||||
line-height: 1; padding: 4px 8px; border-radius: var(--r-chip); border: 1px solid var(--line);
|
||||
color: var(--t2); white-space: nowrap;
|
||||
}
|
||||
.chip::before { content: ""; width: 6px; height: 6px; border-radius: 50%; background: currentColor; flex: none; }
|
||||
.chip.ok { color: var(--alive); background: var(--alive-soft); border-color: transparent; }
|
||||
.chip.err { color: var(--error); background: var(--error-soft); border-color: transparent; }
|
||||
.chip.warn { color: var(--act); background: var(--act-soft); border-color: transparent; }
|
||||
.chip.unknown { color: var(--unknown); }
|
||||
.chip.unconfigured { color: var(--unconfigured); border-style: dashed; background: transparent; }
|
||||
.chip.info { color: var(--action); background: var(--action-soft); border-color: transparent; }
|
||||
|
||||
/* ---------- 双灯：LINK=链路健康（低亮常亮），ACT=真实活动（事件时短闪） ---------- */
|
||||
.lamp { width: 7px; height: 7px; border-radius: 50%; display: inline-block; flex: none; transition: background var(--dur-fast) var(--ease-out); }
|
||||
.lamp.link.on { background: var(--alive); box-shadow: 0 0 6px rgba(60,203,154,.55); }
|
||||
.lamp.link.off { background: var(--bg3); }
|
||||
.lamp.link.err { background: var(--error); box-shadow: 0 0 6px rgba(229,83,75,.55); }
|
||||
.lamp.act { background: var(--bg3); }
|
||||
.lamp.act.on { background: var(--act); box-shadow: 0 0 7px rgba(240,167,60,.8); }
|
||||
.lamp.act.on.err { background: var(--error); box-shadow: 0 0 7px rgba(229,83,75,.8); }
|
||||
.lamp-tag { font-family: var(--font-mono); font-size: 9px; color: var(--t3); letter-spacing: .08em; }
|
||||
|
||||
/* ---------- 按钮：蓝色只给"可操作" ---------- */
|
||||
.btn {
|
||||
font-family: var(--font-cn); font-size: var(--fs-small); color: var(--action);
|
||||
background: var(--action-soft); border: 1px solid transparent; border-radius: var(--r-chip);
|
||||
padding: 5px 12px; cursor: pointer; transition: background var(--dur-fast) var(--ease-out), transform var(--dur-fast) var(--ease-out);
|
||||
}
|
||||
.btn:hover { background: var(--action); color: #fff; }
|
||||
.btn:active { transform: translateY(1px); }
|
||||
.btn:disabled { opacity: .5; cursor: not-allowed; }
|
||||
.btn.ghost { background: transparent; border-color: var(--line); color: var(--t2); }
|
||||
.btn.ghost:hover { border-color: var(--action); color: var(--action); background: transparent; }
|
||||
.btn.ghost.active { border-color: var(--action); color: var(--action); background: var(--action-soft); }
|
||||
.btn.danger { color: var(--error); background: var(--error-soft); }
|
||||
.btn.danger:hover { background: var(--error); color: #fff; }
|
||||
.btn.mini { padding: 3px 8px; font-size: 11px; }
|
||||
|
||||
/* ---------- 表格 ---------- */
|
||||
table.grid { width: 100%; border-collapse: collapse; font-size: var(--fs-small); }
|
||||
table.grid th {
|
||||
text-align: left; font-weight: 500; color: var(--t3); font-size: 11px;
|
||||
padding: 6px 10px; border-bottom: 1px solid var(--line); white-space: nowrap;
|
||||
}
|
||||
table.grid td { padding: 6px 10px; color: var(--t2); border-bottom: 1px solid var(--line-soft); white-space: nowrap; }
|
||||
table.grid td.k { color: var(--t1); }
|
||||
table.grid tr:last-child td { border-bottom: none; }
|
||||
|
||||
/* ---------- 登录 / 改密（三页共用，跟布局无关） ---------- */
|
||||
.auth-panel {
|
||||
max-width: 420px; margin: 12vh auto; padding: 30px 32px; border-radius: 14px;
|
||||
background: var(--glass); border: 1px solid var(--line); backdrop-filter: blur(14px);
|
||||
box-shadow: var(--shadow-pop);
|
||||
}
|
||||
.auth-panel h1 { margin: 0 0 6px; font-size: 21px; }
|
||||
.auth-panel label { display: block; margin: 12px 0; font-size: 13px; color: var(--t2); }
|
||||
.auth-panel input {
|
||||
width: 100%; margin-top: 6px; padding: 9px 11px; border: 1px solid var(--line); border-radius: 8px;
|
||||
background: var(--bg2); color: var(--t1); font-family: var(--font-cn);
|
||||
}
|
||||
.auth-panel input:focus { outline: 2px solid var(--action); outline-offset: 1px; }
|
||||
.auth-panel .badge-sim { display: inline-block; margin-bottom: 10px; font-size: 11px; padding: 3px 10px; border-radius: 999px; letter-spacing: .1em; color: var(--act); border: 1px solid var(--act); opacity: .85; }
|
||||
.auth-panel button[type="submit"] {
|
||||
background: var(--action); color: #fff; border: 0; border-radius: 8px; padding: 8px 16px; font-size: 13px;
|
||||
}
|
||||
|
||||
/* ---------- 顶栏：品牌 + 交易日 + 阶段 + A/B/C 切换 + 全局控制 ---------- */
|
||||
#topbar {
|
||||
height: 48px; display: flex; align-items: center; gap: 12px; padding: 0 16px;
|
||||
border-bottom: 1px solid var(--line); background: var(--bg1); position: sticky; top: 0; z-index: 20;
|
||||
}
|
||||
#topbar .title { font-size: 15px; font-weight: 600; color: var(--t1); white-space: nowrap; }
|
||||
#topbar .title em { font-style: normal; color: var(--action); }
|
||||
#topbar .vdiv { width: 1px; height: 16px; background: var(--line); flex: none; }
|
||||
#topbar .tdate { font-size: 13px; color: var(--t2); white-space: nowrap; }
|
||||
#topbar .spacer { flex: 1; }
|
||||
#topbar .who { font-size: 12px; color: var(--t2); white-space: nowrap; }
|
||||
|
||||
/* A/B/C 切换：一直可见，写清中文名称，当前页状态明确 */
|
||||
#layoutSwitch {
|
||||
display: flex; align-items: center; gap: 2px; padding: 3px; border-radius: 8px;
|
||||
background: var(--bg2); border: 1px solid var(--line); flex: none;
|
||||
}
|
||||
#layoutSwitch button {
|
||||
font-family: var(--font-cn); font-size: 12.5px; color: var(--t2); background: transparent;
|
||||
border: 0; border-radius: 6px; padding: 6px 11px; white-space: nowrap;
|
||||
transition: background var(--dur-ui) var(--ease-out), color var(--dur-ui) var(--ease-out);
|
||||
}
|
||||
#layoutSwitch button .k { font-family: var(--font-mono); font-weight: 700; margin-right: 5px; opacity: .7; }
|
||||
#layoutSwitch button[aria-current="page"] {
|
||||
background: var(--action); color: #fff; box-shadow: var(--shadow-pop);
|
||||
}
|
||||
#layoutSwitch button[aria-current="page"] .k { opacity: 1; }
|
||||
#layoutSwitch button:not([aria-current="page"]):hover { background: var(--bg3); color: var(--t1); }
|
||||
|
||||
/* ---------- 主内容容器 ---------- */
|
||||
#page-root { min-height: calc(100vh - 48px); background: var(--bg0); }
|
||||
.page-shell { padding: var(--sp4); max-width: 1584px; margin: 0 auto; }
|
||||
|
||||
/* 三方向表格容器统一命名以 Table 结尾：窄列/1024 档宁可局部横向滚动，
|
||||
也不可撑破外层 grid/flex 轨道导致整页错位。 */
|
||||
[id$="Table"] { overflow-x: auto; }
|
||||
|
||||
/* ---------- reduced-motion 全局兜底：不留半成品动效 ---------- */
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
* { animation-duration: .001ms !important; animation-iteration-count: 1 !important; transition-duration: .001ms !important; }
|
||||
}
|
||||
html.reduced * { animation-duration: .001ms !important; transition-duration: .001ms !important; }
|
||||
@@ -0,0 +1,51 @@
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--bg: #f4f5f7;
|
||||
--surface: #ffffff;
|
||||
--text: #1f2329;
|
||||
--muted: #646a73;
|
||||
--line: #dee0e3;
|
||||
--action: #3370ff;
|
||||
--danger: #e04536;
|
||||
--ok: #16a34a;
|
||||
--warn: #b45309;
|
||||
--radius: 8px;
|
||||
--pad: 16px;
|
||||
font-family: "Segoe UI", "PingFang SC", "Noto Sans SC", sans-serif;
|
||||
}
|
||||
:root[data-theme="night"] {
|
||||
color-scheme: dark;
|
||||
--bg: #111318;
|
||||
--surface: #1b1e24;
|
||||
--text: #e8eaed;
|
||||
--muted: #9aa0a6;
|
||||
--line: #2a2f38;
|
||||
--action: #5b8cff;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; background: var(--bg); color: var(--text); }
|
||||
.panel, header.top, nav, main { background: var(--surface); }
|
||||
.auth-panel { max-width: 420px; margin: 12vh auto; padding: 28px; border-radius: var(--radius); border: 1px solid var(--line); }
|
||||
label { display: block; margin: 12px 0; }
|
||||
input, select { width: 100%; padding: 8px 10px; border: 1px solid var(--line); border-radius: 4px; background: var(--bg); color: var(--text); }
|
||||
button { background: var(--action); color: #fff; border: 0; border-radius: 4px; padding: 8px 14px; cursor: pointer; }
|
||||
button.ghost { background: transparent; color: var(--text); border: 1px solid var(--line); }
|
||||
button.danger { background: var(--danger); }
|
||||
.muted { color: var(--muted); }
|
||||
.error { color: var(--danger); }
|
||||
.top { display: flex; gap: 12px; align-items: center; padding: 10px var(--pad); border-bottom: 1px solid var(--line); }
|
||||
nav { display: flex; gap: 4px; padding: 8px var(--pad); border-bottom: 1px solid var(--line); }
|
||||
nav button { background: transparent; color: var(--muted); }
|
||||
nav button.active { color: var(--action); background: transparent; font-weight: 600; }
|
||||
main { padding: var(--pad); min-height: calc(100vh - 96px); }
|
||||
.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 12px; margin-bottom: 16px; }
|
||||
.card { border: 1px solid var(--line); border-radius: var(--radius); padding: 12px; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 13px; }
|
||||
th, td { text-align: left; padding: 8px; border-bottom: 1px solid var(--line); vertical-align: top; }
|
||||
.pill { font-size: 12px; padding: 2px 8px; border-radius: 999px; border: 1px solid var(--line); }
|
||||
.ok { color: var(--ok); }
|
||||
.warn { color: var(--warn); }
|
||||
.fail { color: var(--danger); }
|
||||
.toolbar { display: flex; gap: 8px; flex-wrap: wrap; margin: 12px 0; align-items: end; }
|
||||
.toolbar label { margin: 0; }
|
||||
dialog { border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); color: var(--text); padding: 20px; }
|
||||
@@ -1,79 +0,0 @@
|
||||
/* xiaobai-datahub 数据中枢后台 —— 第九版视觉基础 Token（A/B/C 三方向共用，唯一来源）
|
||||
来源：HEL-545 视觉规范 + prototype-A-flowline.html 原稿。
|
||||
原则：精密、克制、成熟。禁止霓虹泛光、粒子、发光描边、蓝图网格、星球/轨道造型、大蓝球/星空。
|
||||
任何视觉改动只改这个文件里的变量，禁止在各布局 CSS 里另起一套颜色/间距。 */
|
||||
:root {
|
||||
color-scheme: light;
|
||||
/* ---- 日间（默认） ---- */
|
||||
--bg0: #F4F6F8;
|
||||
--bg1: #FFFFFF;
|
||||
--bg2: #EEF1F4;
|
||||
--bg3: #E4E9EF;
|
||||
--line: rgba(15, 23, 42, .12);
|
||||
--line-soft: rgba(15, 23, 42, .06);
|
||||
|
||||
--t1: #16202C;
|
||||
--t2: #4C5C6F;
|
||||
--t3: #8593A5;
|
||||
|
||||
--action: #2F6FE4;
|
||||
--action-soft: rgba(47, 111, 228, .10);
|
||||
--alive: #0E9F6E;
|
||||
--alive-soft: rgba(14, 159, 110, .10);
|
||||
--act: #C77F1A;
|
||||
--act-soft: rgba(199, 127, 26, .12);
|
||||
--error: #D0342C;
|
||||
--error-soft: rgba(208, 52, 44, .10);
|
||||
--unknown: #7B8898;
|
||||
--unconfigured: #9AA7B5;
|
||||
|
||||
--packet: #2F6FE4;
|
||||
--shadow-pop: 0 8px 24px rgba(15, 23, 42, .12);
|
||||
--glass: rgba(255, 255, 255, .86);
|
||||
|
||||
--font-cn: "PingFang SC", "Microsoft YaHei", "Noto Sans SC", system-ui, sans-serif;
|
||||
--font-mono: ui-monospace, "SF Mono", "JetBrains Mono", "Cascadia Mono", Menlo, Consolas, monospace;
|
||||
|
||||
--fs-hero: 22px;
|
||||
--fs-title: 15px;
|
||||
--fs-body: 13px;
|
||||
--fs-small: 12px;
|
||||
--fs-label: 10.5px;
|
||||
|
||||
--sp1: 4px; --sp2: 8px; --sp3: 12px; --sp4: 16px; --sp5: 24px; --sp6: 32px;
|
||||
--r-card: 8px; --r-chip: 5px; --r-pill: 999px;
|
||||
|
||||
--ease-out: cubic-bezier(.23, 1, .32, 1);
|
||||
--dur-fast: 130ms;
|
||||
--dur-ui: 220ms;
|
||||
--dur-travel: 900ms;
|
||||
}
|
||||
|
||||
:root[data-theme="night"] {
|
||||
color-scheme: dark;
|
||||
--bg0: #0A0E13;
|
||||
--bg1: #10151C;
|
||||
--bg2: #161D26;
|
||||
--bg3: #1C2530;
|
||||
--line: rgba(148, 163, 184, .16);
|
||||
--line-soft: rgba(148, 163, 184, .08);
|
||||
|
||||
--t1: #E8EDF4;
|
||||
--t2: #97A4B6;
|
||||
--t3: #5E6C7F;
|
||||
|
||||
--action: #4C8DFF;
|
||||
--action-soft: rgba(76, 141, 255, .14);
|
||||
--alive: #3CCB9A;
|
||||
--alive-soft: rgba(60, 203, 154, .13);
|
||||
--act: #F0A73C;
|
||||
--act-soft: rgba(240, 167, 60, .13);
|
||||
--error: #E5534B;
|
||||
--error-soft: rgba(229, 83, 75, .13);
|
||||
--unknown: #8B98A9;
|
||||
--unconfigured: #5A6878;
|
||||
|
||||
--packet: #8FB8FF;
|
||||
--shadow-pop: 0 8px 24px rgba(0, 0, 0, .45);
|
||||
--glass: rgba(13, 18, 28, .82);
|
||||
}
|
||||
@@ -6,9 +6,12 @@ from typing import Any
|
||||
from datahub.adapters import RESERVED
|
||||
from datahub.auth import AuthService
|
||||
from datahub.db import HubDB
|
||||
from datahub import lineage as lineage_module
|
||||
from datahub import observability
|
||||
from datahub.pipeline import OFFICIAL_DATASETS, STOCKS_DATASET, Pipeline
|
||||
from datahub.scheduler import Scheduler
|
||||
from datahub.serving import ApiError
|
||||
from datahub import source_catalog
|
||||
from datahub.timeutil import isoformat, now_shanghai, session_phase, yyyymmdd
|
||||
|
||||
|
||||
@@ -105,6 +108,53 @@ class AdminAPI:
|
||||
raise ApiError("INVALID_ARGUMENT", f"unknown provider: {provider}")
|
||||
return adapter.probe()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# HEL-543: read-only side-channel status/catalog/lineage. These never
|
||||
# change routing, credentials, or adapters; they only read the
|
||||
# provider_call_log/provider_health tables (observability.py) plus the
|
||||
# static registries in source_catalog.py / lineage.py.
|
||||
# ------------------------------------------------------------------
|
||||
def providers_status(self, provider: str = "", limit: int = 50) -> dict[str, Any]:
|
||||
if not observability.is_enabled(self.db):
|
||||
return {"enabled": False, "health": [], "recent_calls": []}
|
||||
limit = max(1, min(int(limit or 50), 200))
|
||||
health_sql = "SELECT * FROM provider_health"
|
||||
params: tuple[Any, ...] = ()
|
||||
if provider:
|
||||
health_sql += " WHERE provider = ?"
|
||||
params = (provider,)
|
||||
health_sql += " ORDER BY provider, interface"
|
||||
health = self.db.fetchall(health_sql, params)
|
||||
calls_sql = "SELECT * FROM provider_call_log"
|
||||
if provider:
|
||||
calls_sql += " WHERE provider = ?"
|
||||
calls_sql += " ORDER BY id DESC LIMIT ?"
|
||||
recent = self.db.fetchall(calls_sql, (*params, limit))
|
||||
return {"enabled": True, "health": health, "recent_calls": recent}
|
||||
|
||||
def source_catalog(self) -> dict[str, Any]:
|
||||
if not observability.is_enabled(self.db):
|
||||
return {"enabled": False, "items": []}
|
||||
return {"enabled": True, "items": source_catalog.snapshot(self.db, self.auth)}
|
||||
|
||||
def lineage(self, trade_date: str = "") -> dict[str, Any]:
|
||||
day = yyyymmdd(trade_date) if trade_date else yyyymmdd(now_shanghai())
|
||||
if not observability.is_enabled(self.db):
|
||||
return {"enabled": False, "trade_date": day, "items": []}
|
||||
return {"enabled": True, "trade_date": day, "items": lineage_module.snapshot(self.db, day)}
|
||||
|
||||
def lineage_affected(self, provider: str = "", interface: str = "") -> dict[str, Any]:
|
||||
if not provider:
|
||||
raise ApiError("INVALID_ARGUMENT", "provider is required")
|
||||
if not observability.is_enabled(self.db):
|
||||
return {"enabled": False, "provider": provider, "interface": interface, "items": []}
|
||||
return {
|
||||
"enabled": True,
|
||||
"provider": provider,
|
||||
"interface": interface,
|
||||
"items": lineage_module.affected(self.db, provider, interface),
|
||||
}
|
||||
|
||||
def jobs(self) -> dict[str, Any]:
|
||||
runs = self.db.fetchall("SELECT * FROM job_runs ORDER BY id DESC LIMIT 100")
|
||||
stocks_times = "/".join(self.pipeline.settings.stocks_refresh_times) or "20:00"
|
||||
|
||||
@@ -8,6 +8,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from datahub.datasets_ext import EXTENDED_DATASET_TABLES, EXTENDED_SCHEMA
|
||||
from datahub.observability import OBS_SCHEMA
|
||||
from datahub.timeutil import isoformat
|
||||
|
||||
_BASE_SCHEMA = """
|
||||
@@ -296,7 +297,7 @@ CREATE INDEX IF NOT EXISTS idx_eod_bars_date ON eod_bars(trade_date, batch_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_calendar_open ON trade_calendar(is_open, cal_date);
|
||||
"""
|
||||
|
||||
SCHEMA = _BASE_SCHEMA + EXTENDED_SCHEMA
|
||||
SCHEMA = _BASE_SCHEMA + EXTENDED_SCHEMA + OBS_SCHEMA
|
||||
|
||||
DATASET_TABLES = {
|
||||
"daily": ("eod_bars", "staging_bars"),
|
||||
|
||||
@@ -122,6 +122,26 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
if path == "/admin/api/sources" and method == "GET":
|
||||
self._json(self.hub.admin.sources(), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/providers/status" and method == "GET":
|
||||
query = parse_query(urlparse(self.path).query)
|
||||
provider = (query.get("provider") or [""])[0]
|
||||
limit = (query.get("limit") or ["50"])[0]
|
||||
self._json(self.hub.admin.providers_status(provider, int(limit or 50)), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/source-catalog" and method == "GET":
|
||||
self._json(self.hub.admin.source_catalog(), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/lineage" and method == "GET":
|
||||
query = parse_query(urlparse(self.path).query)
|
||||
date = (query.get("date") or [""])[0]
|
||||
self._json(self.hub.admin.lineage(date), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/lineage/affected" and method == "GET":
|
||||
query = parse_query(urlparse(self.path).query)
|
||||
provider = (query.get("provider") or [""])[0]
|
||||
interface = (query.get("interface") or [""])[0]
|
||||
self._json(self.hub.admin.lineage_affected(provider, interface), HTTPStatus.OK)
|
||||
return
|
||||
if path.startswith("/admin/api/sources/") and path.endswith("/probe") and method == "POST":
|
||||
provider = path.split("/")[4]
|
||||
self._json(self.hub.admin.probe(provider), HTTPStatus.OK)
|
||||
|
||||
@@ -24,6 +24,12 @@ class Hub:
|
||||
raise SystemExit("DATAHUB_ENCRYPTION_KEY 未配置")
|
||||
self.settings = settings
|
||||
self.db = HubDB(settings.db_path)
|
||||
# HEL-543: carry the observability kill switch on the db handle so
|
||||
# every call site that already threads `db` through (pipeline,
|
||||
# realtime_serve, steward, admin_api) picks it up for free with no
|
||||
# extra plumbing. Missing this attribute (e.g. a bare HubDB built
|
||||
# directly in tests) defaults to enabled — see observability.is_enabled.
|
||||
self.db.observability_enabled = settings.observability_enabled
|
||||
self.vault = SecretVault(settings.encryption_key)
|
||||
self.auth = AuthService(self.db, self.vault, settings.api_token, settings.admin_password)
|
||||
token = settings.tushare_token or self.auth.load_credential("tushare_token")
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Read-only lineage/impact inventory (HEL-543).
|
||||
|
||||
Answers, without changing any routing decision: for a given main-site data
|
||||
item, which datahub dataset backs it, which provider/interface currently
|
||||
serves it (primary and backup), and — when a provider/interface is
|
||||
unhealthy — which datasets and, best-effort, which main-site consumers are
|
||||
affected.
|
||||
|
||||
Every row cites where it was verified so a reviewer does not have to trust
|
||||
a paraphrase:
|
||||
|
||||
- ``v1_endpoint``/``primary_source``/``backup_source`` are taken verbatim
|
||||
from ``datahub/serving.py`` (the ``source=`` string literal passed to
|
||||
``_published_rows``/``_official_meta``) or from the provider/interface
|
||||
pairs wired into ``datahub/realtime_serve.py`` for HEL-543.
|
||||
- ``known_consumers`` lists only call sites this round actually found via
|
||||
code search in the ``xiaobai-review`` website tree (cited as
|
||||
``file:line`` in the comment above each dataset). Anything not backed by
|
||||
a citation is left out rather than guessed; a fuller page-by-page map is
|
||||
tracked separately (HEL-549) and can extend this table later without
|
||||
touching its shape.
|
||||
|
||||
This module never talks to a provider and never mutates anything; it only
|
||||
reads ``provider_health``/``provider_call_log`` (HEL-543) and the existing
|
||||
``publications``/``batches`` tables to attach live status to each row.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
# Verified against backend/features/screener/data_sync.py (calendar,
|
||||
# stock_basic, daily, daily_basic, index_daily-as-benchmark, stk_auction,
|
||||
# moneyflow, ths_hot, dc_hot all called via `self.client.query(...)`) and
|
||||
# backend/features/heaven/market_context.py (stock_basic, index_daily via
|
||||
# `self._tushare_client().query(...)` / `client.query(...)`).
|
||||
DATASETS: list[dict[str, Any]] = [
|
||||
{
|
||||
"dataset": "calendar",
|
||||
"tier": "official",
|
||||
"v1_endpoint": "/v1/calendar",
|
||||
"primary_source": "tushare:trade_cal",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(data_sync.py 交易日历解析)", "问天(交易日推算)"],
|
||||
},
|
||||
{
|
||||
"dataset": "stocks",
|
||||
"tier": "official",
|
||||
"v1_endpoint": "/v1/stocks",
|
||||
"primary_source": "tushare:stock_basic",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(股票主档)", "问天(market_context.py 股票代码/名称解析)"],
|
||||
},
|
||||
{
|
||||
"dataset": "daily",
|
||||
"tier": "official",
|
||||
"v1_endpoint": "/v1/bars/daily",
|
||||
"primary_source": "tushare:daily",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(data_sync.py 日K因子)", "交易复盘/个股详情日K图表"],
|
||||
},
|
||||
{
|
||||
"dataset": "valuation",
|
||||
"tier": "official",
|
||||
"v1_endpoint": "/v1/valuation",
|
||||
"primary_source": "tushare:daily_basic",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(data_sync.py 估值因子)"],
|
||||
},
|
||||
{
|
||||
"dataset": "moneyflow",
|
||||
"tier": "official",
|
||||
"v1_endpoint": "/v1/moneyflow",
|
||||
"primary_source": "tushare:moneyflow",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(data_sync.py 资金流因子)", "个股详情资金流"],
|
||||
},
|
||||
{
|
||||
"dataset": "auction",
|
||||
"tier": "official",
|
||||
"v1_endpoint": "/v1/auction",
|
||||
"primary_source": "tushare:stk_auction",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["智能选股(data_sync.py 竞价快照)", "竞价板块"],
|
||||
},
|
||||
{
|
||||
"dataset": "index_daily",
|
||||
"tier": "official",
|
||||
"v1_endpoint": "/v1/indexes/bars",
|
||||
"primary_source": "tushare:index_daily",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["问天(market_context.py 指数近20日走势)", "智能选股(基准回看)"],
|
||||
},
|
||||
{
|
||||
"dataset": "limit_events",
|
||||
"tier": "official",
|
||||
"v1_endpoint": "/v1/limit-events",
|
||||
"primary_source": "tushare:limit_list_d",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["涨停梯队(历史/盘后视图)"],
|
||||
},
|
||||
{
|
||||
"dataset": "popularity",
|
||||
"tier": "official",
|
||||
"v1_endpoint": "/v1/popularity",
|
||||
"primary_source": "tushare:ths_hot+dc_hot",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["人气榜", "智能选股(data_sync.py 人气因子)"],
|
||||
},
|
||||
{
|
||||
"dataset": "dragon_tiger",
|
||||
"tier": "official",
|
||||
"v1_endpoint": "/v1/dragon-tiger",
|
||||
"primary_source": "tushare:hm_detail",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["龙虎榜"],
|
||||
},
|
||||
{
|
||||
"dataset": "sector_daily",
|
||||
"tier": "official",
|
||||
"v1_endpoint": "/v1/sectors",
|
||||
"primary_source": "tushare:ths_daily+dc_index+sw_daily",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["主题轮动", "板块梯队"],
|
||||
},
|
||||
{
|
||||
"dataset": "quotes_latest",
|
||||
"tier": "provisional",
|
||||
"v1_endpoint": "/v1/quotes/latest",
|
||||
"primary_source": "eastmoney:ulist/clist",
|
||||
"backup_source": "tencent:qt",
|
||||
"known_consumers": ["竞价/股票池盘中价格", "情绪周期盘中快照"],
|
||||
},
|
||||
{
|
||||
"dataset": "index_quotes",
|
||||
"tier": "provisional",
|
||||
"v1_endpoint": "/v1/indexes/quotes",
|
||||
"primary_source": "eastmoney:ulist",
|
||||
"backup_source": "tencent:qt",
|
||||
"known_consumers": ["首页大盘指数条"],
|
||||
},
|
||||
{
|
||||
"dataset": "sectors_quote",
|
||||
"tier": "provisional",
|
||||
"v1_endpoint": "/v1/sectors/quote",
|
||||
"primary_source": "eastmoney:sw",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["主题轮动盘中板块报价"],
|
||||
},
|
||||
{
|
||||
"dataset": "limit_pool",
|
||||
"tier": "provisional",
|
||||
"v1_endpoint": "/v1/limit-pool",
|
||||
"primary_source": "eastmoney:zt_pool",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["涨停梯队盘中视图"],
|
||||
},
|
||||
{
|
||||
"dataset": "intraday_points",
|
||||
"tier": "provisional",
|
||||
"v1_endpoint": "/v1/intraday/points",
|
||||
"primary_source": "eastmoney:trends2",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["个股详情分时图"],
|
||||
},
|
||||
{
|
||||
"dataset": "ifind_wencai",
|
||||
"tier": "licensed",
|
||||
"v1_endpoint": "/v1/query (api_name=ifind_wencai)",
|
||||
"primary_source": "ifind:smart_stock_picking",
|
||||
"backup_source": None,
|
||||
"known_consumers": ["问师(自然语言选股，需 iFinD 凭证)"],
|
||||
},
|
||||
]
|
||||
|
||||
_KNOWN_PROVIDERS_BY_SOURCE_PREFIX = ("tushare", "eastmoney", "tencent", "ifind")
|
||||
|
||||
|
||||
def _providers_for(primary_source: str, backup_source: str | None) -> list[str]:
|
||||
providers: list[str] = []
|
||||
for source in (primary_source, backup_source or ""):
|
||||
for provider in _KNOWN_PROVIDERS_BY_SOURCE_PREFIX:
|
||||
if source.startswith(provider) and provider not in providers:
|
||||
providers.append(provider)
|
||||
return providers
|
||||
|
||||
|
||||
def snapshot(db: Any, trade_date: str = "") -> list[dict[str, Any]]:
|
||||
"""Attach live status to the static lineage table. Read-only; never
|
||||
raises (a per-row status lookup failure just leaves that row's status
|
||||
empty rather than failing the whole snapshot)."""
|
||||
result: list[dict[str, Any]] = []
|
||||
for entry in DATASETS:
|
||||
row = dict(entry)
|
||||
providers = _providers_for(entry["primary_source"], entry.get("backup_source"))
|
||||
row["providers"] = providers
|
||||
live: list[dict[str, Any]] = []
|
||||
try:
|
||||
if db is not None and providers:
|
||||
placeholders = ",".join("?" for _ in providers)
|
||||
live = db.fetchall(
|
||||
f"SELECT provider, interface, state, last_error, last_fallback_reason, "
|
||||
f"consec_failures, updated_at FROM provider_health WHERE provider IN ({placeholders})",
|
||||
tuple(providers),
|
||||
)
|
||||
except Exception:
|
||||
live = []
|
||||
row["live_provider_health"] = live
|
||||
if entry["tier"] == "official":
|
||||
pub = None
|
||||
try:
|
||||
if db is not None and trade_date:
|
||||
pub = db.fetchone(
|
||||
"SELECT dataset, trade_date, state, published_at FROM publications "
|
||||
"WHERE dataset = ? AND trade_date = ?",
|
||||
(entry["dataset"], trade_date),
|
||||
)
|
||||
except Exception:
|
||||
pub = None
|
||||
row["publication"] = pub
|
||||
result.append(row)
|
||||
return result
|
||||
|
||||
|
||||
def affected(db: Any, provider: str = "", interface: str = "") -> list[dict[str, Any]]:
|
||||
"""Read-only: which datasets/pages are impacted by a given provider (and,
|
||||
optionally, a specific interface) right now. Does not change routing."""
|
||||
provider = str(provider or "").strip()
|
||||
interface = str(interface or "").strip()
|
||||
result: list[dict[str, Any]] = []
|
||||
for entry in DATASETS:
|
||||
providers = _providers_for(entry["primary_source"], entry.get("backup_source"))
|
||||
if provider and provider not in providers:
|
||||
continue
|
||||
row = dict(entry)
|
||||
row["providers"] = providers
|
||||
health: list[dict[str, Any]] = []
|
||||
try:
|
||||
if db is not None:
|
||||
if interface:
|
||||
health = db.fetchall(
|
||||
"SELECT provider, interface, state, last_error, last_fallback_reason, "
|
||||
"consec_failures, updated_at FROM provider_health "
|
||||
"WHERE provider = ? AND interface = ?",
|
||||
(provider, interface),
|
||||
)
|
||||
elif providers:
|
||||
placeholders = ",".join("?" for _ in providers)
|
||||
health = db.fetchall(
|
||||
f"SELECT provider, interface, state, last_error, last_fallback_reason, "
|
||||
f"consec_failures, updated_at FROM provider_health WHERE provider IN ({placeholders})",
|
||||
tuple(providers),
|
||||
)
|
||||
except Exception:
|
||||
health = []
|
||||
row["live_provider_health"] = health
|
||||
result.append(row)
|
||||
return result
|
||||
@@ -0,0 +1,319 @@
|
||||
"""Side-channel provider-call observability (HEL-543).
|
||||
|
||||
This module is additive-only and must never change what any existing call
|
||||
returns or raises. It exists purely to answer, after the fact and without
|
||||
touching routing: which provider/interface was called, whether it
|
||||
succeeded, how stale/complete the payload looked, and why a fallback fired.
|
||||
|
||||
Hard rules enforced here:
|
||||
|
||||
- Every public entry point (`observe`, `record_call`) is wrapped so that a
|
||||
database failure, a classifier bug, or any other internal error is
|
||||
swallowed and logged at DEBUG level. It never raises into the caller and
|
||||
never delays/blocks the caller's real data path beyond a best-effort
|
||||
timing measurement.
|
||||
- `observe()` always returns exactly what `fn()` returned, and re-raises
|
||||
exactly what `fn()` raised (same exception object, unmodified). It does
|
||||
not retry, does not change ordering, and does not add new failure modes.
|
||||
- No mock data is ever produced or returned by this module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from typing import Any, Callable, TypeVar
|
||||
|
||||
from datahub.timeutil import isoformat, now_shanghai
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
# Additive-only schema: two new tables, no changes to any existing table.
|
||||
# Merged into datahub.db.SCHEMA the same way EXTENDED_SCHEMA is.
|
||||
OBS_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS provider_call_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
provider TEXT NOT NULL,
|
||||
interface TEXT NOT NULL,
|
||||
fetched_at TEXT NOT NULL,
|
||||
latency_ms INTEGER,
|
||||
status TEXT NOT NULL,
|
||||
error TEXT,
|
||||
fallback_reason TEXT,
|
||||
data_age_seconds INTEGER,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS provider_health (
|
||||
provider TEXT NOT NULL,
|
||||
interface TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
last_ok_at TEXT,
|
||||
last_error TEXT,
|
||||
last_fallback_reason TEXT,
|
||||
consec_failures INTEGER NOT NULL DEFAULT 0,
|
||||
last_latency_ms INTEGER,
|
||||
last_data_age_seconds INTEGER,
|
||||
updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY (provider, interface)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_provider_call_log_created ON provider_call_log(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_provider_call_log_provider ON provider_call_log(provider, interface, created_at);
|
||||
"""
|
||||
|
||||
DEFAULT_STALE_SECONDS = 300
|
||||
|
||||
_BLOCKED_MARKERS = (
|
||||
"<!doctype", "<html", "expecting value", "verify you are human",
|
||||
"unusual traffic", "captcha", "安全验证", "访问异常", "请完成验证",
|
||||
"拦截", "禁止访问", "forbidden",
|
||||
)
|
||||
|
||||
# Matches provider messages like "Eastmoney returned 0/3 indices" or
|
||||
# "Tencent returned 2/3 indices" (see adapters/eastmoney.py, adapters/tencent.py).
|
||||
_COUNT_MISMATCH_RE = re.compile(r"returned (\d+)\s*/\s*(\d+)")
|
||||
|
||||
|
||||
def _logger():
|
||||
from datahub.logutil import get_logger
|
||||
|
||||
return get_logger()
|
||||
|
||||
|
||||
def is_enabled(db: Any) -> bool:
|
||||
"""Runtime kill switch (``Settings.observability_enabled`` /
|
||||
``DATAHUB_OBSERVABILITY``, wired onto the db handle in ``Hub.__init__``).
|
||||
|
||||
Defaults to enabled when the attribute is absent — e.g. a bare ``HubDB``
|
||||
built directly in a test, or any call site that predates HEL-543 — so
|
||||
this can never accidentally disable an existing deployment. Never
|
||||
raises.
|
||||
"""
|
||||
try:
|
||||
return bool(getattr(db, "observability_enabled", True))
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
return True
|
||||
|
||||
|
||||
def classify_error(message: str) -> tuple[str, str]:
|
||||
"""Best-effort, side-reading classification of an exception message.
|
||||
|
||||
Never raises. Unknown shapes fall back to a generic ``error`` status so a
|
||||
classifier miss can never be mistaken for a healthy call.
|
||||
"""
|
||||
try:
|
||||
lower = (message or "").lower()
|
||||
if any(marker in lower for marker in _BLOCKED_MARKERS):
|
||||
return "blocked", "response_looks_like_intercept_page"
|
||||
if "timeout" in lower or "timed out" in lower:
|
||||
return "timeout", "request_timeout"
|
||||
mismatch = _COUNT_MISMATCH_RE.search(lower)
|
||||
if mismatch and int(mismatch.group(1)) == 0:
|
||||
return "empty", "empty_or_incomplete_response"
|
||||
if mismatch:
|
||||
return "degraded", "partial_or_mismatched_response"
|
||||
if "empty" in lower or "no intraday chart data" in lower or "missing" in lower:
|
||||
return "empty", "empty_or_incomplete_response"
|
||||
if "too small" in lower or "incomplete" in lower or "mismatch" in lower:
|
||||
return "degraded", "partial_or_mismatched_response"
|
||||
return "error", ""
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
return "error", ""
|
||||
|
||||
|
||||
def classify_rows(
|
||||
rows: Any,
|
||||
*,
|
||||
required_fields: tuple[str, ...] | None = None,
|
||||
freshness_field: str | None = "quote_time_epoch",
|
||||
max_age_seconds: int = DEFAULT_STALE_SECONDS,
|
||||
) -> tuple[str, str, int | None]:
|
||||
"""Read-only classification of an already-successful payload.
|
||||
|
||||
Only ever called on a value a caller is about to use as-is; this never
|
||||
mutates ``rows`` and a classifier bug always degrades to ``("ok", "",
|
||||
None)`` rather than mislabeling a real success as a failure.
|
||||
"""
|
||||
try:
|
||||
if isinstance(rows, dict):
|
||||
items = [rows] if rows else []
|
||||
elif isinstance(rows, (list, tuple)):
|
||||
items = [item for item in rows if isinstance(item, dict)]
|
||||
else:
|
||||
items = []
|
||||
if not items:
|
||||
return "empty", "no_rows_returned", None
|
||||
if required_fields:
|
||||
missing: set[str] = set()
|
||||
for item in items:
|
||||
for field in required_fields:
|
||||
if item.get(field) in (None, ""):
|
||||
missing.add(field)
|
||||
if missing:
|
||||
return "missing_fields", "missing:" + ",".join(sorted(missing)), None
|
||||
data_age: int | None = None
|
||||
if freshness_field:
|
||||
now_epoch = time.time()
|
||||
ages: list[int] = []
|
||||
for item in items:
|
||||
raw = item.get(freshness_field)
|
||||
try:
|
||||
epoch = int(raw or 0)
|
||||
except (TypeError, ValueError):
|
||||
epoch = 0
|
||||
if epoch > 0:
|
||||
ages.append(max(0, int(now_epoch - epoch)))
|
||||
if ages:
|
||||
data_age = max(ages)
|
||||
if data_age > max_age_seconds:
|
||||
return "stale", "data_age_exceeds_threshold", data_age
|
||||
return "ok", "", data_age
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
return "ok", "", None
|
||||
|
||||
|
||||
def record_call(
|
||||
db: Any,
|
||||
provider: str,
|
||||
interface: str,
|
||||
*,
|
||||
status: str,
|
||||
latency_ms: int | None = None,
|
||||
error: str = "",
|
||||
fallback_reason: str = "",
|
||||
data_age_seconds: int | None = None,
|
||||
) -> None:
|
||||
"""Fail-open recorder. Never raises; a write failure here must never be
|
||||
able to take down a real, otherwise-successful data path."""
|
||||
if db is None or not is_enabled(db):
|
||||
return
|
||||
try:
|
||||
now = isoformat(now_shanghai())
|
||||
ok = status == "ok"
|
||||
error_text = (error or "")[:500]
|
||||
reason_text = (fallback_reason or "")[:200]
|
||||
with db.write() as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO provider_call_log("
|
||||
"provider, interface, fetched_at, latency_ms, status, error, "
|
||||
"fallback_reason, data_age_seconds, created_at) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
(provider, interface, now, latency_ms, status, error_text, reason_text, data_age_seconds, now),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO provider_health(
|
||||
provider, interface, state, last_ok_at, last_error, last_fallback_reason,
|
||||
consec_failures, last_latency_ms, last_data_age_seconds, updated_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(provider, interface) DO UPDATE SET
|
||||
state = excluded.state,
|
||||
last_ok_at = CASE WHEN excluded.state = 'ok' THEN excluded.last_ok_at ELSE provider_health.last_ok_at END,
|
||||
last_error = CASE WHEN excluded.state = 'ok' THEN '' ELSE excluded.last_error END,
|
||||
last_fallback_reason = excluded.last_fallback_reason,
|
||||
consec_failures = CASE WHEN excluded.state = 'ok' THEN 0 ELSE provider_health.consec_failures + 1 END,
|
||||
last_latency_ms = excluded.last_latency_ms,
|
||||
last_data_age_seconds = excluded.last_data_age_seconds,
|
||||
updated_at = excluded.updated_at
|
||||
""",
|
||||
(
|
||||
provider,
|
||||
interface,
|
||||
status,
|
||||
now if ok else None,
|
||||
"" if ok else (error_text or reason_text or "unknown_error"),
|
||||
reason_text,
|
||||
0 if ok else 1,
|
||||
latency_ms,
|
||||
data_age_seconds,
|
||||
now,
|
||||
),
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
try:
|
||||
_logger().debug(
|
||||
"observability record_call failed (fail-open)",
|
||||
extra={"hub": {"provider": provider, "interface": interface}},
|
||||
exc_info=True,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _safe_record_failure(db: Any, provider: str, interface: str, latency_ms: int, exc: BaseException) -> None:
|
||||
if db is None:
|
||||
return
|
||||
try:
|
||||
message = str(exc)
|
||||
status, reason = classify_error(message)
|
||||
record_call(
|
||||
db, provider, interface,
|
||||
status=status, latency_ms=latency_ms, error=message, fallback_reason=reason,
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
pass
|
||||
|
||||
|
||||
def _safe_record_success(
|
||||
db: Any,
|
||||
provider: str,
|
||||
interface: str,
|
||||
latency_ms: int,
|
||||
result: Any,
|
||||
classify: Callable[[Any], tuple[str, str, int | None] | None] | None,
|
||||
) -> None:
|
||||
if db is None:
|
||||
return
|
||||
status, reason, data_age = "ok", "", None
|
||||
if classify is not None:
|
||||
try:
|
||||
classified = classify(result)
|
||||
if classified:
|
||||
status, reason, data_age = classified
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
# A classifier bug must never mislabel (or hide) a real success;
|
||||
# degrade to a plain "ok" call rather than skipping the log.
|
||||
status, reason, data_age = "ok", "", None
|
||||
try:
|
||||
record_call(
|
||||
db, provider, interface,
|
||||
status=status, latency_ms=latency_ms, fallback_reason=reason, data_age_seconds=data_age,
|
||||
)
|
||||
except Exception: # pragma: no cover - defensive, must never raise
|
||||
pass
|
||||
|
||||
|
||||
def observe(
|
||||
db: Any,
|
||||
provider: str,
|
||||
interface: str,
|
||||
fn: Callable[[], _T],
|
||||
*,
|
||||
classify: Callable[[Any], tuple[str, str, int | None] | None] | None = None,
|
||||
) -> _T:
|
||||
"""Call ``fn()`` and record a side-channel status row.
|
||||
|
||||
Returns exactly what ``fn()`` returns and re-raises exactly what
|
||||
``fn()`` raises. ``db`` may be ``None`` (e.g. in call sites that are not
|
||||
wired to a database yet); in that case this is a transparent passthrough
|
||||
with no recording at all. Same when the ``DATAHUB_OBSERVABILITY`` kill
|
||||
switch is off (see ``is_enabled``): this becomes ``return fn()`` with no
|
||||
timing, no classification, and no db access whatsoever.
|
||||
"""
|
||||
if db is not None and not is_enabled(db):
|
||||
return fn()
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
result = fn()
|
||||
except Exception:
|
||||
latency_ms = round((time.perf_counter() - started) * 1000)
|
||||
import sys
|
||||
|
||||
exc = sys.exc_info()[1]
|
||||
if exc is not None:
|
||||
_safe_record_failure(db, provider, interface, latency_ms, exc)
|
||||
raise
|
||||
latency_ms = round((time.perf_counter() - started) * 1000)
|
||||
_safe_record_success(db, provider, interface, latency_ms, result, classify)
|
||||
return result
|
||||
@@ -20,6 +20,7 @@ from datahub.governance.ratelimit import TokenBucket
|
||||
from datahub.governance.retry import RetryError, retry_call
|
||||
from datahub.logutil import get_logger
|
||||
from datahub.normalize import finite_number, normalize_daily
|
||||
from datahub import observability
|
||||
from datahub.revision import (
|
||||
compare_fields,
|
||||
diff_published_vs_upstream,
|
||||
@@ -1634,6 +1635,7 @@ class Pipeline:
|
||||
deleted += cur.rowcount
|
||||
connection.execute("DELETE FROM job_runs WHERE started_at < ?", (cutoff_jobs,))
|
||||
connection.execute("DELETE FROM src_calls WHERE created_at < ?", (cutoff_jobs,))
|
||||
connection.execute("DELETE FROM provider_call_log WHERE created_at < ?", (cutoff_jobs,))
|
||||
return {"staging_deleted": deleted}
|
||||
|
||||
def audit(self, actor: str, action: str, target: str = "", detail: str = "") -> None:
|
||||
@@ -1768,6 +1770,18 @@ class Pipeline:
|
||||
"INSERT INTO src_calls(provider, endpoint, ok, latency_ms, error, created_at) VALUES (?,?,?,?,?,?)",
|
||||
("tushare", endpoint, 1 if ok else 0, latency_ms, error, isoformat(self.clock())),
|
||||
)
|
||||
# HEL-543 side channel: unified cross-provider call log/health. Kept
|
||||
# strictly additive and fail-open; the src_calls insert above (the
|
||||
# existing, already-compatible Tushare record) is unaffected either
|
||||
# way.
|
||||
if ok:
|
||||
status, reason = "ok", ""
|
||||
else:
|
||||
status, reason = observability.classify_error(error)
|
||||
observability.record_call(
|
||||
self.db, "tushare", endpoint,
|
||||
status=status, latency_ms=latency_ms, error=error, fallback_reason=reason,
|
||||
)
|
||||
|
||||
def _persist_health(self, state: str, error: str = "") -> None:
|
||||
snap = self.breaker.snapshot()
|
||||
|
||||
@@ -18,6 +18,7 @@ from datahub.adapters.tencent import TencentAdapter
|
||||
from datahub.codes import resolve_code
|
||||
from datahub.db import HubDB
|
||||
from datahub.governance.lkg import LastKnownGood
|
||||
from datahub import observability
|
||||
from datahub.timeutil import isoformat, now_shanghai, yyyymmdd
|
||||
|
||||
QUOTE_TTL = 60
|
||||
@@ -25,6 +26,23 @@ INDEX_TTL = 60
|
||||
INTRADAY_TTL = 20
|
||||
QUOTE_BATCH = 60
|
||||
|
||||
# HEL-543 side-channel classifiers. These only *read* an already-successful
|
||||
# payload to decide what to log; they never change the payload itself and a
|
||||
# classifier exception always degrades to "ok" (see observability.classify_rows).
|
||||
|
||||
|
||||
def _classify_quote_rows(rows: Any) -> tuple[str, str, int | None]:
|
||||
return observability.classify_rows(rows, freshness_field="quote_time_epoch")
|
||||
|
||||
|
||||
def _classify_rows_no_freshness(rows: Any) -> tuple[str, str, int | None]:
|
||||
return observability.classify_rows(rows, freshness_field=None)
|
||||
|
||||
|
||||
def _classify_intraday_payload(data: Any) -> tuple[str, str, int | None]:
|
||||
points = data.get("points") if isinstance(data, dict) else None
|
||||
return observability.classify_rows(points or [], freshness_field=None)
|
||||
|
||||
|
||||
class RealtimeApiError(RuntimeError):
|
||||
def __init__(self, code: str, message: str) -> None:
|
||||
@@ -44,12 +62,17 @@ def fetch_index_quotes(db: HubDB) -> dict[str, Any]:
|
||||
cached = _read_cache(db, cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
eastmoney = EastmoneyAdapter()
|
||||
try:
|
||||
rows = eastmoney.fetch_indices()
|
||||
rows = observability.observe(
|
||||
db, "eastmoney", "indices", lambda: EastmoneyAdapter().fetch_indices(),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
source = "eastmoney:ulist"
|
||||
except Exception:
|
||||
rows = TencentAdapter().fetch_indices()
|
||||
rows = observability.observe(
|
||||
db, "tencent", "indices", lambda: TencentAdapter().fetch_indices(),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
source = "tencent:qt"
|
||||
if len(rows) < 3:
|
||||
raise RealtimeApiError("SOURCE_UNAVAILABLE", "index quotes incomplete")
|
||||
@@ -77,7 +100,10 @@ def fetch_market_quotes(db: HubDB) -> dict[str, Any]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
source = ""
|
||||
try:
|
||||
rows = EastmoneyAdapter().fetch_market_quotes()
|
||||
rows = observability.observe(
|
||||
db, "eastmoney", "market_quotes", lambda: EastmoneyAdapter().fetch_market_quotes(),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
source = "eastmoney:clist"
|
||||
except Exception as exc:
|
||||
errors.append(f"eastmoney:{exc}")
|
||||
@@ -85,7 +111,10 @@ def fetch_market_quotes(db: HubDB) -> dict[str, Any]:
|
||||
listed = _listed_ts_codes(db)
|
||||
if not listed:
|
||||
raise AdapterError("no local stock master for tencent market snapshot")
|
||||
rows = _tencent_named_quotes(listed)
|
||||
rows = observability.observe(
|
||||
db, "tencent", "market_quotes_fallback", lambda: _tencent_named_quotes(listed),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
if len(rows) < 200:
|
||||
raise AdapterError(f"Tencent market snapshot too small: {len(rows)}")
|
||||
source = "tencent:qt"
|
||||
@@ -130,7 +159,10 @@ def fetch_quotes(db: HubDB, codes: list[str]) -> dict[str, Any]:
|
||||
|
||||
missing = [code for code in resolved if code not in by_code]
|
||||
try:
|
||||
rows = _eastmoney_named_quotes(missing)
|
||||
rows = observability.observe(
|
||||
db, "eastmoney", "named_quotes", lambda: _eastmoney_named_quotes(missing),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
by_code.update(_quote_map(rows, missing))
|
||||
if rows:
|
||||
sources.append("eastmoney:ulist")
|
||||
@@ -140,7 +172,10 @@ def fetch_quotes(db: HubDB, codes: list[str]) -> dict[str, Any]:
|
||||
missing = [code for code in resolved if code not in by_code]
|
||||
if missing:
|
||||
try:
|
||||
rows = _tencent_named_quotes(missing)
|
||||
rows = observability.observe(
|
||||
db, "tencent", "named_quotes", lambda: _tencent_named_quotes(missing),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
by_code.update(_quote_map(rows, missing))
|
||||
if rows:
|
||||
sources.append("tencent:qt")
|
||||
@@ -203,7 +238,10 @@ def fetch_sector_quote(db: HubDB, code: str, expected_date: str = "") -> dict[st
|
||||
return cached
|
||||
errors: list[str] = []
|
||||
try:
|
||||
row = EastmoneyAdapter().fetch_shenwan_quote(ts_code)
|
||||
row = observability.observe(
|
||||
db, "eastmoney", "sector_quote", lambda: EastmoneyAdapter().fetch_shenwan_quote(ts_code),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
if not _sector_row_matches(row, canonical_name):
|
||||
raise AdapterError(
|
||||
f"industry name mismatch: expected {canonical_name}, got {row.get('name') or '--'}"
|
||||
@@ -270,7 +308,10 @@ def fetch_limit_pool(db: HubDB, trade_date: str = "") -> dict[str, Any]:
|
||||
if cached is not None:
|
||||
return cached
|
||||
try:
|
||||
rows = EastmoneyAdapter().fetch_limit_pool(day)
|
||||
rows = observability.observe(
|
||||
db, "eastmoney", "limit_pool", lambda: EastmoneyAdapter().fetch_limit_pool(day),
|
||||
classify=_classify_rows_no_freshness,
|
||||
)
|
||||
source = "eastmoney:zt_pool"
|
||||
except Exception as exc:
|
||||
recovered = _load_quotes_lkg(db, cache_key)
|
||||
@@ -530,7 +571,10 @@ def warm_realtime(db: HubDB) -> dict[str, Any]:
|
||||
if master_code and master_name:
|
||||
canonical_names.setdefault(master_code, master_name)
|
||||
codes = list(canonical_names)
|
||||
fetched_sector_rows = _eastmoney_sector_quotes(codes)
|
||||
fetched_sector_rows = observability.observe(
|
||||
db, "eastmoney", "sector_quotes_batch", lambda: _eastmoney_sector_quotes(codes),
|
||||
classify=_classify_quote_rows,
|
||||
)
|
||||
for row in fetched_sector_rows:
|
||||
if _row_quote_date(row, today) != today:
|
||||
continue
|
||||
@@ -588,7 +632,10 @@ def fetch_intraday(db: HubDB, code: str, date: str = "") -> dict[str, Any]:
|
||||
return cached
|
||||
adapter = EastmoneyAdapter()
|
||||
try:
|
||||
payload_data = adapter.fetch_intraday(ts_code, date)
|
||||
payload_data = observability.observe(
|
||||
db, "eastmoney", "intraday", lambda: adapter.fetch_intraday(ts_code, date),
|
||||
classify=_classify_intraday_payload,
|
||||
)
|
||||
source = "eastmoney:trends2"
|
||||
except Exception as exc:
|
||||
recovered = _load_intraday_lkg(db, ts_code, date)
|
||||
|
||||
@@ -33,6 +33,11 @@ class Settings:
|
||||
quality: dict[str, Any] = field(default_factory=dict)
|
||||
log_level: str = "INFO"
|
||||
scheduler_enabled: bool = True
|
||||
# HEL-543 kill switch: off disables the provider_call_log/provider_health
|
||||
# side channel entirely (observe()/record_call() become no-ops and the
|
||||
# new read-only admin endpoints report {"enabled": false}). Default on;
|
||||
# existing routing/fetch/publish behavior is identical either way.
|
||||
observability_enabled: bool = True
|
||||
|
||||
@property
|
||||
def tushare_rate_per_minute(self) -> int:
|
||||
@@ -126,4 +131,5 @@ def load_settings(
|
||||
quality=_load_quality(quality_path),
|
||||
log_level=environ.get("DATAHUB_LOG_LEVEL") or "INFO",
|
||||
scheduler_enabled=str(environ.get("DATAHUB_SCHEDULER") or "1") not in {"0", "false", "False"},
|
||||
observability_enabled=str(environ.get("DATAHUB_OBSERVABILITY") or "1") not in {"0", "false", "False", "off", "OFF"},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
"""Minimal, read-only source directory (HEL-543).
|
||||
|
||||
Registers what already exists: providers, their concrete interfaces, what
|
||||
capability/dataset each interface serves, and whether the provider plays a
|
||||
primary or backup role. This module only *describes* the current adapters
|
||||
and datasets already wired in `datahub/hub.py`, `datahub/serving.py`, and
|
||||
`datahub/realtime_serve.py`; it does not add a way to configure or add a new
|
||||
source without code, and it never changes routing, retries, or fallback
|
||||
order.
|
||||
|
||||
Every ``interfaces`` entry below is a docs-as-code mirror of a real call
|
||||
site, cross-referenced in comments so a reviewer can verify each row is
|
||||
accurate rather than aspirational:
|
||||
|
||||
- tushare interfaces mirror ``datahub/serving.py``'s ``_official_meta``/``source=``
|
||||
strings and ``datahub/steward.py``'s live/published dataset table.
|
||||
- eastmoney/tencent interfaces mirror the ``observability.observe(...)``
|
||||
call sites added in ``datahub/realtime_serve.py`` for HEL-543.
|
||||
- ifind interfaces mirror ``datahub/steward.py``'s ``IFIND_APIS`` table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
CATALOG: list[dict[str, Any]] = [
|
||||
{
|
||||
"provider": "tushare",
|
||||
"label": "Tushare",
|
||||
"role": "official_primary",
|
||||
"credential_key": "tushare_token",
|
||||
"status_source": "src_health (legacy, kept) + provider_health (unified, HEL-543)",
|
||||
"interfaces": [
|
||||
{"interface": "trade_cal", "capability": "交易日历", "datasets": ["calendar"]},
|
||||
{"interface": "stock_basic", "capability": "股票主档", "datasets": ["stocks"]},
|
||||
{"interface": "daily", "capability": "个股日K", "datasets": ["daily"]},
|
||||
{"interface": "adj_factor", "capability": "复权因子", "datasets": ["daily"]},
|
||||
{"interface": "daily_basic", "capability": "估值", "datasets": ["valuation"]},
|
||||
{"interface": "index_daily", "capability": "指数日K", "datasets": ["index_daily"]},
|
||||
{"interface": "moneyflow", "capability": "资金流", "datasets": ["moneyflow"]},
|
||||
{"interface": "stk_auction", "capability": "集合竞价", "datasets": ["auction"]},
|
||||
{"interface": "limit_list_d", "capability": "涨跌停池", "datasets": ["limit_events"]},
|
||||
{"interface": "ths_hot", "capability": "同花顺人气榜", "datasets": ["popularity"]},
|
||||
{"interface": "dc_hot", "capability": "东方财富人气榜", "datasets": ["popularity"]},
|
||||
{"interface": "hm_detail", "capability": "龙虎榜游资明细", "datasets": ["dragon_tiger"]},
|
||||
{"interface": "ths_daily", "capability": "同花顺概念行情", "datasets": ["sector_daily"]},
|
||||
{"interface": "dc_index", "capability": "东方财富板块行情", "datasets": ["sector_daily"]},
|
||||
{"interface": "sw_daily", "capability": "申万行业行情", "datasets": ["sector_daily"]},
|
||||
],
|
||||
},
|
||||
{
|
||||
"provider": "eastmoney",
|
||||
"label": "东方财富",
|
||||
"role": "provisional_primary",
|
||||
"credential_key": None,
|
||||
"status_source": "provider_health (unified, HEL-543)",
|
||||
"interfaces": [
|
||||
{"interface": "indices", "capability": "指数实时报价", "datasets": ["index_quotes"]},
|
||||
{"interface": "market_quotes", "capability": "全市场实时快照", "datasets": ["quotes_latest"]},
|
||||
{"interface": "named_quotes", "capability": "指定个股实时报价", "datasets": ["quotes_latest"]},
|
||||
{"interface": "sector_quote", "capability": "申万板块实时报价(单个)", "datasets": ["sectors_quote"]},
|
||||
{"interface": "sector_quotes_batch", "capability": "申万板块批量报价(预热)", "datasets": ["sectors_quote"]},
|
||||
{"interface": "limit_pool", "capability": "涨停/炸板池(盘中)", "datasets": ["limit_pool"]},
|
||||
{"interface": "intraday", "capability": "分时走势", "datasets": ["intraday_points"]},
|
||||
],
|
||||
},
|
||||
{
|
||||
"provider": "tencent",
|
||||
"label": "腾讯行情",
|
||||
"role": "provisional_backup",
|
||||
"credential_key": None,
|
||||
"status_source": "provider_health (unified, HEL-543)",
|
||||
"interfaces": [
|
||||
{"interface": "indices", "capability": "指数实时报价(东财失败时备用)", "datasets": ["index_quotes"]},
|
||||
{
|
||||
"interface": "market_quotes_fallback",
|
||||
"capability": "全市场快照(备用；按本地股票主档逐只请求拼接)",
|
||||
"datasets": ["quotes_latest"],
|
||||
},
|
||||
{"interface": "named_quotes", "capability": "指定个股实时报价(东财失败时备用)", "datasets": ["quotes_latest"]},
|
||||
],
|
||||
},
|
||||
{
|
||||
"provider": "ifind",
|
||||
"label": "同花顺 iFinD",
|
||||
"role": "licensed_optional",
|
||||
"credential_key": "ifind_refresh_token",
|
||||
"status_source": "provider_health (unified, HEL-543) + adapter.status()",
|
||||
"interfaces": [
|
||||
{"interface": "wencai", "capability": "问财自然语言选股", "datasets": ["ifind_wencai"]},
|
||||
{"interface": "snapshots", "capability": "快照", "datasets": ["ifind_snapshots"]},
|
||||
{"interface": "history", "capability": "历史行情", "datasets": ["ifind_history"]},
|
||||
{"interface": "realtime", "capability": "实时行情", "datasets": ["ifind_realtime"]},
|
||||
{"interface": "intraday", "capability": "分时(高频)", "datasets": ["ifind_intraday"]},
|
||||
],
|
||||
},
|
||||
{
|
||||
"provider": "ths",
|
||||
"label": "同花顺(预留)",
|
||||
"role": "reserved",
|
||||
"credential_key": None,
|
||||
"status_source": "adapter.probe()（占位，本阶段未接入真实数据）",
|
||||
"interfaces": [],
|
||||
},
|
||||
{
|
||||
"provider": "xgb",
|
||||
"label": "选股宝(预留)",
|
||||
"role": "reserved",
|
||||
"credential_key": None,
|
||||
"status_source": "adapter.probe()（占位，本阶段未接入真实数据）",
|
||||
"interfaces": [],
|
||||
},
|
||||
{
|
||||
"provider": "akshare",
|
||||
"label": "AKShare(预留)",
|
||||
"role": "reserved",
|
||||
"credential_key": None,
|
||||
"status_source": "adapter.probe()（占位，本阶段未接入真实数据）",
|
||||
"interfaces": [],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def snapshot(db: Any, auth: Any = None) -> list[dict[str, Any]]:
|
||||
"""Merge the static catalog with live credential/health facts.
|
||||
|
||||
Purely read-only: never touches routing, credentials, or adapters. Any
|
||||
failure while enriching one entry only degrades that entry's live data;
|
||||
it never drops the entry or raises, so a directory read can never break
|
||||
on a partially-unhealthy database.
|
||||
"""
|
||||
result: list[dict[str, Any]] = []
|
||||
for entry in CATALOG:
|
||||
item: dict[str, Any] = {
|
||||
"provider": entry["provider"],
|
||||
"label": entry.get("label", entry["provider"]),
|
||||
"role": entry["role"],
|
||||
"status_source": entry["status_source"],
|
||||
"interfaces": [dict(i) for i in entry.get("interfaces", [])],
|
||||
}
|
||||
cred_key = entry.get("credential_key")
|
||||
if cred_key:
|
||||
cred = None
|
||||
try:
|
||||
if auth is not None:
|
||||
cred = auth.credential_status(cred_key)
|
||||
except Exception:
|
||||
cred = None
|
||||
item["credential"] = cred or {"configured": False, "last4": "", "updated_at": ""}
|
||||
else:
|
||||
item["credential"] = {"configured": True, "last4": "", "updated_at": "", "note": "无需凭证"}
|
||||
health_rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
if db is not None:
|
||||
health_rows = db.fetchall(
|
||||
"SELECT interface, state, last_ok_at, last_error, last_fallback_reason, "
|
||||
"consec_failures, last_latency_ms, last_data_age_seconds, updated_at "
|
||||
"FROM provider_health WHERE provider = ? ORDER BY interface",
|
||||
(entry["provider"],),
|
||||
)
|
||||
except Exception:
|
||||
health_rows = []
|
||||
item["live_interfaces"] = health_rows
|
||||
result.append(item)
|
||||
return result
|
||||
@@ -12,6 +12,7 @@ from typing import Any
|
||||
|
||||
from datahub.adapters.base import AdapterError
|
||||
from datahub.adapters.tushare import TUSHARE_FIELDS
|
||||
from datahub import observability
|
||||
from datahub.numbers import finite_number
|
||||
from datahub.realtime_serve import (
|
||||
RealtimeApiError,
|
||||
@@ -152,8 +153,12 @@ def _ifind_query(api, api_name: str, params: dict[str, Any], fields: str) -> dic
|
||||
)
|
||||
if not adapter.configured:
|
||||
raise ApiError("SOURCE_UNAVAILABLE", "iFinD 尚未配置")
|
||||
db = getattr(api, "db", None)
|
||||
try:
|
||||
rows = adapter.fetch(dataset, dict(params))
|
||||
rows = observability.observe(
|
||||
db, "ifind", dataset, lambda: adapter.fetch(dataset, dict(params)),
|
||||
classify=lambda r: observability.classify_rows(r, freshness_field=None),
|
||||
)
|
||||
except AdapterError as exc:
|
||||
raise ApiError("SOURCE_UNAVAILABLE", str(exc)) from exc
|
||||
return envelope(
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from datahub.adapters.tushare import TushareAdapter
|
||||
from datahub.crypto import SecretVault
|
||||
from datahub.httpapp import make_handler
|
||||
from datahub.hub import Hub
|
||||
from datahub.settings import Settings
|
||||
from tests.fixtures import TRADE_DATE, fake_transport
|
||||
|
||||
|
||||
class AdminObservabilityApiTests(unittest.TestCase):
|
||||
"""HEL-543: new read-only admin endpoints for provider status, source
|
||||
catalog and lineage. These must never require write access and must
|
||||
never touch the existing routing/publish logic."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
settings = Settings(
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="z" * 32,
|
||||
admin_password="StartPass1",
|
||||
tushare_token="real-tushare-token-abcdef",
|
||||
db_path=Path(self.tmp.name) / "hub.db",
|
||||
scheduler_enabled=False,
|
||||
)
|
||||
self.hub = Hub(settings, adapter=TushareAdapter("real-tushare-token-abcdef", transport=fake_transport))
|
||||
handler = make_handler(self.hub)
|
||||
self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
threading.Thread(target=self.server.serve_forever, daemon=True).start()
|
||||
self.base = f"http://127.0.0.1:{self.server.server_address[1]}"
|
||||
|
||||
_, body, cookie_header = self._json(
|
||||
"/admin/api/login", "POST", {"username": "hub_admin", "password": "StartPass1"}
|
||||
)
|
||||
cookie = cookie_header.split(";")[0]
|
||||
csrf = body["csrf"]
|
||||
self._json(
|
||||
"/admin/api/change-password",
|
||||
"POST",
|
||||
{"current": "StartPass1", "new_password": "NewPass123"},
|
||||
cookie=cookie,
|
||||
csrf=csrf,
|
||||
)
|
||||
self.cookie = cookie
|
||||
self.csrf = csrf
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _json(self, path, method="GET", body=None, cookie="", csrf=""):
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if cookie:
|
||||
headers["Cookie"] = cookie
|
||||
if csrf:
|
||||
headers["X-CSRF-Token"] = csrf
|
||||
req = Request(self.base + path, data=data, headers=headers, method=method)
|
||||
with urlopen(req, timeout=5) as resp:
|
||||
set_cookie = resp.headers.get("Set-Cookie", "")
|
||||
return resp.status, json.loads(resp.read().decode()), set_cookie
|
||||
|
||||
def _get(self, path):
|
||||
return self._json(path, cookie=self.cookie, csrf=self.csrf)
|
||||
|
||||
def test_providers_status_reflects_real_pipeline_activity(self) -> None:
|
||||
pipeline = self.hub.pipeline
|
||||
pipeline.ingest_reference(TRADE_DATE)
|
||||
pipeline.run_dataset("daily", TRADE_DATE)
|
||||
|
||||
status, body, _ = self._get("/admin/api/providers/status")
|
||||
self.assertEqual(status, 200)
|
||||
health = body["health"]
|
||||
self.assertTrue(any(item["provider"] == "tushare" and item["interface"] == "daily" for item in health))
|
||||
row = next(item for item in health if item["provider"] == "tushare" and item["interface"] == "daily")
|
||||
self.assertEqual(row["state"], "ok")
|
||||
recent = body["recent_calls"]
|
||||
self.assertTrue(any(item["provider"] == "tushare" and item["interface"] == "daily" for item in recent))
|
||||
|
||||
def test_providers_status_filters_by_provider(self) -> None:
|
||||
pipeline = self.hub.pipeline
|
||||
pipeline.ingest_reference(TRADE_DATE)
|
||||
pipeline.run_dataset("daily", TRADE_DATE)
|
||||
|
||||
status, body, _ = self._get("/admin/api/providers/status?provider=tushare")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertTrue(all(item["provider"] == "tushare" for item in body["health"]))
|
||||
self.assertTrue(all(item["provider"] == "tushare" for item in body["recent_calls"]))
|
||||
|
||||
status, body, _ = self._get("/admin/api/providers/status?provider=eastmoney")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body["health"], [])
|
||||
self.assertEqual(body["recent_calls"], [])
|
||||
|
||||
def test_source_catalog_lists_known_providers_without_leaking_secrets(self) -> None:
|
||||
status, body, _ = self._get("/admin/api/source-catalog")
|
||||
self.assertEqual(status, 200)
|
||||
blob = json.dumps(body)
|
||||
self.assertNotIn("real-tushare-token-abcdef", blob)
|
||||
providers = {item["provider"] for item in body["items"]}
|
||||
self.assertIn("tushare", providers)
|
||||
self.assertIn("eastmoney", providers)
|
||||
self.assertIn("tencent", providers)
|
||||
self.assertIn("ifind", providers)
|
||||
|
||||
def test_lineage_snapshot_and_affected_query(self) -> None:
|
||||
status, body, _ = self._get("/admin/api/lineage")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertTrue(len(body["items"]) > 0)
|
||||
datasets = {item["dataset"] for item in body["items"]}
|
||||
self.assertIn("stocks", datasets)
|
||||
|
||||
status, body, _ = self._get("/admin/api/lineage/affected?provider=tushare&interface=daily")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body["provider"], "tushare")
|
||||
self.assertEqual(body["interface"], "daily")
|
||||
|
||||
def test_disabled_kill_switch_reports_enabled_false_with_empty_structure(self) -> None:
|
||||
# HEL-543 total-review 🔴: flip the runtime kill switch the same way
|
||||
# Hub.__init__ wires Settings.observability_enabled onto the db
|
||||
# handle, then confirm every new endpoint reports disabled with an
|
||||
# explicit empty structure rather than silently going quiet.
|
||||
self.hub.db.observability_enabled = False
|
||||
try:
|
||||
pipeline = self.hub.pipeline
|
||||
pipeline.ingest_reference(TRADE_DATE)
|
||||
pipeline.run_dataset("daily", TRADE_DATE) # must still fully succeed
|
||||
|
||||
status, body, _ = self._get("/admin/api/providers/status")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body, {"enabled": False, "health": [], "recent_calls": []})
|
||||
|
||||
status, body, _ = self._get("/admin/api/source-catalog")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(body, {"enabled": False, "items": []})
|
||||
|
||||
status, body, _ = self._get("/admin/api/lineage")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertFalse(body["enabled"])
|
||||
self.assertEqual(body["items"], [])
|
||||
|
||||
status, body, _ = self._get("/admin/api/lineage/affected?provider=tushare")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(
|
||||
body, {"enabled": False, "provider": "tushare", "interface": "", "items": []}
|
||||
)
|
||||
|
||||
# Nothing was ever written while disabled.
|
||||
self.assertEqual(self.hub.db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
finally:
|
||||
self.hub.db.observability_enabled = True
|
||||
|
||||
def test_must_change_password_blocks_new_endpoints_too(self) -> None:
|
||||
from urllib.error import HTTPError
|
||||
|
||||
_, body, cookie_header = self._json(
|
||||
"/admin/api/login", "POST", {"username": "hub_admin", "password": "NewPass123"}
|
||||
)
|
||||
# Freshly logged-in user has already changed password in setUp, so
|
||||
# this login should not require a change; verify the endpoint is
|
||||
# reachable with a valid, non-must-change session (regression guard
|
||||
# against accidentally bypassing the must-change gate for these new
|
||||
# routes).
|
||||
cookie = cookie_header.split(";")[0]
|
||||
csrf = body["csrf"]
|
||||
status, _, _ = self._json("/admin/api/source-catalog", cookie=cookie, csrf=csrf)
|
||||
self.assertEqual(status, 200)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,105 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from datahub.adapters.ifind import IfindAdapter
|
||||
from datahub.db import HubDB
|
||||
from datahub.serving import ApiError
|
||||
from datahub.steward import steward_query
|
||||
|
||||
|
||||
class _Resp:
|
||||
def __init__(self, payload: dict, status: int = 200) -> None:
|
||||
import json
|
||||
|
||||
self.status = status
|
||||
self._raw = json.dumps(payload).encode("utf-8")
|
||||
|
||||
def read(self):
|
||||
return self._raw
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
|
||||
class IfindObservabilityTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db = HubDB(Path(self.tmp.name) / "hub.db")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _adapter_with_urlopen(self, urlopen) -> IfindAdapter:
|
||||
return IfindAdapter(refresh_token="rt", access_token="at", urlopen=urlopen)
|
||||
|
||||
def test_successful_fetch_is_logged_without_changing_rows(self) -> None:
|
||||
def urlopen(request, timeout=None):
|
||||
return _Resp(
|
||||
{
|
||||
"errorcode": 0,
|
||||
"tables": [{"thscode": ["000001.SZ"], "table": {"涨停原因": ["重组"]}}],
|
||||
}
|
||||
)
|
||||
|
||||
adapter = self._adapter_with_urlopen(urlopen)
|
||||
|
||||
class _Api:
|
||||
ifind = adapter
|
||||
db = self.db
|
||||
|
||||
payload = steward_query(
|
||||
_Api(), {"api_name": "ifind_wencai", "params": {"query": "涨停"}}
|
||||
)
|
||||
self.assertEqual(payload["data"][0]["thscode"], "000001.SZ")
|
||||
log = self.db.fetchone("SELECT * FROM provider_call_log WHERE provider = 'ifind'")
|
||||
self.assertIsNotNone(log)
|
||||
self.assertEqual(log["interface"], "wencai")
|
||||
self.assertEqual(log["status"], "ok")
|
||||
|
||||
def test_failed_fetch_reraises_and_logs_error(self) -> None:
|
||||
def urlopen(request, timeout=None):
|
||||
return _Resp({"errorcode": -9999, "errmsg": "quota exceeded"})
|
||||
|
||||
adapter = self._adapter_with_urlopen(urlopen)
|
||||
|
||||
class _Api:
|
||||
ifind = adapter
|
||||
db = self.db
|
||||
|
||||
with self.assertRaises(ApiError) as ctx:
|
||||
steward_query(_Api(), {"api_name": "ifind_wencai", "params": {"query": "涨停"}})
|
||||
self.assertEqual(ctx.exception.code, "SOURCE_UNAVAILABLE")
|
||||
log = self.db.fetchone("SELECT * FROM provider_call_log WHERE provider = 'ifind'")
|
||||
self.assertIsNotNone(log)
|
||||
self.assertEqual(log["status"], "error")
|
||||
|
||||
def test_status_check_alone_does_not_dial_or_log_a_fetch_call(self) -> None:
|
||||
class _Api:
|
||||
ifind = IfindAdapter()
|
||||
db = self.db
|
||||
|
||||
payload = steward_query(_Api(), {"api_name": "ifind_status", "params": {}})
|
||||
self.assertFalse(payload["data"][0]["configured"])
|
||||
log = self.db.fetchall("SELECT * FROM provider_call_log WHERE provider = 'ifind'")
|
||||
self.assertEqual(log, [])
|
||||
|
||||
def test_api_double_without_db_attribute_still_works(self) -> None:
|
||||
# Mirrors tests/test_ifind_adapter.py's `_Api` double, which has no
|
||||
# `db` attribute at all. Observability must not require it.
|
||||
class _Api:
|
||||
ifind = IfindAdapter()
|
||||
|
||||
payload = steward_query(_Api(), {"api_name": "ifind_status", "params": {}})
|
||||
self.assertFalse(payload["data"][0]["configured"])
|
||||
with self.assertRaises(ApiError):
|
||||
steward_query(_Api(), {"api_name": "ifind_wencai", "params": {"query": "涨停"}})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,306 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from datahub import observability
|
||||
from datahub.db import HubDB
|
||||
|
||||
|
||||
class _BrokenDB:
|
||||
"""A db double whose write() always raises, to prove fail-open."""
|
||||
|
||||
@contextmanager
|
||||
def write(self):
|
||||
raise RuntimeError("disk is full")
|
||||
yield None # pragma: no cover - unreachable, keeps this a generator
|
||||
|
||||
def fetchall(self, sql, params=()):
|
||||
raise RuntimeError("disk is full")
|
||||
|
||||
def fetchone(self, sql, params=()):
|
||||
raise RuntimeError("disk is full")
|
||||
|
||||
|
||||
class ClassifyRowsTests(unittest.TestCase):
|
||||
def test_empty_list_is_flagged_empty(self):
|
||||
status, reason, age = observability.classify_rows([])
|
||||
self.assertEqual(status, "empty")
|
||||
self.assertEqual(reason, "no_rows_returned")
|
||||
self.assertIsNone(age)
|
||||
|
||||
def test_empty_dict_result_is_flagged_empty(self):
|
||||
status, reason, _ = observability.classify_rows({})
|
||||
self.assertEqual(status, "empty")
|
||||
self.assertEqual(reason, "no_rows_returned")
|
||||
|
||||
def test_missing_required_field_is_flagged(self):
|
||||
rows = [{"ts_code": "600000.SH", "close": 10.2}, {"ts_code": "000001.SZ"}]
|
||||
status, reason, _ = observability.classify_rows(rows, required_fields=("close",), freshness_field=None)
|
||||
self.assertEqual(status, "missing_fields")
|
||||
self.assertIn("close", reason)
|
||||
|
||||
def test_fresh_rows_are_ok(self):
|
||||
import time
|
||||
|
||||
rows = [{"ts_code": "600000.SH", "close": 10.2, "quote_time_epoch": int(time.time())}]
|
||||
status, reason, age = observability.classify_rows(rows)
|
||||
self.assertEqual(status, "ok")
|
||||
self.assertEqual(reason, "")
|
||||
self.assertIsNotNone(age)
|
||||
self.assertLess(age, 5)
|
||||
|
||||
def test_stale_rows_are_flagged(self):
|
||||
import time
|
||||
|
||||
rows = [{"ts_code": "600000.SH", "close": 10.2, "quote_time_epoch": int(time.time()) - 3600}]
|
||||
status, reason, age = observability.classify_rows(rows, max_age_seconds=300)
|
||||
self.assertEqual(status, "stale")
|
||||
self.assertEqual(reason, "data_age_exceeds_threshold")
|
||||
self.assertGreaterEqual(age, 3600 - 5)
|
||||
|
||||
def test_classifier_never_raises_on_garbage_input(self):
|
||||
status, reason, age = observability.classify_rows(object())
|
||||
self.assertEqual(status, "empty")
|
||||
self.assertIsNone(age)
|
||||
# Malformed rows inside a list must not raise either.
|
||||
status, _, _ = observability.classify_rows(["not-a-dict", 123, None])
|
||||
self.assertEqual(status, "empty")
|
||||
|
||||
|
||||
class ClassifyErrorTests(unittest.TestCase):
|
||||
def test_blocked_page_markers_are_detected(self):
|
||||
status, reason = observability.classify_error("eastmoney request failed: Expecting value: line 1 column 1")
|
||||
self.assertEqual(status, "blocked")
|
||||
self.assertEqual(reason, "response_looks_like_intercept_page")
|
||||
|
||||
def test_timeout_is_detected(self):
|
||||
status, _ = observability.classify_error("tencent request failed: timed out")
|
||||
self.assertEqual(status, "timeout")
|
||||
|
||||
def test_generic_error_falls_back(self):
|
||||
status, reason = observability.classify_error("connection reset by peer")
|
||||
self.assertEqual(status, "error")
|
||||
self.assertEqual(reason, "")
|
||||
|
||||
def test_never_raises_on_none(self):
|
||||
status, reason = observability.classify_error(None)
|
||||
self.assertEqual(status, "error")
|
||||
|
||||
|
||||
class RecordCallTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db = HubDB(Path(self.tmp.name) / "hub.db")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_record_call_writes_log_and_health(self):
|
||||
observability.record_call(self.db, "eastmoney", "indices", status="ok", latency_ms=42)
|
||||
log_rows = self.db.fetchall("SELECT * FROM provider_call_log")
|
||||
self.assertEqual(len(log_rows), 1)
|
||||
self.assertEqual(log_rows[0]["provider"], "eastmoney")
|
||||
self.assertEqual(log_rows[0]["interface"], "indices")
|
||||
self.assertEqual(log_rows[0]["status"], "ok")
|
||||
health = self.db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = ? AND interface = ?",
|
||||
("eastmoney", "indices"),
|
||||
)
|
||||
self.assertIsNotNone(health)
|
||||
self.assertEqual(health["state"], "ok")
|
||||
self.assertEqual(health["consec_failures"], 0)
|
||||
|
||||
def test_consecutive_failures_increment_and_reset(self):
|
||||
observability.record_call(self.db, "tencent", "named_quotes", status="error", error="boom")
|
||||
observability.record_call(self.db, "tencent", "named_quotes", status="error", error="boom again")
|
||||
health = self.db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = ? AND interface = ?",
|
||||
("tencent", "named_quotes"),
|
||||
)
|
||||
self.assertEqual(health["consec_failures"], 2)
|
||||
self.assertEqual(health["state"], "error")
|
||||
observability.record_call(self.db, "tencent", "named_quotes", status="ok")
|
||||
health = self.db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = ? AND interface = ?",
|
||||
("tencent", "named_quotes"),
|
||||
)
|
||||
self.assertEqual(health["consec_failures"], 0)
|
||||
self.assertEqual(health["state"], "ok")
|
||||
|
||||
def test_none_db_is_a_silent_noop(self):
|
||||
# Must not raise even though there is nowhere to write.
|
||||
observability.record_call(None, "ifind", "wencai", status="ok")
|
||||
|
||||
def test_broken_db_write_does_not_raise(self):
|
||||
observability.record_call(_BrokenDB(), "eastmoney", "indices", status="ok")
|
||||
|
||||
|
||||
class ObserveTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db = HubDB(Path(self.tmp.name) / "hub.db")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_returns_exact_success_value_unmodified(self):
|
||||
sentinel = {"ts_code": "600000.SH", "close": 10.2}
|
||||
result = observability.observe(self.db, "eastmoney", "indices", lambda: sentinel)
|
||||
self.assertIs(result, sentinel)
|
||||
rows = self.db.fetchall("SELECT * FROM provider_call_log")
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["status"], "ok")
|
||||
|
||||
def test_reraises_exact_exception_on_failure(self):
|
||||
boom = ValueError("upstream exploded")
|
||||
|
||||
def fn():
|
||||
raise boom
|
||||
|
||||
with self.assertRaises(ValueError) as ctx:
|
||||
observability.observe(self.db, "eastmoney", "indices", fn)
|
||||
self.assertIs(ctx.exception, boom)
|
||||
rows = self.db.fetchall("SELECT * FROM provider_call_log")
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["status"], "error")
|
||||
self.assertIn("upstream exploded", rows[0]["error"])
|
||||
|
||||
def test_classify_downgrades_success_to_stale_without_changing_return_value(self):
|
||||
sentinel = [{"ts_code": "600000.SH", "quote_time_epoch": 1}]
|
||||
result = observability.observe(
|
||||
self.db, "eastmoney", "indices", lambda: sentinel,
|
||||
classify=lambda rows: observability.classify_rows(rows),
|
||||
)
|
||||
self.assertIs(result, sentinel)
|
||||
row = self.db.fetchone("SELECT * FROM provider_call_log")
|
||||
self.assertEqual(row["status"], "stale")
|
||||
|
||||
def test_broken_db_never_breaks_a_successful_call(self):
|
||||
sentinel = {"ok": True}
|
||||
result = observability.observe(_BrokenDB(), "eastmoney", "indices", lambda: sentinel)
|
||||
self.assertIs(result, sentinel)
|
||||
|
||||
def test_broken_db_never_masks_a_real_failure(self):
|
||||
def fn():
|
||||
raise RuntimeError("real upstream failure")
|
||||
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
observability.observe(_BrokenDB(), "eastmoney", "indices", fn)
|
||||
self.assertEqual(str(ctx.exception), "real upstream failure")
|
||||
|
||||
def test_classifier_exception_does_not_break_the_call(self):
|
||||
sentinel = {"ok": True}
|
||||
|
||||
def bad_classify(_result):
|
||||
raise KeyError("classifier bug")
|
||||
|
||||
result = observability.observe(self.db, "eastmoney", "indices", lambda: sentinel, classify=bad_classify)
|
||||
self.assertIs(result, sentinel)
|
||||
row = self.db.fetchone("SELECT * FROM provider_call_log")
|
||||
# A classifier bug must degrade to "ok", never silently drop the row
|
||||
# nor claim the call failed when it did not.
|
||||
self.assertEqual(row["status"], "ok")
|
||||
|
||||
def test_none_db_is_transparent_passthrough(self):
|
||||
sentinel = object()
|
||||
result = observability.observe(None, "eastmoney", "indices", lambda: sentinel)
|
||||
self.assertIs(result, sentinel)
|
||||
|
||||
|
||||
class _ToggleDB(HubDB):
|
||||
"""A real HubDB subclass so we can flip the HEL-543 kill switch the same
|
||||
way Hub.__init__ does, without needing a full Hub/Settings wiring."""
|
||||
|
||||
|
||||
class KillSwitchTests(unittest.TestCase):
|
||||
"""HEL-543 total-review 🔴: the observability side channel must be
|
||||
disable-able at runtime, and disabling it must leave existing behavior
|
||||
completely unchanged (pure passthrough, zero db access)."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db = _ToggleDB(Path(self.tmp.name) / "hub.db")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_is_enabled_defaults_true_when_attribute_absent(self):
|
||||
# A bare HubDB (as used throughout the rest of this test suite, and
|
||||
# by any pre-HEL-543 call site) must default to enabled.
|
||||
self.assertTrue(observability.is_enabled(self.db))
|
||||
self.assertTrue(observability.is_enabled(None))
|
||||
|
||||
def test_disabled_record_call_writes_nothing(self):
|
||||
self.db.observability_enabled = False
|
||||
observability.record_call(self.db, "eastmoney", "indices", status="ok", latency_ms=1)
|
||||
self.assertEqual(self.db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
self.assertEqual(self.db.fetchall("SELECT * FROM provider_health"), [])
|
||||
|
||||
def test_disabled_observe_is_a_pure_passthrough_on_success(self):
|
||||
self.db.observability_enabled = False
|
||||
sentinel = {"ts_code": "600000.SH"}
|
||||
calls = {"n": 0}
|
||||
|
||||
def fn():
|
||||
calls["n"] += 1
|
||||
return sentinel
|
||||
|
||||
result = observability.observe(self.db, "eastmoney", "indices", fn)
|
||||
self.assertIs(result, sentinel)
|
||||
self.assertEqual(calls["n"], 1)
|
||||
self.assertEqual(self.db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
|
||||
def test_disabled_observe_still_reraises_the_exact_exception(self):
|
||||
self.db.observability_enabled = False
|
||||
boom = RuntimeError("upstream exploded")
|
||||
|
||||
def fn():
|
||||
raise boom
|
||||
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
observability.observe(self.db, "eastmoney", "indices", fn)
|
||||
self.assertIs(ctx.exception, boom)
|
||||
self.assertEqual(self.db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
|
||||
def test_re_enabling_resumes_recording(self):
|
||||
self.db.observability_enabled = False
|
||||
observability.observe(self.db, "eastmoney", "indices", lambda: {"ok": True})
|
||||
self.assertEqual(self.db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
self.db.observability_enabled = True
|
||||
observability.observe(self.db, "eastmoney", "indices", lambda: {"ok": True})
|
||||
self.assertEqual(len(self.db.fetchall("SELECT * FROM provider_call_log")), 1)
|
||||
|
||||
|
||||
class SettingsToggleTests(unittest.TestCase):
|
||||
"""The kill switch follows the same env-var pattern as DATAHUB_SCHEDULER."""
|
||||
|
||||
def test_defaults_to_enabled(self):
|
||||
from datahub.settings import load_settings
|
||||
|
||||
settings = load_settings(env={})
|
||||
self.assertTrue(settings.observability_enabled)
|
||||
|
||||
def test_datahub_observability_zero_disables(self):
|
||||
from datahub.settings import load_settings
|
||||
|
||||
settings = load_settings(env={"DATAHUB_OBSERVABILITY": "0"})
|
||||
self.assertFalse(settings.observability_enabled)
|
||||
|
||||
def test_datahub_observability_off_disables(self):
|
||||
from datahub.settings import load_settings
|
||||
|
||||
settings = load_settings(env={"DATAHUB_OBSERVABILITY": "off"})
|
||||
self.assertFalse(settings.observability_enabled)
|
||||
|
||||
def test_datahub_observability_one_keeps_enabled(self):
|
||||
from datahub.settings import load_settings
|
||||
|
||||
settings = load_settings(env={"DATAHUB_OBSERVABILITY": "1"})
|
||||
self.assertTrue(settings.observability_enabled)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from datahub.pipeline import RetryError
|
||||
from tests.fixtures import TRADE_DATE
|
||||
from tests.test_pipeline import make_pipeline
|
||||
|
||||
|
||||
class PipelineObservabilityTests(unittest.TestCase):
|
||||
"""HEL-543: Tushare calls must keep writing the existing `src_calls`
|
||||
record unchanged, while also feeding the new cross-provider
|
||||
`provider_call_log` / `provider_health` side channel."""
|
||||
|
||||
def test_successful_fetch_logs_to_both_src_calls_and_provider_call_log(self) -> None:
|
||||
pipe, db = make_pipeline()
|
||||
pipe.ingest_reference(TRADE_DATE)
|
||||
result = pipe.run_dataset("daily", TRADE_DATE)
|
||||
self.assertEqual(result["state"], "published")
|
||||
|
||||
src_calls = db.fetchall("SELECT * FROM src_calls WHERE provider = 'tushare' AND endpoint = 'daily'")
|
||||
self.assertTrue(any(row["ok"] == 1 for row in src_calls))
|
||||
|
||||
log = db.fetchall(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'tushare' AND interface = 'daily'"
|
||||
)
|
||||
self.assertTrue(len(log) >= 1)
|
||||
self.assertEqual(log[-1]["status"], "ok")
|
||||
|
||||
health = db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = 'tushare' AND interface = 'daily'"
|
||||
)
|
||||
self.assertIsNotNone(health)
|
||||
self.assertEqual(health["state"], "ok")
|
||||
self.assertEqual(health["consec_failures"], 0)
|
||||
|
||||
def test_failed_fetch_logs_error_to_both_channels_and_still_raises(self) -> None:
|
||||
pipe, db = make_pipeline()
|
||||
pipe.ingest_reference(TRADE_DATE)
|
||||
|
||||
def boom(dataset, params):
|
||||
raise RuntimeError("tushare upstream 500")
|
||||
|
||||
pipe.adapter.fetch = boom # type: ignore[assignment]
|
||||
|
||||
with self.assertRaises(RetryError):
|
||||
pipe.run_dataset("daily", TRADE_DATE, attempts=1)
|
||||
|
||||
src_calls = db.fetchall("SELECT * FROM src_calls WHERE provider = 'tushare' AND endpoint = 'daily' AND ok = 0")
|
||||
self.assertTrue(len(src_calls) >= 1)
|
||||
self.assertIn("tushare upstream 500", src_calls[-1]["error"])
|
||||
|
||||
log = db.fetchall(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'tushare' AND interface = 'daily' AND status != 'ok'"
|
||||
)
|
||||
self.assertTrue(len(log) >= 1)
|
||||
self.assertIn("tushare upstream 500", log[-1]["error"])
|
||||
|
||||
health = db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = 'tushare' AND interface = 'daily'"
|
||||
)
|
||||
self.assertIsNotNone(health)
|
||||
self.assertNotEqual(health["state"], "ok")
|
||||
self.assertGreaterEqual(health["consec_failures"], 1)
|
||||
|
||||
def test_provider_call_log_is_purged_by_existing_cleanup_job(self) -> None:
|
||||
pipe, db = make_pipeline()
|
||||
pipe.ingest_reference(TRADE_DATE)
|
||||
pipe.run_dataset("daily", TRADE_DATE)
|
||||
self.assertTrue(db.fetchall("SELECT * FROM provider_call_log"))
|
||||
|
||||
# Force everything to look ancient so cleanup() sweeps it.
|
||||
db.execute("UPDATE provider_call_log SET created_at = '2000-01-01T00:00:00+08:00'")
|
||||
db.execute("UPDATE src_calls SET created_at = '2000-01-01T00:00:00+08:00'")
|
||||
db.execute("UPDATE job_runs SET started_at = '2000-01-01T00:00:00+08:00'")
|
||||
|
||||
pipe.cleanup()
|
||||
self.assertEqual(db.fetchall("SELECT * FROM provider_call_log"), [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,163 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from datahub.adapters.base import AdapterError
|
||||
from datahub.db import HubDB
|
||||
from datahub.realtime_serve import fetch_index_quotes, fetch_intraday, fetch_quotes
|
||||
|
||||
|
||||
class _WriteBreaksDB:
|
||||
"""Wraps a real HubDB but breaks only the write path, to prove the
|
||||
real serving path (reads/caches) is untouched by an observability
|
||||
failure while still exercising real fetch/cache code around it."""
|
||||
|
||||
def __init__(self, real: HubDB) -> None:
|
||||
self._real = real
|
||||
|
||||
def fetchall(self, sql, params=()):
|
||||
return self._real.fetchall(sql, params)
|
||||
|
||||
def fetchone(self, sql, params=()):
|
||||
return self._real.fetchone(sql, params)
|
||||
|
||||
def execute(self, sql, params=()):
|
||||
return self._real.execute(sql, params)
|
||||
|
||||
def executemany(self, sql, rows):
|
||||
return self._real.executemany(sql, rows)
|
||||
|
||||
@contextmanager
|
||||
def write(self):
|
||||
raise RuntimeError("db is not writable right now")
|
||||
yield None # pragma: no cover
|
||||
|
||||
|
||||
class RealtimeObservabilityTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db = HubDB(Path(self.tmp.name) / "hub.db")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_eastmoney_success_is_logged_without_changing_payload(self) -> None:
|
||||
rows = [
|
||||
{"ts_code": "000001.SH", "code": "000001", "name": "上证指数", "price": 3000.0,
|
||||
"previous_close": 2990.0, "quote_time_epoch": 0, "source": "eastmoney_push2"},
|
||||
{"ts_code": "399001.SZ", "code": "399001", "name": "深证成指", "price": 9000.0,
|
||||
"previous_close": 8990.0, "quote_time_epoch": 0, "source": "eastmoney_push2"},
|
||||
{"ts_code": "399006.SZ", "code": "399006", "name": "创业板指", "price": 1800.0,
|
||||
"previous_close": 1790.0, "quote_time_epoch": 0, "source": "eastmoney_push2"},
|
||||
]
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as mocked:
|
||||
mocked.return_value.fetch_indices.return_value = rows
|
||||
payload = fetch_index_quotes(self.db)
|
||||
self.assertEqual(payload["data"], rows)
|
||||
self.assertEqual(payload["meta"]["source"], "eastmoney:ulist")
|
||||
log = self.db.fetchall("SELECT * FROM provider_call_log WHERE provider = 'eastmoney'")
|
||||
self.assertEqual(len(log), 1)
|
||||
self.assertEqual(log[0]["interface"], "indices")
|
||||
self.assertEqual(log[0]["status"], "ok")
|
||||
health = self.db.fetchone(
|
||||
"SELECT * FROM provider_health WHERE provider = 'eastmoney' AND interface = 'indices'"
|
||||
)
|
||||
self.assertEqual(health["state"], "ok")
|
||||
|
||||
def test_eastmoney_failure_falls_back_to_tencent_and_logs_both(self) -> None:
|
||||
tencent_rows = [
|
||||
{"ts_code": "000001.SH", "code": "000001", "name": "上证指数", "price": 3000.0,
|
||||
"previous_close": 2990.0, "quote_time_epoch": 0, "source": "tencent_qt"},
|
||||
{"ts_code": "399001.SZ", "code": "399001", "name": "深证成指", "price": 9000.0,
|
||||
"previous_close": 8990.0, "quote_time_epoch": 0, "source": "tencent_qt"},
|
||||
{"ts_code": "399006.SZ", "code": "399006", "name": "创业板指", "price": 1800.0,
|
||||
"previous_close": 1790.0, "quote_time_epoch": 0, "source": "tencent_qt"},
|
||||
]
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as eastmoney, patch(
|
||||
"datahub.realtime_serve.TencentAdapter"
|
||||
) as tencent:
|
||||
eastmoney.return_value.fetch_indices.side_effect = AdapterError("Eastmoney returned 0/3 indices")
|
||||
tencent.return_value.fetch_indices.return_value = tencent_rows
|
||||
payload = fetch_index_quotes(self.db)
|
||||
self.assertEqual(payload["meta"]["source"], "tencent:qt")
|
||||
self.assertEqual(payload["data"], tencent_rows)
|
||||
east_log = self.db.fetchone("SELECT * FROM provider_call_log WHERE provider = 'eastmoney'")
|
||||
self.assertEqual(east_log["status"], "empty")
|
||||
tencent_log = self.db.fetchone("SELECT * FROM provider_call_log WHERE provider = 'tencent'")
|
||||
self.assertEqual(tencent_log["status"], "ok")
|
||||
|
||||
def test_observability_db_failure_never_breaks_a_real_successful_fetch(self) -> None:
|
||||
rows = [
|
||||
{"ts_code": "000001.SH", "price": 3000.0, "previous_close": 2990.0, "quote_time_epoch": 0},
|
||||
{"ts_code": "399001.SZ", "price": 9000.0, "previous_close": 8990.0, "quote_time_epoch": 0},
|
||||
{"ts_code": "399006.SZ", "price": 1800.0, "previous_close": 1790.0, "quote_time_epoch": 0},
|
||||
]
|
||||
broken = _WriteBreaksDB(self.db)
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as mocked:
|
||||
mocked.return_value.fetch_indices.return_value = rows
|
||||
payload = fetch_index_quotes(broken)
|
||||
self.assertEqual(payload["data"], rows)
|
||||
self.assertEqual(payload["meta"]["source"], "eastmoney:ulist")
|
||||
|
||||
def test_observability_db_failure_never_masks_a_real_source_outage(self) -> None:
|
||||
broken = _WriteBreaksDB(self.db)
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as eastmoney, patch(
|
||||
"datahub.realtime_serve.TencentAdapter"
|
||||
) as tencent:
|
||||
eastmoney.return_value.fetch_indices.side_effect = AdapterError("down")
|
||||
tencent.return_value.fetch_indices.side_effect = AdapterError("also down")
|
||||
with self.assertRaises(Exception):
|
||||
fetch_index_quotes(broken)
|
||||
|
||||
def test_named_quotes_records_both_providers_on_partial_merge(self) -> None:
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as eastmoney, patch(
|
||||
"datahub.realtime_serve.TencentAdapter"
|
||||
) as tencent:
|
||||
eastmoney.return_value.fetch_quotes.return_value = [
|
||||
{"ts_code": "000001.SZ", "close": 10, "pre_close": 9, "quote_date": "20260907"},
|
||||
]
|
||||
tencent.return_value.fetch_quotes.return_value = [
|
||||
{"ts_code": "000002.SZ", "close": 20, "pre_close": 19, "quote_date": "20260907"},
|
||||
]
|
||||
payload = fetch_quotes(self.db, ["000001.SZ", "000002.SZ"])
|
||||
self.assertEqual(payload["meta"]["complete"], True)
|
||||
east_log = self.db.fetchone(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'eastmoney' AND interface = 'named_quotes'"
|
||||
)
|
||||
self.assertEqual(east_log["status"], "ok")
|
||||
tencent_log = self.db.fetchone(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'tencent' AND interface = 'named_quotes'"
|
||||
)
|
||||
self.assertEqual(tencent_log["status"], "ok")
|
||||
|
||||
def test_intraday_success_is_logged_as_ok(self) -> None:
|
||||
payload_data = {
|
||||
"entity_type": "stock", "ts_code": "601318.SH", "trade_date": "2026-09-07",
|
||||
"previous_close": 55.8, "points": [{"date": "2026-09-07", "time": "09:30", "close": 55.9}],
|
||||
}
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as mocked:
|
||||
mocked.return_value.fetch_intraday.return_value = payload_data
|
||||
payload = fetch_intraday(self.db, "601318.SH")
|
||||
self.assertEqual(payload["data"], payload_data)
|
||||
log = self.db.fetchone(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'eastmoney' AND interface = 'intraday'"
|
||||
)
|
||||
self.assertEqual(log["status"], "ok")
|
||||
|
||||
def test_intraday_failure_is_logged_as_empty(self) -> None:
|
||||
with patch("datahub.realtime_serve.EastmoneyAdapter") as mocked:
|
||||
mocked.return_value.fetch_intraday.side_effect = AdapterError("No intraday chart data returned")
|
||||
with self.assertRaises(Exception):
|
||||
fetch_intraday(self.db, "000001.SZ")
|
||||
log = self.db.fetchone(
|
||||
"SELECT * FROM provider_call_log WHERE provider = 'eastmoney' AND interface = 'intraday'"
|
||||
)
|
||||
self.assertEqual(log["status"], "empty")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user