leefer/xiaobai-review
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(HEL-529): 按定稿100%重做三页数据中枢 + 数据修正1-5

Browse Source 
视觉：admin/app.js 按已确认打样 hub-kimi.html 逐行重写三页 DOM 与动效
（sparkline/缓存年龄秒增/延迟变化闪烁/EVENT TAPE 预装滚动/分组接口表/
更新频率列/时钟冒号 blink/雷达 blip），CSS 补真实调用脉冲 node-ping。
数据修正：
1) pipeline._stage 批内按暂存表业务键确定性去重（保留最后一条），
   修复人气榜/龙虎榜自 09-07 起每日 UNIQUE constraint 落库失败；
2) overview.anomalies 收敛为「最新批次未成功且当日未发布」的当前异常，
   历史已恢复批次留在审计明细；
3) source_catalog 接口补真实批次分组 + 观测 join（接口名或数据集名
   双向匹配，标注 observed/observed_basis），消除「全部未配置/0/30」误报；
4) lineage 逐数据集按其服务接口过滤健康行（接口级状态），
   provisional 已配置无观测显示「已配置 · 待观测」，仅 iFinD 为未配置；
5) lineage 补 update_freq 真实频率字段。
测试：新增 tests/test_hel529_rework.py（8 项），全套 191 项通过
（1 项环境依赖失败在基线 d9358ab 上同样复现，与本改动无关）。

Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
施工员2号
2026-09-15 08:57:29 +08:00
co-authored by multica-agent
parent d9358ab377
commit 3bafd30aad
8 changed files with 826 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
xiaobai-datahub/admin/app.js
+457 -182
View File
@@ -1,12 +1,13 @@
'use strict';
/* =====================================================================
小白复盘 · 数据中枢 admin/app.js — HEL-529 第十二版
三页真实数据中枢：运行总览 / 数据源配置 / 数据血缘。
视觉系统逐行照抄已通过打样 hub-kimi.html + SPEC-V11.md（v12 血缘压缩增补），
小白复盘 · 数据中枢 admin/app.js — HEL-529 返工：按定稿 100% 还原
视觉与 DOM 结构逐行照抄已确认打样 hub-kimi.html（v12 血缘压缩增补），
但本文件不含任何随机/模拟数据——全部状态、数字、事件均来自真实接口：
既有 /admin/api/{overview,sources,jobs,batches,datasets,audit,...}
与 HEL-543 新增只读旁路 /admin/api/{providers/status,source-catalog,
lineage,lineage/affected}。
/admin/api/{overview,sources,jobs,batches,audit,...} 与 HEL-543 只读旁路
/admin/api/{providers/status,source-catalog,lineage,lineage/affected}。
动效遵循 SPEC-V11：marquee 46s 线性 / flowline 1.4s / radar 4.2s /
blink 1.1s steps(2) / ledPing 1.6s / sparkBlink 1.4s / 时钟冒号 blink /
延迟仅在真实值变化时闪烁（禁止随机伪造数值）/ 缓存年龄每秒真实增长。
===================================================================== */
/* ================= 基础：DOM helper / API / CSRF ================= */
@@ -136,27 +137,23 @@ function kpiHtml(label, value, sub, tone, live) {
const tc = { txt: '', mint: 'glow-mint', amb: 'glow-amb', rd: 'glow-rd', cy: 'glow-cy' }[tone || 'txt'];
return `<div class="kpi"><span class="lab">${label}</span><span class="num kv ${tc}">${value}${live ? '<span class="blink" style="color:#22d3ee;margin-left:2px">_</span>' : ''}</span>${sub ? `<span class="ks">${sub}</span>` : ''}</div>`;
}
function radarBlipsHtml(pulseOk) {
return pulseOk
? `<div class="blip pulse-dot" style="left:64%;top:30%;background:#34d399"></div><div class="blip pulse-dot" style="left:30%;top:62%;background:#f87171;animation-delay:.6s"></div>`
: '';
}
/* 延迟：真实值 + 变化时才闪（打样 jit 的真实数据版：不随机抖动数字，
仅当新一轮真实调用改写该值时给 .flash 青色闪烁）。 */
function latHtml(ms, warn) {
if (ms == null) return '<span style="color:#54637e">—</span>';
const color = ms > 1000 ? '#fbbf24' : (warn ? '#fbbf24' : '#d7e1f0');
return `<span class="num" style="color:${color}">${Number(ms).toLocaleString('en-US')}ms</span>`;
}
function ageFromIso(iso) {
if (!iso) return null;
const t = Date.parse(iso.includes('T') ? iso : iso.replace(' ', 'T'));
if (Number.isNaN(t)) return null;
return Math.max(0, Math.floor((Date.now() - t) / 1000));
return `<span class="num jit" data-lat="${Number(ms)}" style="color:${color}">${Number(ms).toLocaleString('en-US')}ms</span>`;
}
/* 缓存年龄：记录真实时间戳，由 updateDynamics 每秒重算（真实增长）。 */
function ageHtml(iso) {
const s = ageFromIso(iso);
if (s == null) return '<span style="color:#54637e">—</span>';
const txt = s < 90 ? `${s}s` : s < 3600 ? `${Math.floor(s / 60)}m` : `${Math.floor(s / 3600)}h`;
return `<span class="num" style="color:${s > 90 ? '#fbbf24' : '#8b9bb4'}">${txt}</span>`;
const ms = Date.parse(String(iso || '').includes('T') ? iso : String(iso || '').replace(' ', 'T'));
if (!iso || Number.isNaN(ms)) return '<span style="color:#54637e">—</span>';
return `<span class="num" data-age-epoch="${ms}" style="color:#8b9bb4">—</span>`;
}
function fmtAgeSec(s) {
if (s == null) return '—';
// 打样口径：缓存年龄一律按秒展示、每秒增长（>30s 变琥珀）
return `${Number(s).toLocaleString('en-US')}s`;
}
function hm(iso) {
if (!iso) return '—';
@@ -167,13 +164,66 @@ function hm(iso) {
}
const p2 = (x) => String(x).padStart(2, '0');
/* ================= sparkline（真实历史：来自 provider_call_log） ================= */
const SPARK_COLORS = { cy: '#22d3ee', mint: '#34d399', amb: '#fbbf24', rd: '#f87171' };
function sparkHtml(jid, tone, w, h) {
return `<svg width="${w}" height="${h}" class="spark" data-jid="${esc(jid)}" data-w="${w}" data-h="${h}" style="display:block;opacity:.9"><polyline points="" fill="none" stroke="${SPARK_COLORS[tone]}" stroke-width="1.2" stroke-opacity=".8"/><circle class="spark-end" r="2" fill="${SPARK_COLORS[tone]}" cx="0" cy="0"/></svg>`;
}
function sparkSeries(jid) {
const hist = LatHist.get(jid) || [];
if (hist.length < 2) return null;
const max = Math.max(...hist, 1);
return hist.map((v) => v / max);
}
function sparkPaint(svg) {
const jid = svg.dataset.jid, w = +svg.dataset.w, h = +svg.dataset.h;
const data = sparkSeries(jid);
const poly = svg.querySelector('polyline');
const dot = svg.querySelector('circle');
if (!data) {
const y = (h / 2).toFixed(2);
poly.setAttribute('points', `0,${y} ${w},${y}`);
poly.setAttribute('stroke-opacity', '.25');
dot.style.display = 'none';
return;
}
poly.setAttribute('stroke-opacity', '.8');
dot.style.display = '';
const pts = data.map((v, i) => `${((i / (data.length - 1)) * w).toFixed(2)},${(h - v * (h - 3) - 1).toFixed(2)}`).join(' ');
poly.setAttribute('points', pts);
dot.setAttribute('cx', (w - 1).toFixed(2));
dot.setAttribute('cy', (h - data[data.length - 1] * (h - 3) - 1).toFixed(2));
}
/* 每个“观测对象”（接口或数据集）的真实延迟历史（旧→新），来自
provider_call_log 最近 200 条。 */
const LatHist = new Map();
const CALL_MATCHERS = {};
function buildLatHist() {
const calls = (RAW.providers && RAW.providers.recent_calls) || [];
const buckets = new Map();
[...calls].reverse().forEach((c) => {
if (c.latency_ms == null) return;
const key = `${c.provider}:${c.interface}`;
(buckets.get(key) || buckets.set(key, []).get(key)).push(c.latency_ms);
});
LatHist.clear();
buckets.forEach((arr, key) => LatHist.set(key, arr.slice(-24)));
// 数据集维度的历史：tushare 观测行写的是数据集名，直接按数据集聚合
const dsBuckets = new Map();
[...calls].reverse().forEach((c) => {
if (c.provider !== 'tushare' || c.latency_ms == null) return;
(dsBuckets.get(c.interface) || dsBuckets.set(c.interface, []).get(c.interface)).push(c.latency_ms);
});
dsBuckets.forEach((arr, ds) => LatHist.set(`ds:${ds}`, arr.slice(-24)));
}
/* ================= 数据层：拉取真实接口并汇总为一份快照 ================= */
const RAW = {
overview: null, sources: null, providers: null, catalog: null,
lineage: null, jobs: null, batches: null, datasets: null, audit: null,
lineage: null, jobs: null, batches: null, audit: null,
};
async function fetchAll() {
const [overview, sources, providers, catalog, lineage, jobs, batches, datasets, audit] = await Promise.all([
const [overview, sources, providers, catalog, lineage, jobs, batches, audit] = await Promise.all([
api('/admin/api/overview'),
api('/admin/api/sources'),
api('/admin/api/providers/status?limit=200'),
@@ -181,31 +231,14 @@ async function fetchAll() {
api('/admin/api/lineage'),
api('/admin/api/jobs'),
api(`/admin/api/batches?date=${today()}`),
api(`/admin/api/datasets?date=${today()}`),
api('/admin/api/audit'),
]);
RAW.overview = overview; RAW.sources = sources; RAW.providers = providers; RAW.catalog = catalog;
RAW.lineage = lineage; RAW.jobs = jobs; RAW.batches = batches; RAW.datasets = datasets; RAW.audit = audit;
RAW.lineage = lineage; RAW.jobs = jobs; RAW.batches = batches; RAW.audit = audit;
buildLatHist();
}
function today() { return RAW.overview ? RAW.overview.trade_date : ''; }
/* 按 provider(+interface) 索引 provider_health，供各页复用 */
function healthIndex() {
const idx = {};
const rows = (RAW.providers && RAW.providers.health) || [];
for (const row of rows) {
idx[row.provider] = idx[row.provider] || [];
idx[row.provider].push(row);
}
return idx;
}
function callsIndex() {
const idx = {};
const rows = (RAW.providers && RAW.providers.recent_calls) || [];
for (const row of rows) { (idx[row.provider] = idx[row.provider] || []).push(row); }
return idx;
}
const OFFICIAL_DATASET_IDS = ['daily', 'valuation', 'moneyflow', 'auction', 'index_daily', 'limit_events', 'popularity', 'dragon_tiger', 'sector_daily', 'stocks'];
const DATASET_NAME = {
daily: '日K', valuation: '估值', moneyflow: '资金流', auction: '竞价', index_daily: '指数日K',
@@ -213,7 +246,7 @@ const DATASET_NAME = {
};
/* =====================================================================
总览页
总览页（结构照抄 hub-kimi.html，数据全部真实）
===================================================================== */
function buildOverviewVM() {
const ov = RAW.overview || {};
@@ -223,7 +256,6 @@ function buildOverviewVM() {
(ov.publications || []).forEach((p) => { pubsByDs[p.dataset] = p; });
const batchByDs = {};
((RAW.batches && RAW.batches.batches) || []).forEach((b) => { if (!batchByDs[b.dataset] || b.started_at > batchByDs[b.dataset].started_at) batchByDs[b.dataset] = b; });
const hidx = healthIndex();
const totalOfficial = OFFICIAL_DATASET_IDS.length;
const publishedCount = OFFICIAL_DATASET_IDS.filter((id) => pubsByDs[id] && pubsByDs[id].state === 'published').length;
const anomalyCount = (ov.anomalies || []).length;
@@ -231,7 +263,7 @@ function buildOverviewVM() {
const successRate = recentCalls.length ? (recentCalls.filter((c) => c.ok).length / recentCalls.length * 100) : null;
// 主站回退页面：主源异常且备源健康 → 判定为“回退中”的 provisional 数据集
let fallbackPages = 0;
const fallbackPages = [];
(RAW.lineage && RAW.lineage.items || []).forEach((it) => {
if (it.tier !== 'provisional' || !it.backup_source) return;
const rows = it.live_provider_health || [];
@@ -239,7 +271,9 @@ function buildOverviewVM() {
const backupProv = (it.providers || [])[1];
const primaryState = worstBucket(rows.filter((r) => r.provider === primaryProv).map((r) => bucketOf(r.state)));
const backupState = worstBucket(rows.filter((r) => r.provider === backupProv).map((r) => bucketOf(r.state)));
if ((primaryState === 'fail' || primaryState === 'degrade' || primaryState === 'slow') && backupState === 'ok') fallbackPages += 1;
if ((primaryState === 'fail' || primaryState === 'degrade' || primaryState === 'slow') && backupState === 'ok') {
(it.known_consumers || []).forEach((c) => fallbackPages.push(c));
}
});
const datasetCards = OFFICIAL_DATASET_IDS.map((id) => {
@@ -257,37 +291,31 @@ function buildOverviewVM() {
return { id, name: DATASET_NAME[id] || id, bucket, time: note && !batch?.error ? note : '', rows, err: batch && batch.error ? batch.error : '' };
});
// 实时观察层：eastmoney / tencent 的真实 provider_health 行
// 实时观察层：eastmoney / tencent 目录接口的真实观测（observed=true）
const catalogByProv = {};
(RAW.catalog && RAW.catalog.items || []).forEach((it) => { catalogByProv[it.provider] = it; });
const observers = [];
['eastmoney', 'tencent'].forEach((prov) => {
const rows = hidx[prov] || [];
const cat = catalogByProv[prov];
rows.forEach((row) => {
const iface = (cat && cat.interfaces.find((i) => i.interface === row.interface)) || {};
if (!cat) return;
cat.interfaces.forEach((iface) => {
if (!iface.observed) return;
observers.push({
name: iface.capability || row.interface,
provider: prov,
interface: row.interface,
lastOk: row.last_ok_at,
lat: row.last_latency_ms,
bucket: bucketOf(row.state),
note: row.last_error || row.last_fallback_reason || '',
jid: `${prov}:${iface.interface}`,
name: iface.capability, provider: prov, interface: iface.interface,
lastOk: iface.observed_at, lat: iface.observed_latency_ms,
bucket: bucketOf(iface.observed_state),
note: iface.observed_note || '',
});
});
});
// 主站影响：lineage() 条目按 known_consumers 展开
// 主站影响：lineage() 条目按 known_consumers 展开（接口级状态）
const siteImpact = [];
(RAW.lineage && RAW.lineage.items || []).forEach((it) => {
const rows = it.live_provider_health || [];
const bucket = it.tier === 'official'
? (it.publication ? (it.publication.state === 'published' ? 'ok' : (it.publication.state === 'degraded' ? 'degrade' : 'fail')) : (rows.length ? worstBucket(rows.map((r) => bucketOf(r.state))) : 'plan'))
: (rows.length ? worstBucket(rows.map((r) => bucketOf(r.state))) : 'off');
const note = rows.find((r) => r.last_error)?.last_error || rows.find((r) => r.last_fallback_reason)?.last_fallback_reason || '';
const bucket = lineageItemBucket(it);
(it.known_consumers || []).forEach((c) => {
siteImpact.push({ page: c, dataset: it.dataset, bucket, note });
siteImpact.push({ page: c, short: pageNameOf(c), dataset: it.dataset, bucket, note: lineageItemNote(it) });
});
});
const okPages = siteImpact.filter((s) => s.bucket === 'ok').length;
@@ -304,14 +332,16 @@ function buildOverviewVM() {
return { t: (j.at.split('-')[0] || j.at).slice(0, 5), label: j.title, state: st, jobId: j.id };
});
// 待处理异常：批次失败/停滞 + 主源连续失败 + 未配置授权源
// 待处理异常：overview.anomalies 已在后端收敛为“仍未恢复的最新异常”
const incidents = [];
(ov.anomalies || []).forEach((b) => {
incidents.push({ sev: 'fail', tag: b.state, title: `${DATASET_NAME[b.dataset] || b.dataset} ${b.state === 'failed' ? '失败' : '停滞'}`, meta: `${b.error ? esc(b.error) : '无错误详情'} · 批次 ${b.batch_id}`, dataset: b.dataset });
});
if (eod.state === 'waiting_upstream' || eod.state === 'cutoff_failed') {
incidents.push({ sev: eod.state === 'cutoff_failed' ? 'fail' : 'slow', tag: eod.state, title: eod.state === 'cutoff_failed' ? '盘后批已截止失败' : '等待上游重试', meta: `已试 ${eod.attempts || 0} 次${(eod.missing_datasets || []).length ? ' · 缺 ' + eod.missing_datasets.join(',') : ''}`, dataset: (eod.missing_datasets || [])[0] || '' });
incidents.push({ sev: eod.state === 'cutoff_failed' ? 'fail' : 'slow', tag: eod.state, title: eod.state === 'cutoff_failed' ? '盘后批已截止失败' : '自动重试窗口', meta: `15:15–23:30 · 每 30 分钟 · 成功即停 · 已试 ${eod.attempts || 0} 次${(eod.missing_datasets || []).length ? ' · 缺 ' + eod.missing_datasets.join(',') : ''}`, dataset: (eod.missing_datasets || [])[0] || '', cd: eod.next_retry_at });
}
const hidx = {};
((RAW.providers && RAW.providers.health) || []).forEach((r) => { (hidx[r.provider] = hidx[r.provider] || []).push(r); });
Object.keys(hidx).forEach((prov) => {
const bad = (hidx[prov] || []).filter((r) => (r.consec_failures || 0) > 0 && bucketOf(r.state) === 'fail');
if (bad.length) incidents.push({ sev: 'fail', tag: prov, title: `${prov} 部分接口连续失败`, meta: bad.map((r) => `${r.interface}×${r.consec_failures}`).join(' · '), provider: prov });
@@ -325,6 +355,7 @@ function buildOverviewVM() {
trade_date: ov.trade_date, session_phase: ov.session_phase, eod, rev,
publishedCount, totalOfficial, anomalyCount, successRate,
fallbackPages, datasetCards, observers, siteImpact, okPages, timeline, incidents,
unpublished: OFFICIAL_DATASET_IDS.filter((id) => !pubsByDs[id] || pubsByDs[id].state !== 'published'),
};
}
@@ -335,7 +366,7 @@ function heroHtml(vm) {
if (vm.rev.state === 'waiting_review') tags.push(tagHtml('cy', '复核 · 待发布'));
const subParts = [];
if (vm.eod.attempts) subParts.push(`已试 <span style="color:#fbbf24">${vm.eod.attempts}</span> 次`);
if (vm.eod.next_retry_at) subParts.push(`下次重试 <span style="color:#fbbf24">${hm(vm.eod.next_retry_at)}</span>`);
if (vm.eod.next_retry_at) subParts.push(`下次重试 <span class="num" data-cd-epoch="${Date.parse(String(vm.eod.next_retry_at).replace(' ', 'T'))}" data-cdfmt="mmss" style="color:#fbbf24">--:--</span>`);
if ((vm.eod.missing_datasets || []).length) subParts.push(`缺 <span style="color:#f87171">${esc(vm.eod.missing_datasets.join(','))}</span>`);
if (vm.rev.window) subParts.push(`复核窗口 <span style="color:#22d3ee">${esc(vm.rev.window)}</span>`);
return `<div class="panel hero">` +
@@ -345,23 +376,25 @@ function heroHtml(vm) {
(subParts.length ? `<div class="hero-sub">${subParts.join('<span class="sep">|</span>')}</div>` : '') +
`</div><span class="flex1"></span>` +
`<div class="kpis">` +
kpiHtml('已发布数据集', `<span class="glow-mint">${vm.publishedCount}</span><span style="color:#54637e;font-size:18px">/${vm.totalOfficial}</span>`, null) +
kpiHtml('已发布数据集', `<span class="glow-mint">${vm.publishedCount}</span><span style="color:#54637e;font-size:18px">/${vm.totalOfficial}</span>`, vm.unpublished.length ? `${vm.unpublished.map((d) => DATASET_NAME[d] || d).join(' · ')} 未发布` : '全部已发布') +
kpiHtml('今日调用成功率', vm.successRate == null ? '<span style="color:#54637e">—</span>' : `<span>${vm.successRate.toFixed(1)}%</span>`, null, 'mint', vm.successRate != null) +
kpiHtml('主站回退页面', `${vm.fallbackPages} 页`, null, vm.fallbackPages ? 'amb' : 'txt') +
kpiHtml('主站回退页面', `${vm.fallbackPages.length} 页`, vm.fallbackPages.length ? esc(vm.fallbackPages[0]) : '实时层主源正常', vm.fallbackPages.length ? 'amb' : 'txt') +
kpiHtml('EOD 尝试', `${vm.eod.attempts || 0} 次`, '上限 5 · 23:30 止', vm.eod.attempts ? 'rd' : 'txt') +
`</div></div>`;
}
function datasetStripHtml(vm) {
return `<div class="ds-grid">${vm.datasetCards.map((d) => {
const ring = d.bucket === 'fail' ? 'fail' : d.bucket === 'degrade' ? 'slow' : d.bucket === 'review' ? 'review' : '';
const sparkTone = d.bucket === 'fail' ? 'rd' : d.bucket === 'degrade' ? 'amb' : d.bucket === 'review' ? 'cy' : 'mint';
let note = '';
if (d.time) note += `<span class="num" style="color:#8b9bb4">${esc(d.time)} · </span>`;
if (d.rows) note += `<span class="num">${Number(d.rows).toLocaleString('en-US')} 行</span>`;
if (d.err) note += `<span style="color:#f87171">${esc(d.err)}</span>`;
if (d.err) { const brief = String(d.err).length > 22 ? String(d.err).slice(0, 22) + '…' : d.err; note += `<span style="color:#f87171" title="${esc(d.err)}">${esc(brief)}</span>`; }
if (!d.time && !d.rows && !d.err) note = '<span style="color:#54637e">今日尚未发布</span>';
return `<div class="panel dsc ${ring}" data-dataset="${d.id}">` +
`<div class="dsc-top"><span class="dsc-name">${esc(d.name)}</span>${pillHtml(d.bucket)}</div>` +
`<div class="dsc-note">${note}</div>` +
sparkHtml(`ds:${d.id}`, sparkTone, 96, 16) +
`</div>`;
}).join('')}</div>`;
}
@@ -373,20 +406,30 @@ function observatoryHtml(vm) {
`<td style="color:#8b9bb4">${o.provider === 'eastmoney' ? '东方财富' : '腾讯行情'}</td>` +
`<td>${ageHtml(o.lastOk)}</td>` +
`<td>${latHtml(o.lat, o.bucket !== 'ok')}</td>` +
`<td>${sparkHtml(o.jid, o.bucket === 'slow' ? 'amb' : 'mint', 64, 16)}</td>` +
`<td><span style="display:flex;align-items:center;gap:8px">${pillHtml(o.bucket)}${o.note ? `<span class="obs-note">${esc(o.note)}</span>` : ''}</span></td>` +
`</tr>`).join('') : `<tr><td colspan="5" class="empty-hint">尚无真实观测记录（等待首次探测/调用）</td></tr>`;
const body = `<table class="dtable"><thead><tr><th>观察项</th><th>当前来源</th><th>最近成功</th><th>延迟</th><th>状态 · 备注</th></tr></thead><tbody>${rows}</tbody></table>`;
`</tr>`).join('') : `<tr><td colspan="6" class="empty-hint">尚无真实观测记录（等待首次探测/调用）</td></tr>`;
const body = `<table class="dtable"><thead><tr><th>观察项</th><th>当前来源</th><th>缓存年龄</th><th>延迟</th><th>趋势</th><th>状态 · 备注</th></tr></thead><tbody>${rows}</tbody></table>`;
return panelHtml('实时观察层', right, body, false);
}
function siteImpactHtml(vm) {
const bad = vm.siteImpact.filter((s) => s.bucket !== 'ok');
const right = `<span style="font-size:10px"><span style="color:#34d399">${vm.okPages} 项正常</span><span style="color:#54637e"> · </span><span style="color:#fbbf24">${vm.siteImpact.length - vm.okPages} 项待关注</span></span>`;
const rows = vm.siteImpact.length ? vm.siteImpact.map((s) => `<tr>` +
`<td style="color:#e8f1ff;width:36%">${esc(s.page)}</td>` +
`<td style="width:18%">${pillHtml(s.bucket)}</td>` +
const rows = bad.length ? bad.slice(0, 5).map((s) => `<tr>` +
`<td style="color:#e8f1ff;width:40%">${esc(s.page)}</td>` +
`<td style="width:20%">${pillHtml(s.bucket)}</td>` +
`<td style="font-size:11px;color:#54637e">${s.note ? esc(s.note) : '—'}</td>` +
`<td style="text-align:right"><button class="tbtn mini" data-goto-lineage="${esc(s.dataset)}">查看血缘</button></td>` +
`</tr>`).join('') : `<tr><td colspan="4" class="empty-hint">尚无血缘记录</td></tr>`;
return panelHtml('主站影响', right, `<table class="dtable"><tbody>${rows}</tbody></table>`, false);
`</tr>`).join('') : `<tr><td style="color:#e8f1ff;width:40%">全部页面正常</td><td>${pillHtml('ok')}</td><td style="font-size:11px;color:#54637e">—</td><td></td></tr>`;
const footPages = vm.siteImpact.map((s) => ({ name: s.short, bucket: s.bucket }));
const byPage = new Map();
footPages.forEach((p) => { const cur = byPage.get(p.name); if (!cur || SEV[p.bucket] > SEV[cur]) byPage.set(p.name, p.bucket); });
const foot = `<div class="site-foot"><span style="color:#34d399;margin-right:4px">正常 ${vm.okPages}</span>` +
[...byPage.entries()].map(([name, bucket]) => bucket === 'ok'
? `<span style="display:inline-flex;align-items:center;gap:6px"><span>${esc(name)}</span><span style="color:#243152">·</span></span>`
: `<span style="display:inline-flex;align-items:center;gap:6px;color:${HEALTH[bucket].color}">${esc(name)}(${HEALTH[bucket].label})</span><span style="color:#243152">·</span></span>`).join('') +
`</div>`;
return panelHtml('主站影响', right, `<table class="dtable"><tbody>${rows}</tbody></table>${foot}`, false);
}
function timelineHtml(vm) {
const nodes = vm.timeline;
@@ -422,17 +465,21 @@ function timelineHtml(vm) {
s += `<g><line x1="${nowX}" y1="${y - 44}" x2="${nowX}" y2="${y + 44}" stroke="#22d3ee" stroke-width="1" stroke-dasharray="2 3" opacity=".7"/>` +
`<circle cx="${nowX}" cy="${y}" r="3.6" fill="#22d3ee" class="now-dot now-r"/>` +
`<text x="${nowX}" y="${y - 50}" text-anchor="middle" font-size="9" fill="#22d3ee" font-weight="600" class="num">NOW</text></g></svg>`;
const right = `<span style="font-size:10px;color:#54637e">现在 <span class="num" style="color:#22d3ee">${p2(now.getHours())}:${p2(now.getMinutes())}</span> · <span style="color:#8b9bb4">SCHEDULER.PY</span></span>`;
const range = nodes.length ? `${nodes[0].t} → ${nodes[nodes.length - 1].t}` : '';
const right = `<span style="font-size:10px;color:#54637e">现在 <span class="num" style="color:#22d3ee">${p2(now.getHours())}:${p2(now.getMinutes())}</span> · ${range} · <span style="color:#8b9bb4">SCHEDULER.PY</span></span>`;
return panelHtml('今晚时间线', right, s, false);
}
function incidentsHtml(vm) {
if (!vm.incidents.length) return panelHtml('待处理异常', tagHtml('mint', '0 OPEN'), '<div class="empty-hint">当前无待处理异常</div>', true);
const cards = vm.incidents.map((it) => {
const tag = it.sev === 'fail' ? tagHtml('rd', esc(it.tag)) : it.sev === 'slow' ? tagHtml('amb', esc(it.tag)) : tagHtml('dashed', esc(it.tag));
const meta = it.cd
? `${it.meta}<span class="num" style="color:#fbbf24"> · 下次重试 <span data-cd-epoch="${Date.parse(String(it.cd).replace(' ', 'T'))}" data-cdfmt="ms">--m--s</span></span>`
: it.meta;
return `<div class="inc ${it.sev}">${tag}` +
`<div style="flex:1;min-width:0">` +
`<div class="inc-title"${it.sev === 'off' ? ' style="color:#8b9bb4"' : ''}>${esc(it.title)}</div>` +
`<div class="inc-meta">${esc(it.meta)}</div>` +
`<div class="inc-meta">${meta}</div>` +
`</div>` +
`<button class="tbtn" style="flex:none" data-incident-action data-dataset="${esc(it.dataset || '')}" data-provider="${esc(it.provider || '')}">去处理 →</button>` +
`</div>`;
@@ -447,44 +494,62 @@ function overviewHtml() {
}
/* =====================================================================
数据源配置页
数据源配置页（分组接口表照抄打样 mx-grid 结构，数据全部真实）
===================================================================== */
const srcUiState = {};
const PROV_ROLE_TEXT = {
tushare: 'official · 盘后正式数据唯一源',
eastmoney: 'free · 盘中观察主源',
tencent: 'free · 盘中观察备源',
ifind: 'licensed · 图表 / 问财 / 实时快照',
};
function buildSourcesVM() {
const catalog = (RAW.catalog && RAW.catalog.items) || [];
const legacy = {};
(RAW.sources && RAW.sources.items || []).forEach((it) => { legacy[it.provider] = it; });
const hidx = healthIndex();
const cidx = callsIndex();
const cidx = {};
((RAW.providers && RAW.providers.recent_calls) || []).forEach((c) => { if (!cidx[c.provider]) cidx[c.provider] = c; });
const active = catalog.filter((c) => c.role !== 'reserved');
const reserved = catalog.filter((c) => c.role === 'reserved');
const totalIfaces = active.reduce((n, c) => n + c.interfaces.length, 0);
const usedIfaces = active.reduce((n, c) => n + c.interfaces.filter((i) => (hidx[c.provider] || []).some((h) => h.interface === i.interface)).length, 0);
// “在用”= 已登记且已观察到调用（接口名或数据集名匹配，后端 join 已标注）
const usedIfaces = active.reduce((n, c) => n + c.interfaces.filter((i) => i.observed).length, 0);
const cards = active.map((c) => {
const leg = legacy[c.provider] || {};
const legState = (leg.health && (leg.health.state || leg.health.status)) || null;
const rows = hidx[c.provider] || [];
const worst = rows.length ? worstBucket(rows.map((r) => bucketOf(r.state))) : (legState ? bucketOf(legState) : 'off');
const cred = c.credential || {};
const lastCall = (cidx[c.provider] || [])[0];
const observedRows = c.interfaces.filter((i) => i.observed);
const worst = observedRows.length ? worstBucket(observedRows.map((i) => bucketOf(i.observed_state))) : null;
const configured = cred.configured;
let bucket;
if (worst) bucket = worst;
else if (configured) bucket = 'plan';
else bucket = 'off';
const lastCall = cidx[c.provider];
if (!(c.provider in srcUiState)) srcUiState[c.provider] = { open: true, probing: false, probeResult: null };
const ifaceRows = c.interfaces.map((i) => {
const h = rows.find((r) => r.interface === i.interface);
return {
api: i.interface, use: i.capability,
ok: h ? (h.last_ok_at ? hm(h.last_ok_at) : '—') : '—',
lat: h ? h.last_latency_ms : null,
st: h ? bucketOf(h.state) : 'off',
err: h && h.state !== 'ok' ? (h.last_error || h.last_fallback_reason || '') : '',
};
const groups = [];
const groupMap = new Map();
c.interfaces.forEach((i) => {
const g = i.group || '其他';
if (!groupMap.has(g)) groupMap.set(g, []);
groupMap.get(g).push(i);
});
groupMap.forEach((items, name) => groups.push({ name, items }));
const ifaceRows = (group) => group.items.map((i) => ({
api: i.interface, use: i.capability,
ok: i.observed ? (i.observed_at ? hm(i.observed_at) : '—') : '—',
lat: i.observed ? i.observed_latency_ms : null,
st: i.observed ? bucketOf(i.observed_state) : (configured ? 'plan' : 'off'),
obs: i.observed, basis: i.observed_basis,
err: i.observed && i.observed_state !== 'ok' ? (i.observed_note || '') : '',
}));
return {
id: c.provider, name: c.label, role: c.role, tag: c.credential_key ? 'licensed' : 'free',
bucket: worst, credential: cred, usedCount: ifaceRows.filter((r) => r.st !== 'off').length, totalCount: ifaceRows.length,
lastProbe: lastCall ? `${hm(lastCall.created_at)} · ${lastCall.ok ? `✓ ${lastCall.latency_ms}ms` : `× ${lastCall.error || 'ERROR'}`}` : '—',
latency: rows.length ? (rows.map((r) => r.last_latency_ms).filter((v) => v != null)[0] || null) : null,
error: rows.find((r) => r.last_error)?.last_error || '',
ifaceRows,
id: c.provider, name: c.label, roleText: PROV_ROLE_TEXT[c.provider] || c.role,
tag: c.credential_key ? 'licensed' : 'free',
bucket, credential: cred,
usedCount: c.interfaces.filter((i) => i.observed).length, totalCount: c.interfaces.length,
lastProbe: lastCall ? `${hm(lastCall.fetched_at)} ${lastCall.status === 'ok' ? `✓ ${lastCall.latency_ms ?? ''}ms` : `× ${lastCall.error || 'ERROR'}`}` : '—',
latency: observedRows.length ? (observedRows.map((r) => r.observed_latency_ms).filter((v) => v != null)[0] || null) : null,
error: observedRows.find((r) => r.observed_state !== 'ok' && r.observed_note)?.observed_note || '',
groups: groups.map((g) => ({ name: g.name, rows: ifaceRows(g) })),
};
});
return { cards, reserved, totalIfaces, usedIfaces };
@@ -501,36 +566,48 @@ function sourceCardHtml(card) {
// 靠 note 字段区分，这里必须原样展示 note，不能一律折叠成"已配置"。
const credBadge = card.credential.note ? card.credential.note :
card.credential.configured ? `已配置${card.credential.last4 ? ' · ' + card.credential.last4 : ''}` : '未配置';
const badges = [credBadge, card.role];
const badges = [credBadge];
if (card.credential.configured && card.credential.updated_at) badges.push(`${String(card.credential.updated_at).slice(5, 10).replace('-', '/')} 更新`);
badges.push(card.id === 'tushare' ? '主（无备）' : (card.id === 'ifind' ? '图表主源（未启用）' : (card.id === 'tencent' ? '实时层备' : '实时层主')));
let h = `<section class="panel" style="${ringStyle}">`;
h += `<header class="src-head">` +
ledHtml(ledSt, card.bucket === 'fail') +
`<span class="src-name">${esc(card.name)}</span>` +
`<span class="src-role">${esc(card.tag)}</span>` +
pillHtml(card.bucket) +
`<span class="src-role">${esc(card.roleText)}</span>` +
pillHtml(card.bucket, card.bucket === 'plan' ? '已配置 · 待观测' : '') +
`<span style="display:flex;gap:6px">${badges.map((b) => tagHtml(card.bucket === 'off' ? 'dashed' : 'dim', esc(b))).join('')}</span>` +
`<span class="flex1"></span>` +
`<button class="tbtn" data-probe="${card.id}"${ui.probing ? ' disabled' : ''}>${ui.probing ? '<span class="animate-spin" style="margin-right:4px">◌</span>探测中…' : '探测一次'}</button>` +
`<button class="tbtn" data-toggle="${card.id}">${ui.open ? '收起接口清单' : '展开接口清单'}</button>` +
`<button class="tbtn" data-probe="${card.id}"${ui.probing || card.bucket === 'off' ? ' disabled' : ''}>${ui.probing ? '<span class="animate-spin" style="margin-right:4px">◌</span>探测中…' : '探测一次'}</button>` +
(card.bucket === 'off'
? `<button class="tbtn" data-cred-hint="${card.id}">更新凭证</button>`
: `<button class="tbtn" data-toggle="${card.id}">${ui.open ? '收起接口清单' : '展开接口清单'}</button>`) +
`</header>`;
const probeCell = ui.probeResult
? `<span class="num" style="color:${ui.probeResult.ok ? '#34d399' : '#f87171'}">${ui.probeResult.t} ${ui.probeResult.ok ? `✓ ${ui.probeResult.ms}ms` : '× TIMEOUT'}</span>`
: `<span class="num"${card.lastProbe.includes('✓') ? ' style="color:#34d399"' : card.lastProbe.includes('×') ? ' style="color:#f87171"' : ''}>${esc(card.lastProbe)}</span>`;
h += `<div class="src-meta">` +
`<span>接口 <span class="num" style="color:#22d3ee">${card.usedCount}/${card.totalCount}</span> 在用</span>` +
`<span class="msep">|</span><span>最后探测 <span class="num" style="color:${card.lastProbe.includes('×') ? '#f87171' : '#8b9bb4'}">${esc(card.lastProbe)}</span></span>` +
`<span>接口 <span class="num" style="color:${card.bucket === 'off' ? '#54637e' : '#22d3ee'}">${card.usedCount}/${card.totalCount}</span> 在用</span>` +
`<span class="msep">|</span><span>最后探测 ${probeCell}</span>` +
`<span class="msep">|</span><span>延迟 ${latHtml(card.latency)}</span>` +
(card.error ? `<span class="msep">|</span><span style="color:#f87171">最近错误 ${esc(card.error)}</span>` : `<span class="msep">|</span><span style="color:#54637e">无最近错误</span>`) +
(card.error ? `<span class="msep">|</span><span style="color:#f87171">最近错误 ${esc(card.error)}</span>` : (card.bucket !== 'off' ? `<span class="msep">|</span><span style="color:#54637e">无最近错误</span>` : '')) +
`</div>`;
if (ui.open && card.ifaceRows.length) {
h += `<div class="mx-grid"><div style="min-width:0;grid-column:1/-1">` +
if (ui.open && card.bucket !== 'off') {
h += `<div class="mx-grid">` + card.groups.map((g) => `<div style="min-width:0">` +
`<div class="lab" style="margin-bottom:6px">${esc(g.name)}</div>` +
`<table class="dtable"><thead><tr><th>接口</th><th>用途</th><th>最近成功</th><th>延迟</th><th>状态</th></tr></thead><tbody>` +
card.ifaceRows.map((it) => `<tr>` +
g.rows.map((it) => `<tr>` +
`<td style="color:#22d3ee;font-size:11px;word-break:break-all;min-width:90px">${esc(it.api)}</td>` +
`<td style="color:#8b9bb4;font-size:11px;white-space:nowrap">${esc(it.use)}</td>` +
`<td class="num" style="font-size:11px;color:${it.st === 'fail' ? '#f87171' : '#8b9bb4'}">${esc(it.ok)}</td>` +
`<td class="num" style="font-size:11px;color:${it.ok.includes('×') || it.ok === '昨日' ? '#f87171' : '#8b9bb4'}">${esc(it.ok)}</td>` +
`<td style="font-size:11px">${latHtml(it.lat)}</td>` +
`<td style="white-space:nowrap"><span style="display:flex;align-items:center;gap:6px">${pillHtml(it.st)}${it.err ? `<span style="font-size:10px;color:#f87171">${esc(it.err)}</span>` : ''}</span></td>` +
`</tr>`).join('') + `</tbody></table></div></div>`;
} else if (!card.ifaceRows.length) {
h += `<div style="padding:16px"><div class="offbox">该来源尚无已登记接口</div></div>`;
`<td style="white-space:nowrap"><span style="display:flex;align-items:center;gap:6px">${pillHtml(it.st, it.st === 'plan' && it.obs === false ? '已配置 · 待观测' : '')}${it.err ? `<span style="font-size:10px;color:#f87171">${esc(it.err)}</span>` : ''}</span></td>` +
`</tr>`).join('') +
`</tbody></table></div>`).join('') + `</div>`;
}
if (card.bucket === 'off') {
h += `<div style="padding:16px"><div class="offbox">凭证未配置 · ${card.totalCount} 个预留接口处于停用状态 · 配置后将成为图表 / 问财 / 实时快照主源` +
`<div style="margin-top:8px;display:flex;justify-content:center;gap:6px;flex-wrap:wrap">${card.groups[0].rows.map((i) => tagHtml('dashed', esc(i.api))).join('')}</div>` +
`</div></div>`;
}
return h + `</section>`;
}
@@ -555,17 +632,57 @@ function sourcesHtml() {
function renderSourceCards() {
const el = $('srcCards');
if (!el) return;
const vm = buildSourcesVM();
el.innerHTML = vm.cards.map(sourceCardHtml).join('');
el.innerHTML = buildSourcesVM().cards.map(sourceCardHtml).join('');
paintSparks();
}
/* =====================================================================
数据血缘页
数据血缘页（接口级状态 + 已配置/待观测 + 真实频率 + 事件脉冲）
===================================================================== */
function pageNameOf(consumer) {
const idx = consumer.indexOf('(');
return (idx > 0 ? consumer.slice(0, idx) : consumer).trim();
}
/* 一条血缘项的服务接口集合（primary/backup 冒号后的真实接口名） */
function servingInterfaces(item) {
const out = new Set([item.dataset]);
[item.primary_source, item.backup_source].forEach((src) => {
if (!src) return;
src.split(':').slice(1).join(':').split('+').forEach((api) => {
const name = api.trim();
if (name) out.add(name);
});
});
return out;
}
/* 接口级状态：只看本数据集真正走的那几条接口，不用提供商级汇总顶替。 */
function lineageItemRowsHealth(item) {
const apis = servingInterfaces(item);
return (item.live_provider_health || []).filter((r) => apis.has(r.interface));
}
function lineageItemBucket(item) {
const rows = lineageItemRowsHealth(item);
if (item.tier === 'official') {
if (item.publication) return item.publication.state === 'published' ? 'ok' : (item.publication.state === 'degraded' ? 'degrade' : 'fail');
return rows.length ? worstBucket(rows.map((r) => bucketOf(r.state))) : 'plan';
}
if (item.tier === 'licensed') {
const cred = ((RAW.catalog && RAW.catalog.items) || []).find((c) => c.provider === 'ifind');
if (cred && !(cred.credential && cred.credential.configured)) return 'off';
return rows.length ? worstBucket(rows.map((r) => bucketOf(r.state))) : 'off';
}
// provisional：已有数据来源；有观测行看接口级状态，暂无观测行=已配置/待观测
return rows.length ? worstBucket(rows.map((r) => bucketOf(r.state))) : 'plan';
}
function lineageItemNote(item) {
const rows = lineageItemRowsHealth(item);
const note = rows.find((r) => r.last_error)?.last_error || rows.find((r) => r.last_fallback_reason)?.last_fallback_reason || '';
if (note) return note;
if (item.tier === 'licensed') return 'iFinD 未配置 · 功能隐藏';
if (item.tier === 'provisional' && !rows.length) return '已配置 · 暂无观测记录';
return '';
}
const linFilter = { onlyBad: false, src: '全部', q: '' };
let linHover = null;
let GW = 1500, GH = 286;
@@ -579,18 +696,12 @@ function buildLineageVM() {
const srcState = {};
const dsState = {};
const pageState = {};
const pageDatasets = {};
const edgesSD = [];
const edgesDP = [];
const rows = [];
items.forEach((it) => {
const rowsHealth = it.live_provider_health || [];
let bucket;
if (it.tier === 'official') {
bucket = it.publication ? (it.publication.state === 'published' ? 'ok' : (it.publication.state === 'degraded' ? 'degrade' : 'fail')) : (rowsHealth.length ? worstBucket(rowsHealth.map((r) => bucketOf(r.state))) : 'plan');
} else {
bucket = rowsHealth.length ? worstBucket(rowsHealth.map((r) => bucketOf(r.state))) : 'off';
}
const rowsHealth = lineageItemRowsHealth(it);
const bucket = lineageItemBucket(it);
dsState[it.dataset] = bucket;
(it.providers || []).forEach((p) => {
srcState[p] = worstBucket([srcState[p] || 'ok', ...rowsHealth.filter((r) => r.provider === p).map((r) => bucketOf(r.state))]);
@@ -601,16 +712,16 @@ function buildLineageVM() {
pages.forEach((pg) => {
pageState[pg] = worstBucket([pageState[pg] || 'ok', bucket]);
edgesDP.push([it.dataset, pg]);
(pageDatasets[pg] = pageDatasets[pg] || []).push(it);
});
const lat = rowsHealth.map((r) => r.last_latency_ms).find((v) => v != null);
const ok = it.tier === 'official' && it.publication ? it.publication.published_at : (rowsHealth.find((r) => r.last_ok_at) || {}).last_ok_at;
rows.push({
page: pages.join(' / ') || '—', item: consumers.join(' / '), dataset: it.dataset,
via: `${it.primary_source}${it.backup_source ? ' → ' + it.backup_source : ''}`,
role: it.backup_source ? '主/备' : (it.tier === 'licensed' ? '主（授权）' : '主（无备）'),
ok: ok ? hm(ok) : '—', lat, st: bucket,
note: rowsHealth.find((r) => r.last_error)?.last_error || rowsHealth.find((r) => r.last_fallback_reason)?.last_fallback_reason || (it.publication && it.publication.state !== 'published' ? '未正式发布' : ''),
freq: it.update_freq || '—',
ok: it.tier === 'official' && it.publication ? it.publication.published_at : (rowsHealth.find((r) => r.last_ok_at) || {}).last_ok_at,
lat: rowsHealth.map((r) => r.last_latency_ms).find((v) => v != null),
st: bucket,
note: lineageItemNote(it),
});
});
const sources = Object.keys(srcState).map((id) => ({ id, name: PROV_LABEL[id] || id, st: srcState[id] }));
@@ -641,6 +752,7 @@ function lineageGraphHtml(vm) {
if (vm.datasets.length) h += `<text x="${gpos['d:' + vm.datasets[0].id].x}" y="12" text-anchor="middle" ${colStyle}>数据集 · ${vm.datasets.length}</text>`;
if (vm.pages.length) h += `<text x="${gpos['p:' + vm.pages[0].id].x}" y="12" text-anchor="middle" ${colStyle}>主站页面 · ${vm.pages.length}</text>`;
const dsBy = {}; vm.datasets.forEach((d) => { dsBy[d.id] = d.st; });
// 已连接线路全部有可辨识的定向流动（flowline），颜色按数据集真实状态
vm.edgesSD.forEach(([s, d]) => {
if (!gpos['s:' + s] || !gpos['d:' + d]) return;
h += `<path data-link data-kind="sd" data-a="${esc(s)}" data-b="${esc(d)}" d="${linkPath(gpos['s:' + s], gpos['d:' + d])}" fill="none" stroke="${stColorOf(dsBy[d])}" stroke-width="1" stroke-opacity="0.28" class="${dsBy[d] === 'off' ? '' : 'flowline'}" style="transition:stroke-opacity .18s ease,stroke-width .18s ease"/>`;
@@ -716,35 +828,45 @@ function renderLinRows(vm) {
const rows = linRowsData(vm);
if (cnt) cnt.textContent = `${rows.length}/${vm.rows.length} 行`;
const cls = (st) => st === 'fail' ? 'row-fail' : st === 'degrade' || st === 'slow' ? 'row-slow' : st === 'off' ? 'row-off' : '';
const stLabel = (r) => (r.st === 'plan' && r.note.includes('待观测')) ? '已配置 · 待观测' : '';
body.innerHTML = rows.map((r) => `<tr class="${cls(r.st)}">` +
`<td style="color:#e8f1ff;white-space:nowrap">${esc(r.page)}</td>` +
`<td style="color:#8b9bb4;font-size:11px;white-space:nowrap">${esc(r.item)}</td>` +
`<td style="color:#22d3ee;font-size:11px">${esc(r.dataset)}</td>` +
`<td style="color:#8b9bb4;font-size:11px;max-width:260px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap" title="${esc(r.via)}">${esc(r.via)}</td>` +
`<td style="color:#54637e;font-size:11px;white-space:nowrap">${esc(r.role)}</td>` +
`<td class="num" style="font-size:11px;white-space:nowrap;color:#8b9bb4">${esc(r.ok)}</td>` +
`<td>${latHtml(r.lat)}</td>` +
`<td><span style="display:flex;align-items:center;gap:8px;white-space:nowrap">${pillHtml(r.st)}${r.note ? `<span style="font-size:10px;color:#54637e">${esc(r.note)}</span>` : ''}</span></td>` +
`</tr>`).join('') || `<tr><td colspan="8" class="empty-hint">无匹配记录 · 调整筛选条件</td></tr>`;
`<td class="num" style="color:#8b9bb4;font-size:11px;white-space:nowrap">${esc(r.freq)}</td>` +
`<td class="num" style="font-size:11px;white-space:nowrap;color:#8b9bb4">${r.ok ? esc(hm(r.ok)) : '—'}</td>` +
`<td class="num" style="font-size:11px;color:${r.lat != null && r.lat > 1000 ? '#fbbf24' : '#8b9bb4'}">${r.lat != null ? Number(r.lat).toLocaleString('en-US') + 'ms' : '—'}</td>` +
`<td><span style="display:flex;align-items:center;gap:8px;white-space:nowrap">${pillHtml(r.st, stLabel(r))}${r.note && !stLabel(r) ? `<span style="font-size:10px;color:#54637e">${esc(r.note)}</span>` : ''}</span></td>` +
`</tr>`).join('') || `<tr><td colspan="9" class="empty-hint">无匹配记录 · 调整筛选条件</td></tr>`;
if (bodyN) bodyN.innerHTML = rows.map((r) => `<tr class="${cls(r.st)}">` +
`<td style="color:#e8f1ff;white-space:nowrap">${esc(r.page)}<span class="sub" style="color:#8b9bb4">${esc(r.item)}</span></td>` +
`<td style="color:#22d3ee;font-size:11px">${esc(r.dataset)}</td>` +
`<td style="color:#8b9bb4;font-size:11px;word-break:break-all">${esc(r.via)}</td>` +
`<td class="num" style="font-size:11px;white-space:nowrap;color:#8b9bb4">${esc(r.ok)}<span class="sub">${r.lat != null ? r.lat + 'ms' : '—'}</span></td>` +
`<td>${pillHtml(r.st)}${r.note ? `<span class="sub">${esc(r.note)}</span>` : ''}</td>` +
`</tr>`).join('') || `<tr><td colspan="5" class="empty-hint">无匹配记录 · 调整筛选条件</td></tr>`;
`<td class="num" style="color:#8b9bb4;font-size:11px;white-space:nowrap">${esc(r.freq)}<span class="sub">${esc(r.role)}</span></td>` +
`<td class="num" style="font-size:11px;white-space:nowrap;color:#8b9bb4">${r.ok ? esc(hm(r.ok)) : '—'}<span class="sub">${r.lat != null ? r.lat + 'ms' : '—'}</span></td>` +
`<td>${pillHtml(r.st, stLabel(r))}${r.note && !stLabel(r) ? `<span class="sub">${esc(r.note)}</span>` : ''}</td>` +
`</tr>`).join('') || `<tr><td colspan="6" class="empty-hint">无匹配记录 · 调整筛选条件</td></tr>`;
}
function lineageHtml() {
const vm = buildLineageVM();
const anomalyPages = new Set();
vm.rows.forEach((r) => { if (r.st === 'fail') (r.page || '').split(' / ').forEach((p) => anomalyPages.add(p)); });
const failDs = vm.datasets.filter((d) => d.st === 'fail');
let impactTag;
if (failDs.length) {
const pages = new Set();
failDs.forEach((d) => { (adj.dp.get(d.id) || new Set()).forEach((p) => pages.add(p)); });
impactTag = tagHtml('rd', `${failDs.map((d) => d.id).join('/')} 断链影响 ${pages.size} 页`);
} else {
impactTag = tagHtml('mint', '暂无断链影响');
}
const html = `<div class="col">` +
`<div class="panel strip" style="gap:8px 20px">` +
`<span style="font-size:13px;color:#e8f1ff">血缘地图</span>` +
`<span class="num" style="font-size:11px;color:#8b9bb4"><span style="color:#22d3ee">${vm.sources.length}</span> 源 → <span style="color:#22d3ee">${vm.datasets.length}</span> 数据集 → <span style="color:#22d3ee">${vm.pages.length}</span> 页面</span>` +
`<span class="legend" style="color:#54637e;letter-spacing:0"><span>${ledHtml('ok')}健康</span><span>${ledHtml('slow')}慢/降级</span><span>${ledHtml('fail')}失败</span><span>${ledHtml('plan')}计划</span><span>${ledHtml('off')}未配置</span></span>` +
`<span class="flex1"></span>` +
(anomalyPages.size ? tagHtml('rd', `${[...anomalyPages][0]} 断链影响 ${anomalyPages.size} 页`) : tagHtml('mint', '暂无断链影响')) +
impactTag +
`</div>` +
`<section class="panel"><div style="position:relative">${lineageGraphHtml(vm)}</div></section>` +
`<section class="panel">` +
@@ -755,11 +877,10 @@ function lineageHtml() {
`<span class="num" style="font-size:10px;color:#54637e" id="linCount"></span>` +
`<input class="lin-q" id="linQ" placeholder="搜索页面 / 接口 / 数据集" value="${esc(linFilter.q)}">` +
`</div>` +
`<div class="lin-scroll"><table class="dtable lin-table"><thead><tr><th>主站页面</th><th>数据项</th><th>数据集</th><th>源</th><th>主/备</th><th>最近成功</th><th>延迟</th><th>状态</th></tr></thead><tbody id="linBody"></tbody></table>` +
`<table class="dtable lin-table-n"><thead><tr><th>主站页面</th><th>数据集</th><th>源</th><th>最近成功</th><th>状态</th></tr></thead><tbody id="linBodyN"></tbody></table></div>` +
`<div class="lin-scroll"><table class="dtable lin-table"><thead><tr><th>主站页面</th><th>数据项</th><th>数据集</th><th>源 : 接口</th><th>主/备</th><th>更新频率</th><th>最近成功</th><th>延迟</th><th>状态</th></tr></thead><tbody id="linBody"></tbody></table>` +
`<table class="dtable lin-table-n"><thead><tr><th>主站页面</th><th>数据集</th><th>源 : 接口</th><th>更新频率</th><th>最近成功</th><th>状态</th></tr></thead><tbody id="linBodyN"></tbody></table></div>` +
`</section>` +
`</div>`;
window.__linVM = vm;
return html;
}
@@ -775,16 +896,19 @@ function renderPage(p) {
state.page = p;
mainEl.dataset.page = p;
document.querySelectorAll('[data-nav]').forEach((b) => b.classList.toggle('act', b.dataset.nav === p));
if (p === 'overview') mainEl.innerHTML = overviewHtml();
else if (p === 'sources') mainEl.innerHTML = sourcesHtml();
if (p === 'overview') { mainEl.innerHTML = overviewHtml(); paintSparks(); flashChangedLatencies(); }
else if (p === 'sources') { mainEl.innerHTML = sourcesHtml(); paintSparks(); }
else {
mainEl.innerHTML = lineageHtml();
renderLinRows(window.__linVM);
applyLinHover(window.__linVM);
const vm = buildLineageVM();
window.__linVM = vm;
renderLinRows(vm);
applyLinHover(vm);
const svg = $('linSvg');
if (svg) svg.addEventListener('mouseleave', () => setLinHover(null, window.__linVM));
fitLineageTable();
}
window.scrollTo(0, 0);
updatePhaseTag();
}
window.addEventListener('hashchange', () => renderPage(pageFromHash()));
@@ -795,11 +919,10 @@ window.addEventListener('resize', () => {
resizeT = setTimeout(() => { if (state.page === 'lineage') renderPage('lineage'); }, 180);
});
/* HEL-529: 血缘详情表随真实数据集数量（当前 17 条，非打样假设的 13 条）
动态撑高；宽档（>1100px，覆盖 1920×920/1440×900 验收位）给表格容器一个
基于剩余可视高度计算的 max-height + 内部滚动，保证整页 scrollHeight
始终 <= clientHeight（硬性验收项），同时全部行仍可在容器内滚动查看。
窄档（<=1100px）保持原有整页纵向滚动，不设上限。 */
/* HEL-529: 血缘详情表随真实数据集数量动态撑高；宽档（>1100px，覆盖
1920×920/1440×900 验收位）给表格容器一个基于剩余可视高度计算的
max-height + 内部滚动，保证整页 scrollHeight 始终 <= clientHeight，
同时全部行仍可在容器内滚动查看。窄档（<=1100px）整页纵滚。 */
function fitLineageTable() {
const wrap = document.querySelector('.lin-scroll');
const tape = document.querySelector('.tape');
@@ -816,6 +939,8 @@ function fitLineageTable() {
function updatePhaseTag() {
const ov = RAW.overview;
if (!ov) return;
const bad = (ov.anomalies || []).length > 0;
$('radarBlipBad').style.display = bad ? '' : 'none';
$('phaseTag').innerHTML = `${ledHtml('rev', true)} ${esc(ov.session_phase || '')}`;
}
@@ -824,6 +949,11 @@ mainEl.addEventListener('click', (e) => {
if (pb) { doProbe(pb.dataset.probe); return; }
const tg = e.target.closest('[data-toggle]');
if (tg) { const ui = srcUiState[tg.dataset.toggle]; if (ui) { ui.open = !ui.open; renderSourceCards(); } return; }
const ch = e.target.closest('[data-cred-hint]');
if (ch) {
toast('iFinD 凭证需在部署侧配置（DATAHUB_IFIND_* 环境变量或 /v1/credentials/ifind），后台暂未开放在线写入', 'err');
return;
}
const goto = e.target.closest('[data-goto-lineage]');
if (goto) { linFilter.q = ''; location.hash = 'lineage'; return; }
const inc = e.target.closest('[data-incident-action]');
@@ -846,14 +976,35 @@ async function doProbe(id) {
if (!ui || ui.probing) return;
ui.probing = true;
renderSourceCards();
try { await api(`/admin/api/sources/${id}/probe`, { method: 'POST', body: '{}' }); } catch (err) { toast(err.message, 'err'); }
try {
await api(`/admin/api/sources/${id}/probe`, { method: 'POST', body: '{}' });
ui.probeResult = { t: `${p2(new Date().getHours())}:${p2(new Date().getMinutes())}`, ok: true, ms: null };
} catch (err) {
ui.probeResult = { t: `${p2(new Date().getHours())}:${p2(new Date().getMinutes())}`, ok: false, ms: null };
toast(err.message, 'err');
}
ui.probing = false;
await refreshQuiet();
renderSourceCards();
}
/* ---------- EVENT TAPE：真实事件差量渲染 ---------- */
const tapeSeen = { call: 0, provCall: 0, audit: 0, job: 0, first: true };
/* ---------- EVENT TAPE：预装最近真实事件 + 持续滚动 + 新事件闪动 ---------- */
const tapeSeen = { call: 0, provCall: 0, audit: 0, job: 0 };
let tapeBuf = [];
let tapeSeeded = false;
function tapeEventHtml(e, fresh) {
return `<span class="tape-item${fresh ? ' flash' : ''}"><span class="tt num">${esc(hm(e.ts))}</span><span class="ts">${esc(e.src)}</span><span class="tm">${esc(e.msg)}</span>` +
(e.ok ? `<span class="tok num">✓${e.ms ? ` ${e.ms}ms` : ''}</span>` : '<span class="tbad">× 失败</span>') +
`<span class="tsep">///</span></span>`;
}
function renderTape(freshIds) {
const items = tapeBuf.slice(-14);
const html = items.length
? items.map((e) => tapeEventHtml(e, freshIds && freshIds.has(e.sortId))).join('')
: `<span class="tape-item"><span class="tm" style="color:#54637e">暂无真实事件 · 等待首次调用/任务</span><span class="tsep">///</span></span>`;
$('tapeA').innerHTML = html;
$('tapeB').innerHTML = html;
}
function tapeEventsFromRaw() {
const evs = [];
((RAW.overview && RAW.overview.recent_calls) || []).forEach((c) => { if (c.id > tapeSeen.call) evs.push({ ts: c.created_at, src: 'tushare', msg: c.endpoint, ok: c.ok, ms: c.latency_ms, sortId: 'c' + c.id }); });
@@ -867,16 +1018,25 @@ function tapeEventsFromRaw() {
tapeSeen.job = Math.max(tapeSeen.job, maxOf((RAW.jobs && RAW.jobs.runs) || [], 'id'));
return evs;
}
let tapeBuf = [];
function seedTape() {
// 首次加载：预装最近的真实事件（打样 EVENT TAPE 开场即有内容滚动）
const evs = [];
((RAW.providers && RAW.providers.recent_calls) || []).forEach((c) => evs.push({ ts: c.fetched_at, src: c.provider, msg: c.interface, ok: c.status === 'ok', ms: c.latency_ms, sortId: 'p' + c.id }));
((RAW.overview && RAW.overview.recent_calls) || []).forEach((c) => evs.push({ ts: c.created_at, src: 'tushare', msg: c.endpoint, ok: c.ok, ms: c.latency_ms, sortId: 'c' + c.id }));
((RAW.jobs && RAW.jobs.runs) || []).forEach((r) => evs.push({ ts: r.finished_at || r.started_at, src: 'scheduler', msg: `${r.job_id} ${r.state}`, ok: r.state !== 'failed', sortId: 'j' + r.id }));
((RAW.audit && RAW.audit.items) || []).forEach((a) => evs.push({ ts: a.created_at, src: 'hub_admin', msg: `${a.action} ${a.target || ''}`.trim(), ok: true, sortId: 'a' + a.id }));
evs.sort((x, y) => String(x.ts).localeCompare(String(y.ts)));
tapeBuf = evs.slice(-14);
tapeSeeded = true;
renderTape(null);
}
function updateTape() {
if (!tapeSeeded) { seedTape(); return; }
const evs = tapeEventsFromRaw();
if (tapeSeen.first) { tapeSeen.first = false; return; } // 首次加载只建基线，不补播历史事件
if (!evs.length) return;
const fresh = new Set(evs.map((e) => e.sortId));
tapeBuf = tapeBuf.concat(evs).slice(-16);
const html = tapeBuf.map((e) => `<span class="tape-item"><span class="tt num">${esc(hm(e.ts))}</span><span class="ts">${esc(e.src)}</span><span class="tm">${esc(e.msg)}</span>` +
(e.ok ? `<span class="tok num">✓${e.ms ? ` ${e.ms}ms` : ''}</span>` : '<span class="tbad">× 失败</span>') +
`<span class="tsep">///</span></span>`).join('');
$('tapeA').innerHTML = html; $('tapeB').innerHTML = html;
renderTape(fresh);
}
/* ---------- 抽屉：调度任务 / 盘后发布 / 审计 ---------- */
@@ -1011,18 +1171,22 @@ function toast(msg, kind) {
toastT = setTimeout(() => { el.classList.remove('show'); }, 3200);
}
/* ================= 顶栏：时钟 / 减少动态 / 主题 ================= */
/* ================= 顶栏：时钟 / 减少动态 ================= */
let clockEls = null;
function updateClock() {
if (!clockEls) clockEls = { d: $('ckD'), h: $('ckH'), m: $('ckM'), s: $('ckS') };
const now = new Date();
$('ckD').textContent = `${now.getFullYear()}-${p2(now.getMonth() + 1)}-${p2(now.getDate())}`;
$('ckH').textContent = p2(now.getHours());
$('ckM').textContent = p2(now.getMinutes());
$('ckS').textContent = p2(now.getSeconds());
const ds = `${now.getFullYear()}-${p2(now.getMonth() + 1)}-${p2(now.getDate())}`;
if (clockEls.d.textContent !== ds) clockEls.d.textContent = ds;
const hs = p2(now.getHours()), ms = p2(now.getMinutes()), ss = p2(now.getSeconds());
if (clockEls.h.textContent !== hs) clockEls.h.textContent = hs;
if (clockEls.m.textContent !== ms) clockEls.m.textContent = ms;
if (clockEls.s.textContent !== ss) clockEls.s.textContent = ss;
}
function applyCalm(on) {
state.calm = on;
// 挂在 #app（覆盖 appRoot 及其兄弟 drawer/modal/toast），不能只挂 #appRoot：
// 否则 .reduce-motion * 与 .reduce-motion .drawer/.toast 选不中旁路层过渡。
// 挂在 body + #app（覆盖 appRoot 及其兄弟 drawer/modal/toast），否则
// .reduce-motion * 选不中旁路层过渡。
document.body.classList.toggle('reduce-motion', on);
const app = $('app');
if (app) app.classList.toggle('reduce-motion', on);
@@ -1037,7 +1201,62 @@ if (window.matchMedia) {
mq.addEventListener ? mq.addEventListener('change', syncMq) : mq.addListener(syncMq);
}
/* ================= 动态更新：200ms tick（打样节奏） =================
- 时钟每秒翻字（冒号 blink 由 CSS 承担）
- 缓存年龄依据真实时间戳每秒增长（>30s 变琥珀）
- 倒计时按真实 next_retry_at 每秒递减
- sparkline 端点闪烁由 CSS 承担，折线只在真实历史变化时重画
- 延迟数字只在真实值变化时闪一下（flash），绝不随机伪造数值 */
function paintSparks() {
document.querySelectorAll('svg.spark').forEach((svg) => sparkPaint(svg));
}
const prevLat = new Map();
function flashChangedLatencies() {
document.querySelectorAll('[data-lat]').forEach((el) => {
const jid = el.closest('[data-jid]') ? el.closest('[data-jid]').dataset.jid : (el.closest('tr') ? 'row:' + [...el.closest('tr').children].indexOf(el) : '');
const v = el.dataset.lat;
const key = state.page + ':' + (el.closest('[data-dataset]')?.dataset.dataset || jid) + ':' + v;
const prev = prevLat.get(el);
if (prev !== undefined && prev !== v) {
el.classList.remove('flash');
void el.offsetWidth;
el.classList.add('flash');
}
prevLat.set(el, v);
});
}
function updateDynamics() {
const now = Date.now();
document.querySelectorAll('[data-age-epoch]').forEach((el) => {
const ms = +el.dataset.ageEpoch;
if (!ms || Number.isNaN(ms)) return;
const s = Math.max(0, Math.floor((now - ms) / 1000));
const txt = fmtAgeSec(s);
if (el.textContent !== txt) el.textContent = txt;
const color = s > 30 ? '#fbbf24' : '#8b9bb4';
if (el.style.color !== color) el.style.color = color;
});
document.querySelectorAll('[data-cd-epoch]').forEach((el) => {
const target = +el.dataset.cdEpoch;
if (!target || Number.isNaN(target)) return;
const left = Math.floor((target - now) / 1000);
if (left > 0) {
const m = Math.floor(left / 60), s = left % 60;
const txt = el.dataset.cdfmt === 'ms' ? `${m}m${p2(s)}s` : `${p2(m)}:${p2(s)}`;
if (el.textContent !== txt) el.textContent = txt;
} else if (el.textContent !== '已到点') {
el.textContent = '已到点';
}
});
}
/* ================= 轮询：拉取真实数据、驱动渲染、可见性/网络门禁 ================= */
const dataVersion = { overview: '', sources: '', lineage: '' };
function snapshotVersion(part) {
if (part === 'overview') return JSON.stringify([RAW.overview, RAW.batches && RAW.batches.batches, RAW.providers && RAW.providers.health, RAW.catalog]);
if (part === 'sources') return JSON.stringify([RAW.catalog, RAW.providers && RAW.providers.recent_calls, RAW.sources]);
return JSON.stringify(RAW.lineage);
}
const Poller = (() => {
let timer = null;
let running = false;
@@ -1047,9 +1266,32 @@ const Poller = (() => {
try {
await fetchAll();
updateTape();
if (state.page === 'overview') mainEl.innerHTML = overviewHtml();
else if (state.page === 'sources') renderSourceCards();
else if (state.page === 'lineage') { /* 血缘图交互态复杂，仅在用户操作或下次轮询后整体重绘一次 */ mainEl.innerHTML = lineageHtml(); renderLinRows(window.__linVM); applyLinHover(window.__linVM); fitLineageTable(); }
const vOv = snapshotVersion('overview'), vSrc = snapshotVersion('sources'), vLin = snapshotVersion('lineage');
if (state.page === 'overview' && vOv !== dataVersion.overview) {
dataVersion.overview = vOv;
mainEl.innerHTML = overviewHtml();
paintSparks();
flashChangedLatencies();
} else if (state.page === 'overview') {
flashChangedLatencies();
}
if (state.page === 'sources' && vSrc !== dataVersion.sources) {
dataVersion.sources = vSrc;
renderSourceCards();
}
if (state.page === 'lineage' && vLin !== dataVersion.lineage) {
dataVersion.lineage = vLin;
mainEl.innerHTML = lineageHtml();
const vm = buildLineageVM();
window.__linVM = vm;
renderLinRows(vm);
applyLinHover(vm);
const svg = $('linSvg');
if (svg) svg.addEventListener('mouseleave', () => setLinHover(null, window.__linVM));
fitLineageTable();
}
// 真实调用事件脉冲：血缘页上对应来源节点亮一圈 ledPing 节奏的光环
pulseSourcesOnNewCalls();
updatePhaseTag();
setNetOk(true);
} catch (err) {
@@ -1072,10 +1314,39 @@ const Poller = (() => {
};
})();
/* 真实调用脉冲：provider_call_log 出现新调用时，给血缘图对应来源节点
叠一圈与 ledPing 同节奏（1.6s ease-out）的扩散光环；这是事件动效，
与 gnode-pulse 的静态呼吸不同，一次即消散。 */
let pulseSeenCall = 0;
function pulseSourcesOnNewCalls() {
if (state.page !== 'lineage') {
pulseSeenCall = ((RAW.providers && RAW.providers.recent_calls) || []).reduce((m, c) => Math.max(m, c.id || 0), pulseSeenCall);
return;
}
const calls = (RAW.providers && RAW.providers.recent_calls) || [];
const fresh = calls.filter((c) => (c.id || 0) > pulseSeenCall);
pulseSeenCall = calls.reduce((m, c) => Math.max(m, c.id || 0), pulseSeenCall);
fresh.forEach((c) => {
const g = document.querySelector(`#linSvg g[data-nid="${CSS.escape(c.provider)}"]`);
if (!g) return;
const circle = g.querySelector('circle');
if (!circle) return;
const ring = document.createElementNS('http://www.w3.org/2000/svg', 'circle');
ring.setAttribute('cx', circle.getAttribute('cx'));
ring.setAttribute('cy', circle.getAttribute('cy'));
ring.setAttribute('r', '5');
ring.setAttribute('fill', 'none');
ring.setAttribute('stroke', '#22d3ee');
ring.setAttribute('stroke-width', '1.2');
ring.setAttribute('class', 'node-ping');
g.appendChild(ring);
setTimeout(() => ring.remove(), 1700);
});
}
function resetTapeBaselines() {
// 恢复轮询时静默重建基线，不补播暂停期间错过的事件。
// 恢复轮询时静默重建基线，不补播暂停期间错过的事件（但保留已上带内容）。
tapeSeen.call = tapeSeen.provCall = tapeSeen.audit = tapeSeen.job = Number.MAX_SAFE_INTEGER;
tapeSeen.first = true;
}
document.addEventListener('visibilitychange', () => {
if (document.hidden) { Poller.stop(); }
@@ -1090,11 +1361,15 @@ async function refreshQuiet() {
/* ================= 启动 ================= */
async function startApp() {
setInterval(updateClock, 1000);
updateClock();
await fetchAll();
tapeSeen.first = true;
dataVersion.overview = snapshotVersion('overview');
dataVersion.sources = snapshotVersion('sources');
dataVersion.lineage = snapshotVersion('lineage');
pulseSeenCall = ((RAW.providers && RAW.providers.recent_calls) || []).reduce((m, c) => Math.max(m, c.id || 0), 0);
renderPage(pageFromHash());
updateTape(); // 预装最近真实事件并开始滚动
updateClock();
setInterval(() => { updateClock(); updateDynamics(); }, 200); // 打样 tick 节奏
await Poller.start();
}
xiaobai-datahub/admin/index.html
+5 -3
View File
@@ -46,8 +46,10 @@
<div class="ring1" style="inset:5.72px"></div>
<div class="ring2" style="inset:9.88px"></div>
<div class="center"></div>
<div class="blip pulse-dot" id="radarBlipOk" style="left:64%;top:30%;background:#34d399"></div>
<div class="blip pulse-dot" id="radarBlipBad" style="left:30%;top:62%;background:#f87171;animation-delay:.6s;display:none"></div>
</div>
<div class="brand"><span class="b1">小白复盘 <span style="color:#22d3ee">·</span> 数据中枢</span><span class="b2">DATA-HUB · HEL-529</span></div>
<div class="brand"><span class="b1">小白复盘 <span style="color:#22d3ee">·</span> 数据中枢</span><span class="b2">DATA-HUB</span></div>
<span class="vsep"></span>
<nav class="nav" id="navEl">
<button class="navbtn" data-nav="overview"><span class="tri">▸</span>运行总览<span class="en">OVERVIEW</span></button>
@@ -57,7 +59,7 @@
<span class="flex1"></span>
<span class="mdtag" id="phaseTag"></span>
<span class="livespan" id="liveSpan"><span class="livedot pulse-dot"></span>LIVE</span>
<span class="clock num" id="clock"><span id="ckD"></span><span class="csep">|</span><span class="ct"><span id="ckH"></span><span class="csep">:</span><span id="ckM"></span><span class="csep">:</span><span id="ckS"></span></span></span>
<span class="clock num" id="clock"><span id="ckD"></span><span class="csep">|</span><span class="ct"><span id="ckH"></span><span class="blink cc">:</span><span id="ckM"></span><span class="blink cc">:</span><span id="ckS"></span></span></span>
<span class="vsep"></span>
<button class="tbtn" id="opsBtn" style="font-size:10px">调度 / 发布 / 审计</button>
<button class="tbtn" id="calmBtn" style="font-size:10px">减少动态</button>
@@ -69,7 +71,7 @@
<main class="main" id="mainEl"></main>
<footer class="tape">
<span class="tape-label" id="tapeLed">EVENT TAPE</span>
<span class="tape-label" id="tapeLed"><span class="led rev pulse"></span>EVENT TAPE</span>
<div class="tape-view"><div class="marquee-track" id="marqueeTrack"><span style="display:inline-flex;align-items:center" id="tapeA"></span><span style="display:inline-flex;align-items:center" id="tapeB"></span></div></div>
</footer>
</div>
xiaobai-datahub/admin/styles.css
+8
View File
@@ -367,6 +367,14 @@ body {
.lin-hint { position: absolute; left: 24px; bottom: 4px; font-size: 10px; color: #54637e; letter-spacing: .1em; }
@keyframes gnodePulse { 0%,100% { opacity: 1; } 50% { opacity: .55; } }
.gnode-pulse { animation: gnodePulse 1.8s ease-in-out infinite; }
/* 真实调用事件脉冲（HEL-529 数据修正第 6 项）：血缘图来源节点在真实调用
到达时叠一圈与 ledPing 同节奏（1.6s ease-out 放大淡出）的光环；
事件动效，一次即消散，与 gnode-pulse 的静态呼吸区分。 */
.node-ping { transform-box: fill-box; transform-origin: center; animation: nodePing 1.6s ease-out forwards; }
@keyframes nodePing {
0% { transform: scale(.4); opacity: .8; }
80%, 100% { transform: scale(3.2); opacity: 0; }
}
.lin-filter { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; padding: 10px 12px; border-bottom: 1px solid #1a2540; }
.lin-sel, .lin-q { background: #0c1220; border: 1px solid #243152; border-radius: 4px; font-size: 11px; color: #8b9bb4; padding: 6px 8px; outline: none; font-family: inherit; }
.lin-q { color: #d7e1f0; padding: 6px 10px; width: 220px; }
xiaobai-datahub/datahub/admin_api.py
+13 -5
View File
@@ -31,10 +31,18 @@ class AdminAPI:
)
is_open = bool(cal and int(cal["is_open"]) == 1)
pubs = self.db.fetchall("SELECT * FROM publications WHERE trade_date = ?", (today,))
failed = self.db.fetchall(
"SELECT * FROM batches WHERE trade_date = ? AND state IN ('failed','staged')",
(today,),
)
# HEL-529 fix 2: "待处理"只保留当前仍未恢复的最新异常。一条失败/停滞
# 批次若已被同数据集更晚的成功批次或当日发布解决，就不再是当前故障，
# 历史记录仍完整保留在 batches/audit 明细里，不在此重复展示。
latest_by_dataset: dict[str, dict[str, Any]] = {}
for row in self.db.fetchall("SELECT * FROM batches WHERE trade_date = ? ORDER BY started_at, batch_id", (today,)):
latest_by_dataset[row["dataset"]] = row
published_datasets = {row["dataset"] for row in pubs if row["state"] == "published"}
anomalies = [
row
for dataset, row in latest_by_dataset.items()
if dataset not in published_datasets and row["state"] in ("failed", "staged")
]
calls = self.db.fetchall(
"SELECT * FROM src_calls ORDER BY id DESC LIMIT 20",
)
@@ -45,7 +53,7 @@ class AdminAPI:
"eod_status": self.scheduler.eod_status(today),
"revision_status": self.scheduler.revision_status(today),
"publications": pubs,
"anomalies": failed,
"anomalies": anomalies,
"recent_calls": _public_calls(calls),
"source_count": len(self.db.fetchall("SELECT provider FROM src_health")),
}
xiaobai-datahub/datahub/lineage.py
+17
View File
@@ -38,6 +38,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "calendar",
"tier": "official",
"update_freq": "每日 08:45 预检",
"v1_endpoint": "/v1/calendar",
"primary_source": "tushare:trade_cal",
"backup_source": None,
@@ -46,6 +47,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "stocks",
"tier": "official",
"update_freq": "每日 20:00 / 23:10",
"v1_endpoint": "/v1/stocks",
"primary_source": "tushare:stock_basic",
"backup_source": None,
@@ -54,6 +56,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "daily",
"tier": "official",
"update_freq": "盘后 15:05 · 重试至 23:30",
"v1_endpoint": "/v1/bars/daily",
"primary_source": "tushare:daily",
"backup_source": None,
@@ -62,6 +65,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "valuation",
"tier": "official",
"update_freq": "盘后 15:05 · 复核 20:00",
"v1_endpoint": "/v1/valuation",
"primary_source": "tushare:daily_basic",
"backup_source": None,
@@ -70,6 +74,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "moneyflow",
"tier": "official",
"update_freq": "盘后 15:05 · 重试至 23:30",
"v1_endpoint": "/v1/moneyflow",
"primary_source": "tushare:moneyflow",
"backup_source": None,
@@ -78,6 +83,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "auction",
"tier": "official",
"update_freq": "盘后 15:05",
"v1_endpoint": "/v1/auction",
"primary_source": "tushare:stk_auction",
"backup_source": None,
@@ -86,6 +92,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "index_daily",
"tier": "official",
"update_freq": "盘后 15:10 · 重试至 23:30",
"v1_endpoint": "/v1/indexes/bars",
"primary_source": "tushare:index_daily",
"backup_source": None,
@@ -94,6 +101,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "limit_events",
"tier": "official",
"update_freq": "盘后 16:40 · 重试至 23:30",
"v1_endpoint": "/v1/limit-events",
"primary_source": "tushare:limit_list_d",
"backup_source": None,
@@ -102,6 +110,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "popularity",
"tier": "official",
"update_freq": "盘后 22:40",
"v1_endpoint": "/v1/popularity",
"primary_source": "tushare:ths_hot+dc_hot",
"backup_source": None,
@@ -110,6 +119,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "dragon_tiger",
"tier": "official",
"update_freq": "盘后 16:45 · 重试至 23:30",
"v1_endpoint": "/v1/dragon-tiger",
"primary_source": "tushare:hm_detail",
"backup_source": None,
@@ -118,6 +128,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "sector_daily",
"tier": "official",
"update_freq": "盘后 15:20 · 重试至 23:30",
"v1_endpoint": "/v1/sectors",
"primary_source": "tushare:ths_daily+dc_index+sw_daily",
"backup_source": None,
@@ -126,6 +137,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "quotes_latest",
"tier": "provisional",
"update_freq": "盘中 · 缓存 60s",
"v1_endpoint": "/v1/quotes/latest",
"primary_source": "eastmoney:ulist/clist",
"backup_source": "tencent:qt",
@@ -134,6 +146,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "index_quotes",
"tier": "provisional",
"update_freq": "盘中 · 缓存 60s",
"v1_endpoint": "/v1/indexes/quotes",
"primary_source": "eastmoney:ulist",
"backup_source": "tencent:qt",
@@ -142,6 +155,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "sectors_quote",
"tier": "provisional",
"update_freq": "盘中 · 缓存 60s",
"v1_endpoint": "/v1/sectors/quote",
"primary_source": "eastmoney:sw",
"backup_source": None,
@@ -150,6 +164,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "limit_pool",
"tier": "provisional",
"update_freq": "盘中 · 缓存 60s",
"v1_endpoint": "/v1/limit-pool",
"primary_source": "eastmoney:zt_pool",
"backup_source": None,
@@ -158,6 +173,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "intraday_points",
"tier": "provisional",
"update_freq": "盘中 · 缓存 20s",
"v1_endpoint": "/v1/intraday/points",
"primary_source": "eastmoney:trends2",
"backup_source": None,
@@ -166,6 +182,7 @@ DATASETS: list[dict[str, Any]] = [
{
"dataset": "ifind_wencai",
"tier": "licensed",
"update_freq": "按需调用",
"v1_endpoint": "/v1/query (api_name=ifind_wencai)",
"primary_source": "ifind:smart_stock_picking",
"backup_source": None,
xiaobai-datahub/datahub/pipeline.py
+53
View File
@@ -99,6 +99,58 @@ STAGING_INSERT = {
**EXTENDED_STAGING_INSERT,
}
# Staging-table business keys (PRIMARY KEY minus the constant batch_id),
# mirroring the PRIMARY KEY clauses declared in datahub/db.py and
# datahub/datasets_ext.py. Used only to collapse within-batch duplicates so
# a single upstream response cannot fail the whole batch on a UNIQUE
# constraint (HEL-529: 2026-09-14 dc_hot returned 4 duplicate ts_codes and
# hm_detail 43 duplicate (ts_code, hm_name) keys in one response, which has
# blocked popularity/dragon_tiger publishing every day since 09-07).
STAGING_KEY_FIELDS = {
"stocks": ("ts_code", "trade_date"),
"daily": ("ts_code", "trade_date"),
"valuation": ("ts_code", "trade_date"),
"moneyflow": ("ts_code", "trade_date"),
"auction": ("ts_code", "trade_date"),
"index_daily": ("ts_code", "trade_date"),
"limit_events": ("ts_code", "trade_date", "limit_type"),
"popularity": ("ts_code", "trade_date", "source"),
"dragon_tiger": ("ts_code", "trade_date", "hm_name"),
"sector_daily": ("ts_code", "trade_date", "family"),
}
def _dedupe_staging_rows(dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Collapse within-batch duplicates on the staging table's business key.
Deterministic: keeps the LAST occurrence of each key (the same row the
eod copy's INSERT OR REPLACE would keep), preserves first-seen order, and
never touches rows across batches. Datasets without a declared business
key are returned unchanged.
"""
fields = STAGING_KEY_FIELDS.get(dataset)
if not fields:
return rows
seen: dict[tuple, int] = {}
out: list[dict[str, Any]] = []
dropped = 0
for row in rows:
key = tuple(row.get(field) for field in fields)
if key in seen:
out[seen[key]] = row
dropped += 1
else:
seen[key] = len(out)
out.append(row)
if dropped:
LOGGER.info(
"staging dedupe: %s collapsed %d duplicate rows within batch (kept last)",
dataset,
dropped,
extra={"hub": {"dataset": dataset, "deduped": dropped}},
)
return out
EOD_COPY = {
"stocks": (
"INSERT OR REPLACE INTO eod_stocks "
@@ -1720,6 +1772,7 @@ class Pipeline:
def _stage(self, dataset: str, batch_id: str, rows: list[dict[str, Any]]) -> None:
sql, mapper = STAGING_INSERT[dataset]
rows = _dedupe_staging_rows(dataset, rows)
with self.db.write() as connection:
connection.execute(
f"DELETE FROM {DATASET_TABLES[dataset][1]} WHERE batch_id = ?",
xiaobai-datahub/datahub/source_catalog.py
+60 -29
View File
@@ -31,21 +31,21 @@ CATALOG: list[dict[str, Any]] = [
"credential_key": "tushare_token",
"status_source": "src_health (legacy, kept) + provider_health (unified, HEL-543)",
"interfaces": [
{"interface": "trade_cal", "capability": "交易日历", "datasets": ["calendar"]},
{"interface": "stock_basic", "capability": "股票主档", "datasets": ["stocks"]},
{"interface": "daily", "capability": "个股日K", "datasets": ["daily"]},
{"interface": "adj_factor", "capability": "复权因子", "datasets": ["daily"]},
{"interface": "daily_basic", "capability": "估值", "datasets": ["valuation"]},
{"interface": "index_daily", "capability": "指数日K", "datasets": ["index_daily"]},
{"interface": "moneyflow", "capability": "资金流", "datasets": ["moneyflow"]},
{"interface": "stk_auction", "capability": "集合竞价", "datasets": ["auction"]},
{"interface": "limit_list_d", "capability": "涨跌停池", "datasets": ["limit_events"]},
{"interface": "ths_hot", "capability": "同花顺人气榜", "datasets": ["popularity"]},
{"interface": "dc_hot", "capability": "东方财富人气榜", "datasets": ["popularity"]},
{"interface": "hm_detail", "capability": "龙虎榜游资明细", "datasets": ["dragon_tiger"]},
{"interface": "ths_daily", "capability": "同花顺概念行情", "datasets": ["sector_daily"]},
{"interface": "dc_index", "capability": "东方财富板块行情", "datasets": ["sector_daily"]},
{"interface": "sw_daily", "capability": "申万行业行情", "datasets": ["sector_daily"]},
{"interface": "trade_cal", "capability": "交易日历", "datasets": ["calendar"], "group": "日历 / 主档"},
{"interface": "stock_basic", "capability": "股票主档", "datasets": ["stocks"], "group": "日历 / 主档"},
{"interface": "daily", "capability": "个股日K", "datasets": ["daily"], "group": "盘后 A 批"},
{"interface": "adj_factor", "capability": "复权因子", "datasets": ["daily"], "group": "盘后 A 批"},
{"interface": "daily_basic", "capability": "估值", "datasets": ["valuation"], "group": "盘后 A 批"},
{"interface": "index_daily", "capability": "指数日K", "datasets": ["index_daily"], "group": "指数 B 批"},
{"interface": "moneyflow", "capability": "资金流", "datasets": ["moneyflow"], "group": "盘后 A 批"},
{"interface": "stk_auction", "capability": "集合竞价", "datasets": ["auction"], "group": "盘后 A 批"},
{"interface": "limit_list_d", "capability": "涨跌停池", "datasets": ["limit_events"], "group": "扩展软批"},
{"interface": "ths_hot", "capability": "同花顺人气榜", "datasets": ["popularity"], "group": "扩展软批"},
{"interface": "dc_hot", "capability": "东方财富人气榜", "datasets": ["popularity"], "group": "扩展软批"},
{"interface": "hm_detail", "capability": "龙虎榜游资明细", "datasets": ["dragon_tiger"], "group": "扩展软批"},
{"interface": "ths_daily", "capability": "同花顺概念行情", "datasets": ["sector_daily"], "group": "扩展软批"},
{"interface": "dc_index", "capability": "东方财富板块行情", "datasets": ["sector_daily"], "group": "扩展软批"},
{"interface": "sw_daily", "capability": "申万行业行情", "datasets": ["sector_daily"], "group": "扩展软批"},
],
},
{
@@ -55,13 +55,13 @@ CATALOG: list[dict[str, Any]] = [
"credential_key": None,
"status_source": "provider_health (unified, HEL-543)",
"interfaces": [
{"interface": "indices", "capability": "指数实时报价", "datasets": ["index_quotes"]},
{"interface": "market_quotes", "capability": "全市场实时快照", "datasets": ["quotes_latest"]},
{"interface": "named_quotes", "capability": "指定个股实时报价", "datasets": ["quotes_latest"]},
{"interface": "sector_quote", "capability": "申万板块实时报价(单个)", "datasets": ["sectors_quote"]},
{"interface": "sector_quotes_batch", "capability": "申万板块批量报价(预热)", "datasets": ["sectors_quote"]},
{"interface": "limit_pool", "capability": "涨停/炸板池(盘中)", "datasets": ["limit_pool"]},
{"interface": "intraday", "capability": "分时走势", "datasets": ["intraday_points"]},
{"interface": "indices", "capability": "指数实时报价", "datasets": ["index_quotes"], "group": "实时快照"},
{"interface": "market_quotes", "capability": "全市场实时快照", "datasets": ["quotes_latest"], "group": "实时快照"},
{"interface": "named_quotes", "capability": "指定个股实时报价", "datasets": ["quotes_latest"], "group": "实时快照"},
{"interface": "sector_quote", "capability": "申万板块实时报价(单个)", "datasets": ["sectors_quote"], "group": "实时快照"},
{"interface": "sector_quotes_batch", "capability": "申万板块批量报价(预热)", "datasets": ["sectors_quote"], "group": "实时快照"},
{"interface": "limit_pool", "capability": "涨停/炸板池(盘中)", "datasets": ["limit_pool"], "group": "实时快照"},
{"interface": "intraday", "capability": "分时走势", "datasets": ["intraday_points"], "group": "实时快照"},
],
},
{
@@ -71,13 +71,14 @@ CATALOG: list[dict[str, Any]] = [
"credential_key": None,
"status_source": "provider_health (unified, HEL-543)",
"interfaces": [
{"interface": "indices", "capability": "指数实时报价(东财失败时备用)", "datasets": ["index_quotes"]},
{"interface": "indices", "capability": "指数实时报价(东财失败时备用)", "datasets": ["index_quotes"], "group": "实时备援"},
{
"interface": "market_quotes_fallback",
"capability": "全市场快照(备用；按本地股票主档逐只请求拼接)",
"datasets": ["quotes_latest"],
"group": "实时备援",
},
{"interface": "named_quotes", "capability": "指定个股实时报价(东财失败时备用)", "datasets": ["quotes_latest"]},
{"interface": "named_quotes", "capability": "指定个股实时报价(东财失败时备用)", "datasets": ["quotes_latest"], "group": "实时备援"},
],
},
{
@@ -87,11 +88,11 @@ CATALOG: list[dict[str, Any]] = [
"credential_key": "ifind_refresh_token",
"status_source": "provider_health (unified, HEL-543) + adapter.status()",
"interfaces": [
{"interface": "wencai", "capability": "问财自然语言选股", "datasets": ["ifind_wencai"]},
{"interface": "snapshots", "capability": "快照", "datasets": ["ifind_snapshots"]},
{"interface": "history", "capability": "历史行情", "datasets": ["ifind_history"]},
{"interface": "realtime", "capability": "实时行情", "datasets": ["ifind_realtime"]},
{"interface": "intraday", "capability": "分时(高频)", "datasets": ["ifind_intraday"]},
{"interface": "wencai", "capability": "问财自然语言选股", "datasets": ["ifind_wencai"], "group": "预留接口"},
{"interface": "snapshots", "capability": "快照", "datasets": ["ifind_snapshots"], "group": "预留接口"},
{"interface": "history", "capability": "历史行情", "datasets": ["ifind_history"], "group": "预留接口"},
{"interface": "realtime", "capability": "实时行情", "datasets": ["ifind_realtime"], "group": "预留接口"},
{"interface": "intraday", "capability": "分时(高频)", "datasets": ["ifind_intraday"], "group": "预留接口"},
],
},
{
@@ -161,5 +162,35 @@ def snapshot(db: Any, auth: Any = None) -> list[dict[str, Any]]:
except Exception:
health_rows = []
item["live_interfaces"] = health_rows
# HEL-529 fix 3: 已登记 ≠ 已观测 ≠ 健康。观测记录有两种真实写法：
# eastmoney/tencent 的 observe() 直接写接口名；tushare 官方管线
# _log_call() 写的是数据集名（如 daily_basic 接口对应的数据集
# valuation）。这里按「接口名 或 该接口声明的 datasets 之一」双向
# 匹配，并为每个接口标注观测依据，禁止把"暂无观测"显示成"未配置"。
by_interface = {str(row["interface"]): row for row in health_rows}
for iface in item["interfaces"]:
row = by_interface.get(str(iface["interface"]))
basis = "interface" if row is not None else ""
if row is None:
for dataset in iface.get("datasets", []):
candidate = by_interface.get(str(dataset))
if candidate is not None:
row = candidate
basis = "dataset"
break
if row is None:
iface["observed"] = False
iface["observed_basis"] = ""
iface["observed_state"] = ""
iface["observed_at"] = ""
iface["observed_latency_ms"] = None
iface["observed_note"] = ""
else:
iface["observed"] = True
iface["observed_basis"] = basis
iface["observed_state"] = str(row["state"] or "")
iface["observed_at"] = str(row["last_ok_at"] or row["updated_at"] or "")
iface["observed_latency_ms"] = row["last_latency_ms"]
iface["observed_note"] = str(row["last_error"] or row["last_fallback_reason"] or "")
result.append(item)
return result
xiaobai-datahub/tests/test_hel529_rework.py
+213
View File
@@ -0,0 +1,213 @@
"""HEL-529 rework regressions: staging dedupe, anomaly convergence,
source-catalog observation join (dataset-name ↔ interface-name), lineage
update_freq. All read-only or within-batch fixes; none touch routing, the
8765 main site, or the 问天 frozen zone.
"""
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from datahub.adapters.tushare import TushareAdapter
from datahub.crypto import SecretVault
from datahub.hub import Hub
from datahub.pipeline import _dedupe_staging_rows
from datahub.settings import Settings
from tests.fixtures import fake_transport
class StagingDedupeTests(unittest.TestCase):
def test_popularity_within_batch_duplicates_collapse_keep_last(self) -> None:
rows = [
{"ts_code": "600000.SH", "trade_date": "20260914", "source": "ths", "rank": 1},
{"ts_code": "000868.SZ", "trade_date": "20260914", "source": "dc", "rank": 2},
{"ts_code": "600000.SH", "trade_date": "20260914", "source": "dc", "rank": 3, "hot": 9.9},
{"ts_code": "600000.SH", "trade_date": "20260914", "source": "dc", "rank": 4, "hot": 8.8},
]
out = _dedupe_staging_rows("popularity", rows)
self.assertEqual(len(out), 3) # (600000,ths) (000868,dc) (600000,dc)
dup = [r for r in out if r["ts_code"] == "600000.SH" and r["source"] == "dc"][0]
self.assertEqual(dup["rank"], 4) # keeps LAST occurrence
self.assertEqual(out[0]["ts_code"], "600000.SH") # preserves first-seen order
def test_dragon_tiger_seat_duplicates_collapse(self) -> None:
rows = [
{"ts_code": "300010.SZ", "trade_date": "20260914", "hm_name": "T王", "buy_amount": 100},
{"ts_code": "300010.SZ", "trade_date": "20260914", "hm_name": "T王", "buy_amount": 200},
{"ts_code": "300010.SZ", "trade_date": "20260914", "hm_name": "T王", "buy_amount": 300},
]
out = _dedupe_staging_rows("dragon_tiger", rows)
self.assertEqual(len(out), 1)
self.assertEqual(out[0]["buy_amount"], 300)
def test_different_sources_are_not_duplicates(self) -> None:
rows = [
{"ts_code": "600000.SH", "trade_date": "20260914", "source": "ths"},
{"ts_code": "600000.SH", "trade_date": "20260914", "source": "dc"},
]
self.assertEqual(len(_dedupe_staging_rows("popularity", rows)), 2)
def test_unknown_dataset_passthrough(self) -> None:
rows = [{"a": 1}, {"a": 1}]
self.assertEqual(_dedupe_staging_rows("calendar", rows), rows)
class _Base(unittest.TestCase):
def setUp(self) -> None:
self.tmp = tempfile.TemporaryDirectory()
settings = Settings(
encryption_key=SecretVault.generate_key(),
api_token="z" * 32,
admin_password="StartPass1",
tushare_token="real-tushare-token-abcdef",
db_path=Path(self.tmp.name) / "hub.db",
scheduler_enabled=False,
)
self.hub = Hub(settings, adapter=TushareAdapter("real-tushare-token-abcdef", transport=fake_transport))
def tearDown(self) -> None:
self.tmp.cleanup()
class StagingDedupePublishTests(_Base):
def test_duplicate_popularity_and_dragon_tiger_now_publish(self) -> None:
"""Replays the 2026-09-14 production failure: within-response duplicate
keys used to abort the whole batch at the staging INSERT; with dedupe
the same upstream payload publishes."""
db = self.hub.db
trade_date = "20240902"
# dc_hot returns 600000.SH twice within one response; hm_detail returns
# the same (ts_code, hm_name) seat three times (mirrors live evidence).
popularity_rows = [
{"ts_code": "600000.SH", "trade_date": trade_date, "source": "ths",
"ts_name": "浦发银行", "rank": 1, "pct_change": 1.2, "current_price": 10.2,
"hot": 90.0, "concept": "银行", "data_type": "热股"},
{"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc",
"ts_name": "浦发银行", "rank": 2, "pct_change": 1.2, "current_price": 10.2,
"hot": 80.0, "concept": "银行", "data_type": "A股市场"},
{"ts_code": "600000.SH", "trade_date": trade_date, "source": "dc",
"ts_name": "浦发银行", "rank": 3, "pct_change": 1.3, "current_price": 10.3,
"hot": 81.0, "concept": "银行", "data_type": "A股市场"},
]
dragon_rows = [
{"trade_date": trade_date, "ts_code": "600000.SH", "ts_name": "浦发银行",
"buy_amount": 100, "sell_amount": 200, "net_amount": -100,
"hm_name": "测试游资", "hm_orgs": "某某营业部", "tag": "超买"},
{"trade_date": trade_date, "ts_code": "600000.SH", "ts_name": "浦发银行",
"buy_amount": 300, "sell_amount": 0, "net_amount": 300,
"hm_name": "测试游资", "hm_orgs": "某某营业部", "tag": "超买"},
]
self.hub.pipeline._stage("popularity", "b-dup-pop", popularity_rows)
self.hub.pipeline._stage("dragon_tiger", "b-dup-dt", dragon_rows)
pop = db.fetchall("SELECT * FROM staging_popularity WHERE batch_id = 'b-dup-pop'")
dt = db.fetchall("SELECT * FROM staging_dragon_tiger WHERE batch_id = 'b-dup-dt'")
self.assertEqual(len(pop), 2)
self.assertEqual(len(dt), 1)
kept = dt[0]
self.assertEqual(kept["buy_amount"], 300)
class OverviewAnomalyConvergenceTests(_Base):
def _seed_batches(self, today: str) -> None:
db = self.hub.db
rows = [
# resolved history: earlier failures/stalls, later success
("x-daily-001", "daily", "staged", "empty official batch", "2026-09-14T15:05:00+08:00"),
("x-daily-002", "daily", "failed", "release group not switched", "2026-09-14T16:10:00+08:00"),
("x-daily-006", "daily", "published", "", "2026-09-14T20:00:00+08:00"),
# current unresolved faults
("x-pop-001", "popularity", "failed", "UNIQUE constraint failed: staging_popularity", "2026-09-14T22:40:00+08:00"),
("x-dt-001", "dragon_tiger", "failed", "UNIQUE constraint failed: staging_dragon_tiger", "2026-09-14T16:45:00+08:00"),
("x-dt-002", "dragon_tiger", "failed", "UNIQUE constraint failed: staging_dragon_tiger", "2026-09-14T21:48:00+08:00"),
# staged-empty later published
("x-idx-001", "index_daily", "staged", "empty official batch", "2026-09-14T15:10:00+08:00"),
("x-idx-003", "index_daily", "published", "", "2026-09-14T16:10:00+08:00"),
]
for batch_id, dataset, state, error, started in rows:
db.execute(
"INSERT INTO batches(batch_id, dataset, trade_date, state, attempt, rows_in, rows_out,"
" quality_json, started_at, finished_at, error) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
(batch_id, dataset, today, state, 1, None, None, None, started, started if state != "staged" else None, error or None),
)
db.execute(
"INSERT INTO publications(dataset, trade_date, active_batch, prev_batch, state, published_at)"
" VALUES ('daily', ?, 'x-daily-006', 'x-daily-005', 'published', '2026-09-14T20:00:22+08:00')",
(today,),
)
db.execute(
"INSERT INTO publications(dataset, trade_date, active_batch, prev_batch, state, published_at)"
" VALUES ('index_daily', ?, 'x-idx-003', 'x-idx-002', 'published', '2026-09-14T16:10:12+08:00')",
(today,),
)
def test_anomalies_only_latest_unresolved(self) -> None:
overview = self.hub.admin.overview()
today = overview["trade_date"]
self._seed_batches(today)
anomalies = self.hub.admin.overview()["anomalies"]
got = sorted((a["dataset"], a["batch_id"]) for a in anomalies)
self.assertEqual(
got,
[
("dragon_tiger", "x-dt-002"), # latest failed, never published
("popularity", "x-pop-001"), # latest failed, never published
],
)
class SourceCatalogJoinTests(_Base):
def test_observation_join_matches_dataset_named_health_rows(self) -> None:
db = self.hub.db
now = "2026-09-15T08:00:00+08:00"
# tushare observability writes dataset names (real legacy behavior)
for iface, state in [("valuation", "ok"), ("popularity", "ok"), ("stocks", "ok")]:
db.execute(
"INSERT INTO provider_health(provider, interface, state, last_ok_at, last_error,"
" last_fallback_reason, consec_failures, last_latency_ms, last_data_age_seconds, updated_at)"
" VALUES ('tushare', ?, ?, ?, '', '', 0, 300, NULL, ?)",
(iface, state, now, now),
)
# realtime providers write interface names
db.execute(
"INSERT INTO provider_health(provider, interface, state, last_ok_at, last_error,"
" last_fallback_reason, consec_failures, last_latency_ms, last_data_age_seconds, updated_at)"
" VALUES ('eastmoney', 'indices', 'ok', ?, '', '', 0, 153, NULL, ?)",
(now, now),
)
items = self.hub.admin.source_catalog()["items"]
tushare = [i for i in items if i["provider"] == "tushare"][0]
by_iface = {i["interface"]: i for i in tushare["interfaces"]}
# daily_basic serves valuation → observed via dataset name
self.assertTrue(by_iface["daily_basic"]["observed"])
self.assertEqual(by_iface["daily_basic"]["observed_basis"], "dataset")
self.assertEqual(by_iface["daily_basic"]["observed_state"], "ok")
# ths_hot + dc_hot serve popularity → observed via dataset name
self.assertTrue(by_iface["ths_hot"]["observed"])
self.assertTrue(by_iface["dc_hot"]["observed"])
# stock_basic serves stocks
self.assertTrue(by_iface["stock_basic"]["observed"])
# never-observed interface stays honestly unobserved (not "unconfigured")
self.assertFalse(by_iface["trade_cal"]["observed"])
self.assertEqual(by_iface["trade_cal"]["observed_state"], "")
# interfaces carry real batch groups
groups = {i["interface"]: i["group"] for i in tushare["interfaces"]}
self.assertEqual(groups["daily"], "盘后 A 批")
self.assertEqual(groups["ths_hot"], "扩展软批")
self.assertEqual(groups["index_daily"], "指数 B 批")
eastmoney = [i for i in items if i["provider"] == "eastmoney"][0]
em = {i["interface"]: i for i in eastmoney["interfaces"]}
self.assertTrue(em["indices"]["observed"])
self.assertEqual(em["indices"]["observed_basis"], "interface")
self.assertFalse(em["market_quotes"]["observed"])
def test_lineage_update_freq_present(self) -> None:
items = self.hub.admin.lineage("20240902")["items"]
self.assertTrue(items)
for item in items:
self.assertTrue(item.get("update_freq"), f"missing update_freq for {item['dataset']}")
if __name__ == "__main__":
unittest.main()