Files

478 lines
19 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""v7.3.0 升级回归测试 — 名单完善 + 爬取增强 + 内容识别 + 汇总分析 + 语音交互
覆盖:
1. 机构名单:11 类新增精选名录落盘、CATEGORY_MAP 接线、registry ≥2300 家/43 类
2. crawl_enhancer:classify_response / SmartFetcher 三级降级 / batch 统计与去重
3. web_recognizer:表格/KV/文章条目/分页/数字抽取/页面类型判别
4. data_aggregator:去重/统计/截面/快照聚合/机构简报/Markdown 导出
5. voice_interface:播报文本压缩 / VoiceChat 假回调闭环
全部离线(fake 注入),不发网络请求。
"""
import importlib.util
import json
from pathlib import Path
import pytest
SKILL_DIR = Path(__file__).resolve().parent.parent
def _load_scripts_package():
"""以显式路径加载真实 scripts 包,避免工作区多 scripts 目录造成命名空间包冲突。"""
init_py = SKILL_DIR / "scripts" / "__init__.py"
spec = importlib.util.spec_from_file_location(
"scripts", init_py, submodule_search_locations=[str(SKILL_DIR / "scripts")])
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
from crawl_enhancer import (SmartFetcher, FetchResult, batch_smart_crawl,
classify_response, STATUS_OK, STATUS_EMPTY,
STATUS_SOFT_BLOCKED, STATUS_ERROR)
from web_recognizer import (extract_tables, extract_kv_blocks,
extract_article_items, extract_numbers_with_context,
detect_pagination, recognize_page_type,
smart_extract, strip_tags)
from data_aggregator import (normalize_title, title_similarity, dedupe_articles,
sentiment_statistics, cross_section_summary,
aggregate_snapshots, build_institution_brief,
export_markdown, export_json)
from voice_interface import VoiceChat, summarize_for_speech
DATA_DIR = SKILL_DIR / "data"
CURATED_BUNDLE = DATA_DIR / "curated.json"
# ════════════════════════ 1. 机构名单完善 ════════════════════════
NEW_CURATED = ["securities_consulting", "local_amc", "payment", "credit_rating",
"credit_service", "mutual_insurance", "pension_mgmt",
"wm_subsidiary", "financial_infra", "internet_loan", "fund_sales"]
def test_new_curated_files_exist_and_valid():
"""11 个新增精选名录均在合并文件 curated.json 中且为合法非空字符串数组。"""
assert CURATED_BUNDLE.exists(), "缺少精选名录合并文件 curated.json"
bundle = json.loads(CURATED_BUNDLE.read_text(encoding="utf-8"))
assert isinstance(bundle, dict) and bundle, "curated.json 非法"
for name in NEW_CURATED:
items = bundle.get(name)
assert isinstance(items, list) and items, f"curated.json 缺少类别 {name}"
for it in items:
assert isinstance(it, str) and it.strip(), f"{name} 含非法条目"
def test_category_map_covers_new_curated():
"""扩充引擎 CATEGORY_MAP 已接线全部新增名录。"""
from institution_expander import CATEGORY_MAP
for name in NEW_CURATED:
assert name in CATEGORY_MAP, f"CATEGORY_MAP 缺少 {name}"
type_name, list_file, source = CATEGORY_MAP[name]
assert type_name and list_file and source
def test_registry_expanded():
"""registry 总数 ≥2300 家 / ≥43 类,且含全部新增类型。"""
doc = json.loads((DATA_DIR / "institution_registry.json").read_text(encoding="utf-8"))
cols, rows = doc["c"], doc["d"]
i_type = cols.index("type")
types = {r[i_type] for r in rows}
assert len(rows) >= 2300, f"registry 仅 {len(rows)} 家"
assert len(types) >= 43, f"registry 仅 {len(types)} 类"
for t in ("地方资产管理公司", "支付机构", "信用评级机构", "征信与商查机构",
"相互保险组织", "养老金管理公司", "券商资管子公司",
"金融基础设施机构", "互联网小贷公司", "证券投资咨询机构"):
assert t in types, f"registry 缺少类型 {t}"
def test_new_list_files_synced():
"""新增类型对应的 *_list.json 已同步落盘且 count 一致。"""
from institution_expander import CATEGORY_MAP
for name in NEW_CURATED:
_, list_file, _ = CATEGORY_MAP[name]
path = DATA_DIR / list_file
assert path.exists(), f"缺少 {list_file}"
doc = json.loads(path.read_text(encoding="utf-8"))
assert doc["count"] == len(doc["institutions"])
# ════════════════════════ 2. 爬取增强 ════════════════════════════
class _FakeResp:
def __init__(self, text):
self.text = text
def test_classify_response_categories():
long_html = "<html>" + "正文内容。" * 100 + "</html>"
assert classify_response(long_html) == STATUS_OK
assert classify_response(None) == STATUS_ERROR
assert classify_response(" ") == STATUS_EMPTY
assert classify_response("请完成人机验证") == STATUS_SOFT_BLOCKED
assert classify_response('{"a": 1}') == STATUS_OK # JSON 短响应有效
assert classify_response("x" * 50) == STATUS_EMPTY # 过短非 JSON
def _fetcher_factory(pages: dict, api=None, browser_pages=None):
"""构造注入 fake 依赖的 SmartFetcher。"""
def fake_http(url, timeout=20):
body = pages.get(url)
return _FakeResp(body) if body is not None else None
calls = {"browser": []}
def fake_browser(url):
calls["browser"].append(url)
return (browser_pages or {}).get(url)
return SmartFetcher(http_fetcher=fake_http,
browser_fetcher=fake_browser,
api_lookup=lambda u: api), calls
def test_smart_fetcher_http_ok():
page = "https://a.com/news"
html = "<html>" + "新闻内容。" * 100 + "</html>"
f, calls = _fetcher_factory({page: html})
r = f.fetch(page)
assert r.ok and r.source == "http" and r.text_len == len(html)
assert calls["browser"] == []
def test_smart_fetcher_browser_fallback():
page = "https://a.com/blocked"
blocked = "请完成人机验证"
good = "<html>" + "真实内容。" * 100 + "</html>"
f, calls = _fetcher_factory({page: blocked}, browser_pages={page: good})
r = f.fetch(page)
assert r.ok and r.source == "browser"
assert calls["browser"] == [page]
def test_smart_fetcher_api_priority():
page = "https://a.com/spa"
api_url = "https://a.com/api/list?page=1"
f, calls = _fetcher_factory({page: "壳页面"},
api={"url": api_url},
browser_pages={page: None})
# api 直连返回有效 JSON
f.http_fetcher_orig = f.http_fetcher
def fake_http(url, timeout=20):
if url == api_url:
return _FakeResp('{"list": [1, 2, 3]}')
return _FakeResp("壳页面")
f.http_fetcher = fake_http
r = f.fetch(page)
assert r.ok and r.source == "api"
assert r.extra.get("api_url") == api_url
def test_smart_fetcher_all_fail():
f, _ = _fetcher_factory({})
r = f.fetch("https://a.com/none")
assert not r.ok and r.status in (STATUS_ERROR, STATUS_EMPTY)
def test_batch_smart_crawl_stats(tmp_path):
ok_page = "https://a.com/1"
html = "<html>" + "内容。" * 100 + "</html>"
f, _ = _fetcher_factory({ok_page: html})
def fake_work(url, timeout=20):
if url == ok_page:
return _FakeResp(html)
return None
f.http_fetcher = fake_work
dedup_file = str(tmp_path / "seen.json")
out = batch_smart_crawl(
[ok_page, "https://a.com/bad", ok_page], # 第 3 个为重复
concurrency=2, dedup_file=dedup_file, fetcher=f)
stats = out["stats"]
assert stats["total"] == 3
assert stats["skipped_dup"] == 1
assert stats["fetched"] == 2
assert stats["ok"] == 1
assert stats["success_rate"] == 0.5
assert stats["by_status"].get(STATUS_OK) == 1
assert "a.com" in stats["by_domain"]
# 去重文件已持久化
assert Path(dedup_file).exists()
# ════════════════════════ 3. 网页内容识别 ════════════════════════
LIST_HTML = """
<html><body>
<ul>
<li><a href="/news/1001.html">央行宣布下调存款准备金率</a>
<span class="time">2026-08-20 09:30</span></li>
<li><a href="/news/1002.html">多家银行下调存款利率引发关注</a>
<span>2026-08-19</span></li>
<li><a href="/news/1003.html">基金发行市场回暖迹象明显</a><span>昨天</span></li>
<li><a href="/news/1004.html">券商资管新规征求意见稿发布</a></li>
<li><a href="/news/1005.html">保险资金入市规模持续扩大</a></li>
<li><a href="/news/1006.html">信托行业转型方向逐渐清晰</a></li>
<li><a href="javascript:void(0)">登录</a></li>
<li><a href="/about">关于我们</a></li>
</ul>
<a href="/news?page=2">下一页</a>
<a href="/news?page=5">5</a>
</body></html>
"""
DATA_HTML = """
<html><body>
<div class="fund_info">最新净值: 1.2345 日增长率: +0.85% 基金规模: 120.5亿元</div>
<dl><dt>成立日期</dt><dd>2015-03-01</dd><dt>基金经理</dt><dd>张三</dd></dl>
<table><tr><th>年度</th><th>收益率</th></tr>
<tr><td>2024</td><td>12.5%</td></tr>
<tr><td>2025</td><td>-3.2%</td></tr></table>
</body></html>
"""
ARTICLE_HTML = "<html><body>" + (
"<p>贵州茅台2026年上半年实现营业收入819.3亿元,同比增长17.8%。"
"净利润416.96亿元,同比增长15.2%。公司表示将继续加大市场投入。</p>"
* 6) + "</body></html>"
def test_strip_tags():
assert strip_tags("<p>hello <b>world</b></p>") == "hello world"
assert strip_tags("<script>a=1</script>x") == "x"
def test_extract_tables():
tables = extract_tables(DATA_HTML)
assert len(tables) == 1
t = tables[0]
assert t["headers"] == ["年度", "收益率"]
assert t["n_rows"] == 2 and t["n_cols"] == 2
assert t["rows"][0] == ["2024", "12.5%"]
def test_extract_kv_blocks():
kv = extract_kv_blocks(DATA_HTML)
assert kv.get("成立日期") == "2015-03-01"
assert kv.get("基金经理") == "张三"
assert kv.get("最新净值") == "1.2345"
assert kv.get("基金规模") == "120.5亿元"
def test_extract_article_items():
items = extract_article_items(LIST_HTML, base_url="https://news.example.com")
titles = [i["title"] for i in items]
assert "央行宣布下调存款准备金率" in titles
assert "登录" not in titles # javascript 链接过滤
assert "关于我们" not in titles # 过短标题过滤
first = items[0]
assert first["url"].startswith("https://news.example.com/news/")
dated = [i for i in items if i.get("date")]
assert any(d["date"].startswith("2026-08-20") for d in dated)
def test_detect_pagination():
pag = detect_pagination(LIST_HTML, base_url="https://news.example.com")
assert pag["has_page_param"] is True
assert pag["max_page"] == 5
assert pag["next_url"] and "page=2" in pag["next_url"]
def test_extract_numbers_with_context():
nums = extract_numbers_with_context(ARTICLE_HTML)
raws = [n["raw"] for n in nums]
assert any("819.3" in r and "亿" in r for r in raws)
assert any("17.8" in r and "%" in r for r in raws)
for n in nums:
assert n["context"]
def test_recognize_page_type():
assert recognize_page_type(LIST_HTML, "https://news.example.com/news") == "list"
assert recognize_page_type(DATA_HTML, "https://fund.example.com/fund/123") == "data"
assert recognize_page_type(ARTICLE_HTML, "https://x.com/a/1.html") == "article"
assert recognize_page_type("", "") == "unknown"
def test_smart_extract_list_page():
out = smart_extract(LIST_HTML, url="https://news.example.com/news")
assert out["page_type"] == "list"
assert len(out["articles"]) >= 5
assert out["pagination"]["max_page"] == 5
def test_smart_extract_data_page():
out = smart_extract(DATA_HTML, url="https://fund.example.com/fund/1")
assert out["page_type"] == "data"
assert out["tables"] and out["kv"]
# ════════════════════════ 4. 数据分析汇总 ════════════════════════
def _mk_articles():
return [
{"title": "招商银行因违规被处罚", "target_name": "招商银行",
"source": "财联社", "sentiment": "negative", "sentiment_score": -20.0,
"severity": "中度舆情", "publish_time": "2026-08-20 10:00:00",
"keywords_matched": ["处罚", "违规"]},
{"title": "招商银行因违规被处罚(转发)", "target_name": "招商银行",
"source": "新浪财经", "sentiment": "negative", "sentiment_score": -18.0,
"severity": "中度舆情", "publish_time": "2026-08-20 11:00:00",
"keywords_matched": ["处罚"]},
{"title": "招商银行业绩稳健获机构增持", "target_name": "招商银行",
"source": "财联社", "sentiment": "positive", "sentiment_score": 15.0,
"severity": "低度利好", "publish_time": "2026-08-21 09:00:00",
"keywords_matched": ["增持"]},
{"title": "华夏基金新发产品受热捧", "target_name": "华夏基金",
"source": "东方财富", "sentiment": "positive", "sentiment_score": 10.0,
"severity": "低度利好", "publish_time": "2026-08-21 12:00:00",
"keywords_matched": ["新发"]},
]
def test_title_similarity_and_dedupe():
assert normalize_title("招商银行 因违规被处罚!") == "招商银行因违规被处罚"
assert title_similarity("招商银行因违规被处罚",
"招商银行因违规被处罚(转发)") > 0.85
assert title_similarity("完全不同的标题", "招商银行因违规被处罚") < 0.3
kept = dedupe_articles(_mk_articles())
assert len(kept) == 3 # 转发条目被去掉
def test_sentiment_statistics():
st = sentiment_statistics(dedupe_articles(_mk_articles()))
assert st["total"] == 3
assert st["sentiment"]["negative"] == 1
assert st["sentiment"]["positive"] == 2
assert st["by_day"]["2026-08-21"] == 2
assert ("处罚", 1) in st["hot_keywords"] # 转发条目已去重
assert st["neg_ratio"] == pytest.approx(1 / 3, abs=0.01)
def test_cross_section_summary():
rows = cross_section_summary(dedupe_articles(_mk_articles()))
cmb = next(r for r in rows if r["target_name"] == "招商银行")
assert cmb["count"] == 2 and cmb["negative"] == 1
# 负面占比高的排前面
assert rows[0]["target_name"] == "招商银行"
def test_aggregate_snapshots(tmp_path):
snap = {"snapshot_id": "s1", "created_at": "2026-08-22",
"articles": _mk_articles()}
(tmp_path / "snap1.json").write_text(
json.dumps(snap, ensure_ascii=False), encoding="utf-8")
(tmp_path / "bad.json").write_text("{not json", encoding="utf-8")
agg = aggregate_snapshots(tmp_path)
assert agg["snapshot_count"] == 1
assert agg["raw_count"] == 4
assert agg["dedup_count"] == 3
assert agg["dedup_removed"] == 1
assert agg["by_target"] and agg["statistics"]["total"] == 3
def test_build_institution_brief(tmp_path):
snap = {"snapshot_id": "s1", "created_at": "2026-08-22",
"articles": _mk_articles()}
(tmp_path / "snap1.json").write_text(
json.dumps(snap, ensure_ascii=False), encoding="utf-8")
brief = build_institution_brief("招商银行", snapshot_dir=tmp_path)
assert brief["name"] == "招商银行"
assert brief["sentiment"] and brief["sentiment"]["total"] == 2
assert brief["recent_titles"]
def test_export_markdown_and_json(tmp_path):
agg = aggregate_snapshots.__wrapped__ if hasattr(
aggregate_snapshots, "__wrapped__") else None
snap_dir = tmp_path / "snaps"
snap_dir.mkdir()
(snap_dir / "s.json").write_text(
json.dumps({"snapshot_id": "s", "articles": _mk_articles()},
ensure_ascii=False), encoding="utf-8")
agg = aggregate_snapshots(snap_dir)
md_path = tmp_path / "report.md"
md = export_markdown(agg, md_path)
assert md_path.exists()
assert "# 金融数据爬取汇总报告" in md
assert "情感分布" in md and "目标机构截面" in md
assert "招商银行" in md
json_path = export_json(agg, tmp_path / "agg.json")
assert json.loads(json_path.read_text(encoding="utf-8"))["dedup_count"] == 3
# ════════════════════════ 5. 语音交互 ════════════════════════════
def test_summarize_for_speech_strips_markdown():
reply = "## 爬取结果\n**共 32 条**\n- 负面 5 条,详见:https://x.com/a"
out = summarize_for_speech(reply)
assert "http" not in out
assert "#" not in out and "**" not in out
assert "共32条" in out or "共 32 条" in out # 中文播报文本会去空白
def test_summarize_for_speech_truncates():
reply = "。".join(f"第{i}条舆情内容比较长" for i in range(60))
out = summarize_for_speech(reply, max_chars=100)
assert len(out) <= 120
assert out.endswith("(更多内容见终端输出)")
def test_summarize_for_speech_empty():
assert summarize_for_speech("") == ""
def test_voice_chat_handle_text():
spoken = []
vc = VoiceChat(
listen=lambda timeout=8: None,
speak_fn=lambda t: spoken.append(t) or True,
chat=lambda t: {"reply": f"已处理: {t}"},
tts=True)
reply = vc.handle_text("爬取贵州茅台最近7天的舆情")
assert "已处理" in reply
assert vc.history[-1]["user"] == "爬取贵州茅台最近7天的舆情"
assert spoken, "TTS 未被调用"
def test_voice_chat_no_tts():
vc = VoiceChat(chat=lambda t: {"reply": "ok"}, tts=False)
assert vc.handle_text("你好") == "ok"
def test_voice_chat_run_with_fake_listen(monkeypatch):
"""max_turns 限制下,假 STT 输入能驱动完整循环。"""
inputs = iter(["爬一下舆情", "退出"])
spoken = []
vc = VoiceChat(
listen=lambda timeout=8: next(inputs, None),
speak_fn=lambda t: spoken.append(t) or True,
chat=lambda t: {"reply": f"回复: {t}"},
tts=True)
monkeypatch.setattr("voice_interface.stt_available", lambda: True)
vc.run(max_turns=1)
assert len(vc.history) == 1
assert spoken
def test_voice_chat_default_engine():
"""chat=None 时自动降级内置对话引擎(sentiment_chat),返回非空回复。"""
vc = VoiceChat(chat=None, tts=False)
reply = vc.handle_text("你好")
assert reply, "默认对话引擎未返回回复"
# ════════════════════════ 6. 版本一致性 ══════════════════════════
def test_init_version_is_730():
scripts = _load_scripts_package()
assert scripts.__version__ == "7.7.0"
def test_new_modules_importable_from_package():
scripts = _load_scripts_package()
for name in ("batch_smart_crawl", "smart_extract", "aggregate_snapshots",
"VoiceChat", "speak", "extract_tables"):
assert hasattr(scripts, name), f"scripts 缺少导出 {name}"