# -*- coding: utf-8 -*- """v7.3.0 升级回归测试 — 名单完善 + 爬取增强 + 内容识别 + 汇总分析 + 语音交互 覆盖: 1. 机构名单:11 类新增精选名录落盘、CATEGORY_MAP 接线、registry ≥2300 家/43 类 2. crawl_enhancer:classify_response / SmartFetcher 三级降级 / batch 统计与去重 3. web_recognizer:表格/KV/文章条目/分页/数字抽取/页面类型判别 4. data_aggregator:去重/统计/截面/快照聚合/机构简报/Markdown 导出 5. voice_interface:播报文本压缩 / VoiceChat 假回调闭环 全部离线(fake 注入),不发网络请求。 """ import importlib.util import json from pathlib import Path import pytest SKILL_DIR = Path(__file__).resolve().parent.parent def _load_scripts_package(): """以显式路径加载真实 scripts 包,避免工作区多 scripts 目录造成命名空间包冲突。""" init_py = SKILL_DIR / "scripts" / "__init__.py" spec = importlib.util.spec_from_file_location( "scripts", init_py, submodule_search_locations=[str(SKILL_DIR / "scripts")]) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod from crawl_enhancer import (SmartFetcher, FetchResult, batch_smart_crawl, classify_response, STATUS_OK, STATUS_EMPTY, STATUS_SOFT_BLOCKED, STATUS_ERROR) from web_recognizer import (extract_tables, extract_kv_blocks, extract_article_items, extract_numbers_with_context, detect_pagination, recognize_page_type, smart_extract, strip_tags) from data_aggregator import (normalize_title, title_similarity, dedupe_articles, sentiment_statistics, cross_section_summary, aggregate_snapshots, build_institution_brief, export_markdown, export_json) from voice_interface import VoiceChat, summarize_for_speech DATA_DIR = SKILL_DIR / "data" CURATED_BUNDLE = DATA_DIR / "curated.json" # ════════════════════════ 1. 机构名单完善 ════════════════════════ NEW_CURATED = ["securities_consulting", "local_amc", "payment", "credit_rating", "credit_service", "mutual_insurance", "pension_mgmt", "wm_subsidiary", "financial_infra", "internet_loan", "fund_sales"] def test_new_curated_files_exist_and_valid(): """11 个新增精选名录均在合并文件 curated.json 中且为合法非空字符串数组。""" assert CURATED_BUNDLE.exists(), "缺少精选名录合并文件 curated.json" bundle = json.loads(CURATED_BUNDLE.read_text(encoding="utf-8")) assert isinstance(bundle, dict) and bundle, "curated.json 非法" for name in NEW_CURATED: items = bundle.get(name) assert isinstance(items, list) and items, f"curated.json 缺少类别 {name}" for it in items: assert isinstance(it, str) and it.strip(), f"{name} 含非法条目" def test_category_map_covers_new_curated(): """扩充引擎 CATEGORY_MAP 已接线全部新增名录。""" from institution_expander import CATEGORY_MAP for name in NEW_CURATED: assert name in CATEGORY_MAP, f"CATEGORY_MAP 缺少 {name}" type_name, list_file, source = CATEGORY_MAP[name] assert type_name and list_file and source def test_registry_expanded(): """registry 总数 ≥2300 家 / ≥43 类,且含全部新增类型。""" doc = json.loads((DATA_DIR / "institution_registry.json").read_text(encoding="utf-8")) cols, rows = doc["c"], doc["d"] i_type = cols.index("type") types = {r[i_type] for r in rows} assert len(rows) >= 2300, f"registry 仅 {len(rows)} 家" assert len(types) >= 43, f"registry 仅 {len(types)} 类" for t in ("地方资产管理公司", "支付机构", "信用评级机构", "征信与商查机构", "相互保险组织", "养老金管理公司", "券商资管子公司", "金融基础设施机构", "互联网小贷公司", "证券投资咨询机构"): assert t in types, f"registry 缺少类型 {t}" def test_new_list_files_synced(): """新增类型对应的 *_list.json 已同步落盘且 count 一致。""" from institution_expander import CATEGORY_MAP for name in NEW_CURATED: _, list_file, _ = CATEGORY_MAP[name] path = DATA_DIR / list_file assert path.exists(), f"缺少 {list_file}" doc = json.loads(path.read_text(encoding="utf-8")) assert doc["count"] == len(doc["institutions"]) # ════════════════════════ 2. 爬取增强 ════════════════════════════ class _FakeResp: def __init__(self, text): self.text = text def test_classify_response_categories(): long_html = "" + "正文内容。" * 100 + "" assert classify_response(long_html) == STATUS_OK assert classify_response(None) == STATUS_ERROR assert classify_response(" ") == STATUS_EMPTY assert classify_response("请完成人机验证") == STATUS_SOFT_BLOCKED assert classify_response('{"a": 1}') == STATUS_OK # JSON 短响应有效 assert classify_response("x" * 50) == STATUS_EMPTY # 过短非 JSON def _fetcher_factory(pages: dict, api=None, browser_pages=None): """构造注入 fake 依赖的 SmartFetcher。""" def fake_http(url, timeout=20): body = pages.get(url) return _FakeResp(body) if body is not None else None calls = {"browser": []} def fake_browser(url): calls["browser"].append(url) return (browser_pages or {}).get(url) return SmartFetcher(http_fetcher=fake_http, browser_fetcher=fake_browser, api_lookup=lambda u: api), calls def test_smart_fetcher_http_ok(): page = "https://a.com/news" html = "" + "新闻内容。" * 100 + "" f, calls = _fetcher_factory({page: html}) r = f.fetch(page) assert r.ok and r.source == "http" and r.text_len == len(html) assert calls["browser"] == [] def test_smart_fetcher_browser_fallback(): page = "https://a.com/blocked" blocked = "请完成人机验证" good = "" + "真实内容。" * 100 + "" f, calls = _fetcher_factory({page: blocked}, browser_pages={page: good}) r = f.fetch(page) assert r.ok and r.source == "browser" assert calls["browser"] == [page] def test_smart_fetcher_api_priority(): page = "https://a.com/spa" api_url = "https://a.com/api/list?page=1" f, calls = _fetcher_factory({page: "壳页面"}, api={"url": api_url}, browser_pages={page: None}) # api 直连返回有效 JSON f.http_fetcher_orig = f.http_fetcher def fake_http(url, timeout=20): if url == api_url: return _FakeResp('{"list": [1, 2, 3]}') return _FakeResp("壳页面") f.http_fetcher = fake_http r = f.fetch(page) assert r.ok and r.source == "api" assert r.extra.get("api_url") == api_url def test_smart_fetcher_all_fail(): f, _ = _fetcher_factory({}) r = f.fetch("https://a.com/none") assert not r.ok and r.status in (STATUS_ERROR, STATUS_EMPTY) def test_batch_smart_crawl_stats(tmp_path): ok_page = "https://a.com/1" html = "" + "内容。" * 100 + "" f, _ = _fetcher_factory({ok_page: html}) def fake_work(url, timeout=20): if url == ok_page: return _FakeResp(html) return None f.http_fetcher = fake_work dedup_file = str(tmp_path / "seen.json") out = batch_smart_crawl( [ok_page, "https://a.com/bad", ok_page], # 第 3 个为重复 concurrency=2, dedup_file=dedup_file, fetcher=f) stats = out["stats"] assert stats["total"] == 3 assert stats["skipped_dup"] == 1 assert stats["fetched"] == 2 assert stats["ok"] == 1 assert stats["success_rate"] == 0.5 assert stats["by_status"].get(STATUS_OK) == 1 assert "a.com" in stats["by_domain"] # 去重文件已持久化 assert Path(dedup_file).exists() # ════════════════════════ 3. 网页内容识别 ════════════════════════ LIST_HTML = """ 下一页 5 """ DATA_HTML = """
最新净值: 1.2345 日增长率: +0.85% 基金规模: 120.5亿元
成立日期
2015-03-01
基金经理
张三
年度收益率
202412.5%
2025-3.2%
""" ARTICLE_HTML = "" + ( "

贵州茅台2026年上半年实现营业收入819.3亿元,同比增长17.8%。" "净利润416.96亿元,同比增长15.2%。公司表示将继续加大市场投入。

" * 6) + "" def test_strip_tags(): assert strip_tags("

hello world

") == "hello world" assert strip_tags("x") == "x" def test_extract_tables(): tables = extract_tables(DATA_HTML) assert len(tables) == 1 t = tables[0] assert t["headers"] == ["年度", "收益率"] assert t["n_rows"] == 2 and t["n_cols"] == 2 assert t["rows"][0] == ["2024", "12.5%"] def test_extract_kv_blocks(): kv = extract_kv_blocks(DATA_HTML) assert kv.get("成立日期") == "2015-03-01" assert kv.get("基金经理") == "张三" assert kv.get("最新净值") == "1.2345" assert kv.get("基金规模") == "120.5亿元" def test_extract_article_items(): items = extract_article_items(LIST_HTML, base_url="https://news.example.com") titles = [i["title"] for i in items] assert "央行宣布下调存款准备金率" in titles assert "登录" not in titles # javascript 链接过滤 assert "关于我们" not in titles # 过短标题过滤 first = items[0] assert first["url"].startswith("https://news.example.com/news/") dated = [i for i in items if i.get("date")] assert any(d["date"].startswith("2026-08-20") for d in dated) def test_detect_pagination(): pag = detect_pagination(LIST_HTML, base_url="https://news.example.com") assert pag["has_page_param"] is True assert pag["max_page"] == 5 assert pag["next_url"] and "page=2" in pag["next_url"] def test_extract_numbers_with_context(): nums = extract_numbers_with_context(ARTICLE_HTML) raws = [n["raw"] for n in nums] assert any("819.3" in r and "亿" in r for r in raws) assert any("17.8" in r and "%" in r for r in raws) for n in nums: assert n["context"] def test_recognize_page_type(): assert recognize_page_type(LIST_HTML, "https://news.example.com/news") == "list" assert recognize_page_type(DATA_HTML, "https://fund.example.com/fund/123") == "data" assert recognize_page_type(ARTICLE_HTML, "https://x.com/a/1.html") == "article" assert recognize_page_type("", "") == "unknown" def test_smart_extract_list_page(): out = smart_extract(LIST_HTML, url="https://news.example.com/news") assert out["page_type"] == "list" assert len(out["articles"]) >= 5 assert out["pagination"]["max_page"] == 5 def test_smart_extract_data_page(): out = smart_extract(DATA_HTML, url="https://fund.example.com/fund/1") assert out["page_type"] == "data" assert out["tables"] and out["kv"] # ════════════════════════ 4. 数据分析汇总 ════════════════════════ def _mk_articles(): return [ {"title": "招商银行因违规被处罚", "target_name": "招商银行", "source": "财联社", "sentiment": "negative", "sentiment_score": -20.0, "severity": "中度舆情", "publish_time": "2026-08-20 10:00:00", "keywords_matched": ["处罚", "违规"]}, {"title": "招商银行因违规被处罚(转发)", "target_name": "招商银行", "source": "新浪财经", "sentiment": "negative", "sentiment_score": -18.0, "severity": "中度舆情", "publish_time": "2026-08-20 11:00:00", "keywords_matched": ["处罚"]}, {"title": "招商银行业绩稳健获机构增持", "target_name": "招商银行", "source": "财联社", "sentiment": "positive", "sentiment_score": 15.0, "severity": "低度利好", "publish_time": "2026-08-21 09:00:00", "keywords_matched": ["增持"]}, {"title": "华夏基金新发产品受热捧", "target_name": "华夏基金", "source": "东方财富", "sentiment": "positive", "sentiment_score": 10.0, "severity": "低度利好", "publish_time": "2026-08-21 12:00:00", "keywords_matched": ["新发"]}, ] def test_title_similarity_and_dedupe(): assert normalize_title("招商银行 因违规被处罚!") == "招商银行因违规被处罚" assert title_similarity("招商银行因违规被处罚", "招商银行因违规被处罚(转发)") > 0.85 assert title_similarity("完全不同的标题", "招商银行因违规被处罚") < 0.3 kept = dedupe_articles(_mk_articles()) assert len(kept) == 3 # 转发条目被去掉 def test_sentiment_statistics(): st = sentiment_statistics(dedupe_articles(_mk_articles())) assert st["total"] == 3 assert st["sentiment"]["negative"] == 1 assert st["sentiment"]["positive"] == 2 assert st["by_day"]["2026-08-21"] == 2 assert ("处罚", 1) in st["hot_keywords"] # 转发条目已去重 assert st["neg_ratio"] == pytest.approx(1 / 3, abs=0.01) def test_cross_section_summary(): rows = cross_section_summary(dedupe_articles(_mk_articles())) cmb = next(r for r in rows if r["target_name"] == "招商银行") assert cmb["count"] == 2 and cmb["negative"] == 1 # 负面占比高的排前面 assert rows[0]["target_name"] == "招商银行" def test_aggregate_snapshots(tmp_path): snap = {"snapshot_id": "s1", "created_at": "2026-08-22", "articles": _mk_articles()} (tmp_path / "snap1.json").write_text( json.dumps(snap, ensure_ascii=False), encoding="utf-8") (tmp_path / "bad.json").write_text("{not json", encoding="utf-8") agg = aggregate_snapshots(tmp_path) assert agg["snapshot_count"] == 1 assert agg["raw_count"] == 4 assert agg["dedup_count"] == 3 assert agg["dedup_removed"] == 1 assert agg["by_target"] and agg["statistics"]["total"] == 3 def test_build_institution_brief(tmp_path): snap = {"snapshot_id": "s1", "created_at": "2026-08-22", "articles": _mk_articles()} (tmp_path / "snap1.json").write_text( json.dumps(snap, ensure_ascii=False), encoding="utf-8") brief = build_institution_brief("招商银行", snapshot_dir=tmp_path) assert brief["name"] == "招商银行" assert brief["sentiment"] and brief["sentiment"]["total"] == 2 assert brief["recent_titles"] def test_export_markdown_and_json(tmp_path): agg = aggregate_snapshots.__wrapped__ if hasattr( aggregate_snapshots, "__wrapped__") else None snap_dir = tmp_path / "snaps" snap_dir.mkdir() (snap_dir / "s.json").write_text( json.dumps({"snapshot_id": "s", "articles": _mk_articles()}, ensure_ascii=False), encoding="utf-8") agg = aggregate_snapshots(snap_dir) md_path = tmp_path / "report.md" md = export_markdown(agg, md_path) assert md_path.exists() assert "# 金融数据爬取汇总报告" in md assert "情感分布" in md and "目标机构截面" in md assert "招商银行" in md json_path = export_json(agg, tmp_path / "agg.json") assert json.loads(json_path.read_text(encoding="utf-8"))["dedup_count"] == 3 # ════════════════════════ 5. 语音交互 ════════════════════════════ def test_summarize_for_speech_strips_markdown(): reply = "## 爬取结果\n**共 32 条**\n- 负面 5 条,详见:https://x.com/a" out = summarize_for_speech(reply) assert "http" not in out assert "#" not in out and "**" not in out assert "共32条" in out or "共 32 条" in out # 中文播报文本会去空白 def test_summarize_for_speech_truncates(): reply = "。".join(f"第{i}条舆情内容比较长" for i in range(60)) out = summarize_for_speech(reply, max_chars=100) assert len(out) <= 120 assert out.endswith("(更多内容见终端输出)") def test_summarize_for_speech_empty(): assert summarize_for_speech("") == "" def test_voice_chat_handle_text(): spoken = [] vc = VoiceChat( listen=lambda timeout=8: None, speak_fn=lambda t: spoken.append(t) or True, chat=lambda t: {"reply": f"已处理: {t}"}, tts=True) reply = vc.handle_text("爬取贵州茅台最近7天的舆情") assert "已处理" in reply assert vc.history[-1]["user"] == "爬取贵州茅台最近7天的舆情" assert spoken, "TTS 未被调用" def test_voice_chat_no_tts(): vc = VoiceChat(chat=lambda t: {"reply": "ok"}, tts=False) assert vc.handle_text("你好") == "ok" def test_voice_chat_run_with_fake_listen(monkeypatch): """max_turns 限制下,假 STT 输入能驱动完整循环。""" inputs = iter(["爬一下舆情", "退出"]) spoken = [] vc = VoiceChat( listen=lambda timeout=8: next(inputs, None), speak_fn=lambda t: spoken.append(t) or True, chat=lambda t: {"reply": f"回复: {t}"}, tts=True) monkeypatch.setattr("voice_interface.stt_available", lambda: True) vc.run(max_turns=1) assert len(vc.history) == 1 assert spoken def test_voice_chat_default_engine(): """chat=None 时自动降级内置对话引擎(sentiment_chat),返回非空回复。""" vc = VoiceChat(chat=None, tts=False) reply = vc.handle_text("你好") assert reply, "默认对话引擎未返回回复" # ════════════════════════ 6. 版本一致性 ══════════════════════════ def test_init_version_is_730(): scripts = _load_scripts_package() assert scripts.__version__ == "7.7.0" def test_new_modules_importable_from_package(): scripts = _load_scripts_package() for name in ("batch_smart_crawl", "smart_extract", "aggregate_snapshots", "VoiceChat", "speak", "extract_tables"): assert hasattr(scripts, name), f"scripts 缺少导出 {name}"