Files
cnfinancialscraper/tests/test_sentiment.py
T

448 lines
17 KiB
Python

# -*- coding: utf-8 -*-
"""v4.3 全网舆情爬虫 — 单元测试
覆盖:
1. 关键词词典完整
2. 情感分类器正确
3. 媒体源/目标库加载
4. 自定义目标持久化
5. 对话 NLU 解析
6. 导出器降级 (openpyxl/python-docx 缺失时)
7. ScheduledTask sentiment 字段
8. Snapshot.to_dict 完整性
"""
import sys
import os
import json
import pytest
from pathlib import Path
sys.dont_write_bytecode = True
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
# 强制使 data 目录指向临时项目副本,避免污染真实数据
TEST_DATA_DIR = Path(__file__).resolve().parent / "_test_data"
TEST_DATA_DIR.mkdir(parents=True, exist_ok=True)
import sentiment_keywords as sk
# ============================================================
# 1. 关键词词典
# ============================================================
class TestKeywords:
def test_keys_present(self):
for name in ["POSITIVE_KEYWORDS", "NEGATIVE_KEYWORDS", "NEUTRAL_KEYWORDS"]:
assert hasattr(sk, name), f"missing {name}"
assert isinstance(getattr(sk, name), dict)
def test_enough_keywords(self):
# 至少 200 条负向 + 100 条正向 + 20 条中性
assert len(sk.POSITIVE_KEYWORDS) >= 50
assert len(sk.NEGATIVE_KEYWORDS) >= 100
assert len(sk.NEUTRAL_KEYWORDS) >= 15
def test_severity_levels_ordered(self):
levels = sk.SEVERITY_LEVELS
for a, b in zip(levels, levels[1:]):
assert a[1] <= b[0] + 1, f"overlap: {a} vs {b}"
# ============================================================
# 2. 情感分类
# ============================================================
from sentiment_crawler import SentimentClassifier
class TestClassifier:
def setup_method(self):
self.clf = SentimentClassifier()
def test_positive(self):
s, sc, hits, sev = self.clf.classify("贵州茅台业绩增长11%,分红方案获好评", "")
assert s == "positive", f"got {s}"
assert sc > 0
assert sev in ("低度利好", "中度利好", "重大利好")
def test_negative(self):
s, sc, hits, sev = self.clf.classify("ST退市预警,被证监会处罚责令改正", "")
assert s == "negative", f"got {s}"
assert sc >= 5
assert sev in ("低度关注", "中度舆情", "高危舆情")
def test_neutral(self):
s, sc, hits, sev = self.clf.classify("关于召开股东大会的通知", "")
# 既无正向也无负向重大命中 - 应当归为 neutral
assert s in ("neutral", "positive"), f"got {s}"
# 标题里「股东大会」「通知」是中性词,分数>=2 但应该归 neutral
if s == "neutral":
assert sev == "中性"
def test_severity_negative(self):
# 强烈负面
s, _, _, sev = self.clf.classify("资金链断裂,跑路!董事长失联,财务造假,债务违约", "")
assert s == "negative"
assert sev == "高危舆情"
def test_severity_positive(self):
s, _, _, sev = self.clf.classify("业绩增长,营业收入增长,归母净利润增长,分红方案", "")
assert s == "positive"
assert sev in ("低度利好", "中度利好", "重大利好")
# ============================================================
# 3. 媒体源 / 目标库加载
# ============================================================
from sentiment_crawler import SentimentSourceLoader, SentimentTargetLoader
class TestLoaders:
def test_sources_load(self):
loader = SentimentSourceLoader()
stats = loader.stats()
assert stats.get("authoritative", 0) >= 5
assert stats.get("financial_vertical", 0) >= 5
assert stats.get("self_media", 0) >= 5
assert stats.get("international", 0) >= 5
assert "local_media" not in stats # v6.0.0 已移除官媒分类
def test_targets_load(self):
loader = SentimentTargetLoader()
cats = loader.categories
assert "fund_company" in cats
assert "listed_company" in cats
assert "local_government" not in cats # v6.0.0 已移除政府目标
all_items = loader.all_targets()
assert len(all_items) >= 1
def test_pick_by_name(self):
loader = SentimentSourceLoader()
picked = loader.pick_by_names(["财联社"])
assert len(picked) >= 1
assert picked[0]["name"] == "财联社"
# ============================================================
# 4. 自定义目标
# ============================================================
def test_custom_target_add(tmp_path, monkeypatch):
# 隔离 user custom 文件
fake_user = tmp_path / "sentiment_custom_targets.json"
monkeypatch.setattr("sentiment_crawler.USER_CUSTOM_FILE", fake_user)
from sentiment_crawler import add_custom_sentiment_target
r = add_custom_sentiment_target("custom", f"_test_{tmp_path.name}_bank", aliases=["X银行"])
assert r.get("ok") is True
assert fake_user.exists()
# ============================================================
# 5. 对话 NLU
# ============================================================
from sentiment_chat import SentimentChatParser
class TestChatParser:
def setup_method(self):
self.p = SentimentChatParser()
def test_help(self):
result = self.p.parse("帮助")
assert result["intent"] == "help"
def test_crawl_single(self):
result = self.p.parse("帮我爬一下贵州茅台最近7天的舆情")
assert result["intent"] == "crawl"
assert "贵州茅台" in result["params"]["targets"] or len(result["params"]["targets"]) >= 1
assert result["params"]["days"] == 7
def test_crawl_multiple(self):
result = self.p.parse("看下华夏基金、招商银行、中国人寿过去3天的负面新闻")
assert result["intent"] == "crawl"
targets = result["params"]["targets"]
assert any("华夏基金" in t for t in targets)
assert any("招商银行" in t for t in targets)
assert any("中国人寿" in t for t in targets)
assert result["params"]["days"] == 3
assert result["params"]["negative_only"] is True
def test_crawl_export_word(self):
result = self.p.parse("看下华夏基金今天的正面新闻并生成Word")
assert result["intent"] == "crawl_export"
assert result["params"]["positive_only"] is True
assert result["params"]["export"] == "word"
def test_crawl_export_excel(self):
result = self.p.parse("工银瑞信最近3天的负面新闻,并导出Excel")
assert result["intent"] == "crawl_export"
assert result["params"]["negative_only"] is True
assert result["params"]["export"] == "excel"
def test_schedule_daily(self):
result = self.p.parse("每天早上9点爬取银行板块舆情")
assert result["intent"] == "schedule"
assert result["params"]["frequency"] == "daily"
assert result["params"]["action"] == "crawl_sentiment_export"
def test_schedule_every_n_minutes(self):
result = self.p.parse("每30分钟爬取银行舆情")
assert result["intent"] == "schedule"
assert result["params"]["frequency"] == "every_30_minutes"
def test_list_sources(self):
result = self.p.parse("哪些媒体可用?")
assert result["intent"] == "list"
assert result["params"]["what"] == "sources"
def test_list_targets(self):
result = self.p.parse("有哪些目标")
assert result["intent"] == "list"
assert result["params"]["what"] == "targets"
def test_add_target(self):
result = self.p.parse("新增自定义目标 工银瑞信")
assert result["intent"] == "add_target"
assert result["params"]["name"] == "工银瑞信"
def test_add_target_fund(self):
result = self.p.parse("新增目标 工银瑞信基金")
assert result["intent"] == "add_target"
assert result["params"]["name"] == "工银瑞信基金"
assert result["params"]["category"] == "fund_company"
# ============================================================
# 6. 导出器降级
# ============================================================
from sentiment_crawler import SentimentSnapshot, SentimentArticle
from sentiment_exporter import to_json, to_csv, export as export_sent
def _make_snapshot() -> SentimentSnapshot:
arts = [
SentimentArticle(title="T1", source="财联社", source_type="financial_vertical",
target_name="贵州茅台", target_type="上市公司",
publish_time="2026-07-27 10:00:00", url="https://example.com/1",
sentiment="positive", sentiment_score=10, severity="中度利好",
summary="业绩增长", keywords_matched=["业绩增长"]),
SentimentArticle(title="T2", source="新浪财经", source_type="self_media",
target_name="贵州茅台", target_type="上市公司",
publish_time="2026-07-26 08:00:00", url="https://example.com/2",
sentiment="negative", sentiment_score=20, severity="高危舆情",
summary="处罚决定", keywords_matched=["处罚"]),
]
return SentimentSnapshot(
snapshot_id="sn_test",
created_at="2026-07-27 12:00:00",
target_filter={"names": ["贵州茅台"], "categories": []},
source_filter=["财联社", "新浪财经"],
articles=arts,
stats={
"total": 2,
"by_sentiment": {"positive": 1, "negative": 1, "neutral": 0},
"by_severity": {"中度利好": 1, "高危舆情": 1},
"by_source_type": {"financial_vertical": 1, "self_media": 1},
},
)
class TestExporter:
def test_json(self, tmp_path):
snap = _make_snapshot()
out = to_json(snap, tmp_path / "x.json")
assert Path(out).exists()
loaded = json.loads(Path(out).read_text(encoding="utf-8"))
assert loaded["snapshot_id"] == "sn_test"
assert len(loaded["articles"]) == 2
def test_csv(self, tmp_path):
snap = _make_snapshot()
out = to_csv(snap, tmp_path / "x.csv")
assert Path(out).exists()
text = Path(out).read_text(encoding="utf-8-sig")
assert "财联社" in text
assert "T1" in text and "T2" in text
def test_excel(self, tmp_path):
try:
import openpyxl # noqa
except ImportError:
pytest.skip("openpyxl 未安装")
snap = _make_snapshot()
out = export_sent(snap, fmt="excel", output_path=tmp_path / "sn_test.xlsx")
# 返回 dict: {excel: path}
assert "excel" in out
p = Path(out["excel"])
assert p.exists()
assert p.stat().st_size > 1000
def test_word(self, tmp_path):
try:
import docx # noqa
except ImportError:
pytest.skip("python-docx 未安装")
snap = _make_snapshot()
out = export_sent(snap, fmt="word", output_path=tmp_path / "sn_test.docx")
assert "word" in out
assert Path(out["word"]).exists()
# ============================================================
# 7. ScheduledTask sentiment 字段
# ============================================================
def test_scheduled_task_sentiment_fields():
from crawl_scheduler import ScheduledTask, TaskFrequency, TaskAction
t = ScheduledTask(
task_id="x", name="bank_yuqing",
action=TaskAction.CRAWL_SENTIMENT_EXPORT,
sentiment_targets=["招商银行"],
sentiment_categories=["commercial_bank"],
sentiment_source_categories=["authoritative"],
sentiment_days=7, sentiment_negative_only=True,
sentiment_max=50, sentiment_export_format="all",
)
d = t.to_dict()
assert d["action"] == "crawl_sentiment_export"
assert d["sentiment_targets"] == ["招商银行"]
assert d["sentiment_days"] == 7
t2 = ScheduledTask.from_dict(d)
assert t2.sentiment_export_format == "all"
assert t2.sentiment_targets == ["招商银行"]
# ============================================================
# 8. Snapshot.to_dict
# ============================================================
def test_snapshot_to_dict():
snap = _make_snapshot()
d = snap.to_dict()
assert d["snapshot_id"] == "sn_test"
assert "stats" in d
assert "articles" in d
assert d["stats"]["by_sentiment"]["positive"] == 1
# ============================================================
# run with pytest
# ============================================================
if __name__ == "__main__":
pytest.main([__file__, "-v", "-x"])
# ============================================================
# 10. v4.3.1 优化后的测试 — 速度控制 / 短路 / 异常安全
from sentiment_crawler import SentimentCrawler
# ============================================================
import time as _time
class TestTimeoutsAndShortCircuit:
def test_time_up_default(self):
c = SentimentCrawler()
c._run_start_ts = _time.time()
assert c._time_up() is False
def test_time_up_exceeded(self):
c = SentimentCrawler(max_total_seconds=1)
c._run_start_ts = _time.time() - 5
assert c._time_up() is True
def test_placeholder_dedup(self):
c = SentimentCrawler()
# 第一次应 False
assert c._placeholder_seen("贵州茅台", "财联社") is False
# 第二次 True (占位已用过)
assert c._placeholder_seen("贵州茅台", "财联社") is True
# 不同 source 不互相影响
assert c._placeholder_seen("贵州茅台", "新华") is False
def test_dns_short_circuit(self):
c = SentimentCrawler()
# 不存在域名应短路
bad_url = "https://nonexistent-host-xyz.invalid/foo"
assert c._domain_unresolvable(bad_url) is True
def test_default_max_total_seconds(self):
"""默认 max_total_seconds 应 <=30s(对话体验友好)"""
from sentiment_crawler import crawl_sentiment, SentimentCrawler
c = SentimentCrawler()
assert c.max_total_seconds <= 60, f"got {c.max_total_seconds}"
class TestExceptionsSafe:
def test_chat_exception_safe(self):
from sentiment_chat import chat_handle
for txt in ["", " ", "\n", "你好", None, "💥⚡"]:
try:
r = chat_handle(txt or "empty")
assert "reply" in r
except Exception as e:
pytest.fail(f"chat_handle({txt!r}) crashed: {e}")
def test_classifier_empty(self):
clf = SentimentClassifier()
for s, sc, h, sev in [clf.classify("", ""), clf.classify(" ", None)]:
assert s in ("positive", "negative", "neutral")
class TestStatsCompleteness:
def test_stats_by_source_type(self):
from sentiment_crawler import SentimentSnapshot, SentimentArticle
from sentiment_exporter import to_dialog
# 制造一个 snapshots, 验证 to_dialog 显示 by_source_type
arts = [
SentimentArticle(title="t1", source="财联社", source_type="financial_vertical",
target_name="茅台", sentiment="positive", severity="低度利好"),
SentimentArticle(title="t2", source="财联社", source_type="authoritative",
target_name="茅台", sentiment="negative", severity="高危舆情"),
]
snap = SentimentSnapshot(
snapshot_id="test", created_at="2026-07-27",
target_filter={"names": ["茅台"], "categories": []},
source_filter=["财联社", "新华网"],
articles=arts,
stats={"total": 2, "by_sentiment": {"positive": 1, "negative": 1, "neutral": 0},
"by_severity": {"低度利好": 1, "高危舆情": 1},
"by_source_type": {"financial_vertical": 1, "authoritative": 1},
"elapsed_seconds": 12.3, "timed_out": True},
)
d = to_dialog(snap)
assert "媒体类别 Top" in d
assert "12.3s" in d or "12s" in d
assert "超时" in d
class TestSchedulerRoundtrip:
def test_sentiment_fields_roundtrip(self):
from crawl_scheduler import ScheduledTask, TaskAction
original = ScheduledTask(
task_id="x1", name="test",
action=TaskAction.CRAWL_SENTIMENT_EXPORT,
sentiment_targets=["工银瑞信", "贵州茅台"],
sentiment_categories=["fund_company"],
sentiment_source_categories=["authoritative"],
sentiment_days=5, sentiment_positive_only=True,
sentiment_negative_only=False, sentiment_export_format="word",
)
d = original.to_dict()
# 反序列化
restored = ScheduledTask.from_dict(d)
assert restored.sentiment_targets == ["工银瑞信", "贵州茅台"]
assert restored.sentiment_days == 5
assert restored.action == TaskAction.CRAWL_SENTIMENT_EXPORT
class TestFingerprint:
def test_url_normalized(self):
a = SentimentArticle(title="t", url="https://x.com/a?utm_source=x", target_name="y")
b = SentimentArticle(title="t", url="https://x.com/a?utm_source=y", target_name="y")
# utm 后缀应去掉,URL 同
assert a.fingerprint == b.fingerprint
def test_trailing_slash_normalized(self):
a = SentimentArticle(title="t", url="https://x.com/a/", target_name="y")
b = SentimentArticle(title="t", url="https://x.com/a", target_name="y")
assert a.fingerprint == b.fingerprint