Files

358 lines
16 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""v7.5.0 升级回归测试 — 编码修复 + 下载加固 + 爬取强化 + 机构名单扩充
覆盖:
1. 编码检测:charset 归一化 / 声明编码验证 / meta charset 字节级探测 /
中文启发式 / 域名编码记忆 / GBK JSON API 兼容
2. 爬取强化:SSL 证书失败协议降级重试 / 200 空响应自动重试
3. download_file 加固:Referer 注入 / 文本转码 UTF-8 / 反爬页校验 /
GBK Content-Disposition 文件名乱码修复
4. announcement_scraper:无 requests 时标准库兜底下载 / GBK 文件名修复
5. 机构名单:6 类扩充名录落盘 / load_curated 路径修复 / registry ≥2334 家
全部离线(fake session 注入),不发网络请求。
"""
import json
import ssl
import urllib.error
import urllib.request
from pathlib import Path
from unittest.mock import patch
import pytest
import http_utils as hu
from http_utils import (StdlibResponse, StdlibSession, normalize_charset,
remember_domain_encoding, get_domain_encoding,
looks_like_chinese_binary, _detect_meta_charset_bytes,
_is_ssl_error, _scheme_downgrade, _fix_mojibake_filename,
_looks_html_magic, _is_text_download, _extract_filename,
http_get, download_file)
import announcement_scraper as ann
SKILL_DIR = Path(__file__).resolve().parent.parent
DATA_DIR = SKILL_DIR / "data"
class FakeResp:
"""鸭子类型模拟 StdlibResponse(http_get 成功分支只读这些属性)"""
def __init__(self, code=200, content=b"", headers=None, encoding="utf-8"):
self.code = code
self.content = content
self.headers = headers or {}
self._encoding = encoding
self.text = content.decode(encoding, errors="replace")
class FakeSession:
"""按调用次序逐个弹出响应/异常的假 session"""
def __init__(self, outcomes):
self.outcomes = list(outcomes)
self.calls = []
def get(self, url, headers=None, timeout=30, **kw):
self.calls.append({"url": url, "headers": dict(headers or {})})
item = self.outcomes.pop(0)
if isinstance(item, Exception):
raise item
return item
@pytest.fixture(autouse=True)
def _no_sleep():
"""全量测试不等待重试退避"""
with patch("http_utils.time.sleep"):
yield
@pytest.fixture
def _clean_hints():
saved = dict(hu.domain_encoding_hints)
yield
hu.domain_encoding_hints.clear()
hu.domain_encoding_hints.update(saved)
# ════════════════ 1. charset 归一化与编码检测 ════════════════
class TestNormalizeCharset:
def test_gb_family_unified_to_gb18030(self):
for alias in ("gb2312", "GBK", "cp936", "x-gbk", "chinese"):
assert normalize_charset(alias) == "gb18030"
def test_aliases_and_case_insensitive(self):
assert normalize_charset("UTF8") == "utf-8"
assert normalize_charset(" iso-8859-1 ") == "latin-1"
assert normalize_charset('"gbk"') == "gb18030"
def test_invalid_returns_none(self):
assert normalize_charset("not-a-charset-xxx") is None
assert normalize_charset("") is None
assert normalize_charset(None) is None
class TestDetectEncoding:
def setup_method(self):
self.s = StdlibSession()
def detect(self, data, hdrs=None, url=""):
return self.s._detect_encoding(200, hdrs or {}, data, url)
def test_bom_utf8(self):
assert self.detect(b"\xef\xbb\xbf<html></html>") == "utf-8"
def test_declared_gb2312_normalized(self):
data = "中国人民银行公告".encode("gb18030")
assert self.detect(data, {"Content-Type": "text/html; charset=gb2312"}) == "gb18030"
def test_declared_wrong_charset_not_blindly_trusted(self):
"""声称 utf-8 实为 GBK — 严格解码验证失败后启发式纠错"""
data = "某信托公司发布年度财务报告".encode("gb18030")
enc = self.detect(data, {"Content-Type": "text/html; charset=utf-8"})
assert enc == "gb18030"
assert data.decode(enc) == "某信托公司发布年度财务报告"
def test_meta_charset_bytes_level(self):
body = "基金净值公告".encode("gb18030")
data = b'<html><head><meta charset="gb2312"></head>' + body
assert self.detect(data, {}) == "gb18030"
def test_heuristic_gbk_no_declaration(self):
data = ("关于召开股东大会的通知,请各位股东准时出席。" * 3).encode("gb18030")
assert self.detect(data, {}) == "gb18030"
def test_heuristic_utf8_no_declaration(self):
data = ("关于召开股东大会的通知,请各位股东准时出席。" * 3).encode("utf-8")
assert self.detect(data, {}) == "utf-8"
def test_binary_fallback_latin1(self):
# 高位字节后跟空格(非 GBK 尾字节范围)→ 无法构成 GBK 双字节对,兜底 latin-1
data = bytes([0x80, 0x20] * 8)
assert self.detect(data, {}) == "latin-1"
@pytest.mark.usefixtures("_clean_hints")
def test_domain_memory_reuse(self):
remember_domain_encoding("bank.example.cn", "gb18030")
data = b"\xc4\xe3\xba\xc3" # 即使歧义也直接复用记忆编码
assert self.detect(data, {}, "http://bank.example.cn/list.htm") == "gb18030"
def test_remember_skips_trivial_encodings(self, _clean_hints):
remember_domain_encoding("a.example.cn", "latin-1")
remember_domain_encoding("b.example.cn", "ascii")
assert get_domain_encoding("a.example.cn") is None
assert get_domain_encoding("b.example.cn") is None
class TestChineseHeuristic:
def test_pure_ascii_is_utf8(self):
assert looks_like_chinese_binary(b"hello world 123") == "utf-8"
def test_gbk_bytes_detected(self):
data = ("招商银行 平安银行 兴业银行 民生银行 " * 5).encode("gb18030")
assert looks_like_chinese_binary(data) == "gb18030"
def test_empty_returns_none(self):
assert looks_like_chinese_binary(b"") is None
def test_meta_bytes_detect_http_equiv(self):
data = ('<meta http-equiv="Content-Type" content="text/html; charset=gbk">'
"正文").encode("gb18030")
assert _detect_meta_charset_bytes(data) == "gb18030"
class TestGBKJsonAPI:
def test_json_loads_gbk_bytes(self):
"""json.loads 对 bytes 仅支持 utf-8/16/32 — GBK API 需先解码"""
data = '{"name": "华夏基金", "code": "000001"}'.encode("gb18030")
resp = StdlibResponse("http://a.cn/api", 200, {}, data, encoding="gb18030")
assert resp.json() == {"name": "华夏基金", "code": "000001"}
def test_json_utf8_unchanged(self):
resp = StdlibResponse("http://a.cn/api", 200, {},
b'{"ok": true}', encoding="utf-8")
assert resp.json() == {"ok": True}
# ════════════════ 2. 爬取强化:SSL 降级 + 空响应重试 ════════════════
class TestSSLHelpers:
def test_is_ssl_error_variants(self):
assert _is_ssl_error(ssl.SSLError(1, "[SSL] certificate verify failed"))
assert _is_ssl_error(urllib.error.URLError(ssl.SSLError(1, "cert")))
assert _is_ssl_error(urllib.error.URLError("[SSL: CERTIFICATE_VERIFY_FAILED]"))
assert not _is_ssl_error(urllib.error.URLError("timed out"))
assert not _is_ssl_error(ValueError("not network"))
def test_scheme_downgrade(self):
assert _scheme_downgrade("https://a.cn/x") == "http://a.cn/x"
assert _scheme_downgrade("http://a.cn/x") is None
class TestHttpGetHardening:
def test_ssl_error_downgrades_to_http_once(self):
ok = FakeResp(content=b"<html>ok</html>")
fs = FakeSession([urllib.error.URLError(ssl.SSLError(1, "cert verify")), ok])
resp = http_get("https://bank.example.cn/page", session=fs,
retries=3, use_cache=False)
assert resp is ok
assert fs.calls[0]["url"].startswith("https://")
assert fs.calls[1]["url"] == "http://bank.example.cn/page"
def test_non_ssl_error_no_downgrade(self):
fs = FakeSession([urllib.error.URLError("timed out")] * 3)
resp = http_get("https://slow.example.cn/", session=fs,
retries=3, use_cache=False)
assert resp is None
assert all(c["url"].startswith("https://") for c in fs.calls)
def test_empty_body_retried_then_success(self):
fs = FakeSession([FakeResp(content=b""), FakeResp(content=b"real body")])
resp = http_get("http://empty.example.cn/", session=fs,
retries=3, use_cache=False)
assert resp.content == b"real body"
assert len(fs.calls) == 2
def test_retry_empty_disabled(self):
fs = FakeSession([FakeResp(content=b"")])
resp = http_get("http://empty.example.cn/", session=fs,
retries=3, use_cache=False, retry_empty=False)
assert resp.content == b""
assert len(fs.calls) == 1
def test_success_remembers_domain_encoding(self, _clean_hints):
fs = FakeSession([FakeResp(content=b"<html>hi</html>", encoding="gb18030")])
http_get("http://gbk.example.cn/", session=fs, retries=1, use_cache=False)
assert get_domain_encoding("gbk.example.cn") == "gb18030"
# ════════════════ 3. download_file 加固 ════════════════
class TestDownloadHelpers:
def test_html_magic(self):
assert _looks_html_magic(b"<!DOCTYPE html><html>")
assert _looks_html_magic(b" <html>")
assert not _looks_html_magic(b"%PDF-1.4")
assert not _looks_html_magic(b"")
def test_is_text_download(self):
r1 = FakeResp(headers={"Content-Type": "text/html; charset=gb2312"})
assert _is_text_download("x.bin", r1)
r2 = FakeResp(headers={"Content-Type": "application/pdf"})
assert _is_text_download("page.htm", r2)
assert not _is_text_download("doc.pdf", r2)
def test_fix_mojibake_filename_roundtrip(self):
raw = "2026年年度报告.pdf".encode("gb18030").decode("latin-1")
assert _fix_mojibake_filename(raw) == "2026年年度报告.pdf"
def test_fix_mojibake_filename_keeps_ascii_and_real_cjk(self):
assert _fix_mojibake_filename("report.pdf") == "report.pdf"
assert _fix_mojibake_filename("正常中文.pdf") == "正常中文.pdf"
assert _fix_mojibake_filename("") == ""
def test_extract_filename_from_content_disposition_gbk(self):
mangled = "分红公告.pdf".encode("gb18030").decode("latin-1")
resp = FakeResp(headers={"Content-Disposition": f'filename="{mangled}"'})
assert _extract_filename(resp, "http://a.cn/dl?id=1") == "分红公告.pdf"
def test_extract_filename_utf8_star_form(self):
resp = FakeResp(headers={
"Content-Disposition": "attachment; filename*=UTF-8''%E6%B5%8B%E8%AF%95.pdf"})
assert _extract_filename(resp, "http://a.cn/x") == "测试.pdf"
class TestDownloadFile:
def test_referer_auto_injected(self, tmp_path):
fs = FakeSession([FakeResp(content=b"%PDF-1.4 fake",
headers={"Content-Type": "application/pdf"})])
got = download_file("http://fund.example.cn/ann/1.pdf",
save_dir=str(tmp_path), session=fs, use_cache=False)
assert got and Path(got).read_bytes() == b"%PDF-1.4 fake"
assert fs.calls[0]["headers"].get("Referer") == "http://fund.example.cn/"
def test_text_transcoded_to_utf8(self, tmp_path):
gbk_html = "<html><body>基金净值公告:单位净值 1.234 元</body></html>"
fs = FakeSession([FakeResp(content=gbk_html.encode("gb18030"),
headers={"Content-Type": "text/html"},
encoding="gb18030")])
got = download_file("http://gbk.example.cn/page", save_dir=str(tmp_path),
filename="page.html", session=fs, use_cache=False)
assert got is not None and got.endswith("page.html")
text = Path(got).read_text(encoding="utf-8")
assert "基金净值公告" in text and "1.234" in text
def test_anti_crawl_html_not_saved_as_pdf(self, tmp_path):
fs = FakeSession([FakeResp(content=b"<!DOCTYPE html><html>verify</html>",
headers={"Content-Type": "application/pdf"})])
got = download_file("http://x.example.cn/a.pdf", save_dir=str(tmp_path),
session=fs, use_cache=False)
assert got is None
assert list(tmp_path.iterdir()) == []
def test_empty_body_not_saved(self, tmp_path):
fs = FakeSession([FakeResp(content=b"")])
got = download_file("http://x.example.cn/a.txt", save_dir=str(tmp_path),
session=fs, retries=1, use_cache=False)
assert got is None
# ════════════════ 4. announcement_scraper 下载兜底 ════════════════
class TestAnnouncementDownload:
def test_fix_gbk_filename_static(self):
raw = "招募说明书.pdf".encode("gb18030").decode("latin-1")
assert ann.PDFDownloader._fix_gbk_filename(raw) == "招募说明书.pdf"
assert ann.PDFDownloader._fix_gbk_filename("abc.pdf") == "abc.pdf"
def test_no_requests_session_falls_back_to_stdlib(self, tmp_path):
dl = ann.PDFDownloader(download_dir=str(tmp_path))
dl.session = None # 模拟无 requests 环境
save = tmp_path / "out.pdf"
with patch.object(hu, "download_file", return_value=str(save)) as m:
assert dl._download_file("http://a.cn/x.pdf", save) is True
m.assert_called_once()
kwargs = m.call_args.kwargs
assert kwargs["save_dir"] == str(tmp_path)
assert kwargs["filename"] == "out.pdf"
def test_requests_exception_degrades_to_stdlib(self, tmp_path):
dl = ann.PDFDownloader(download_dir=str(tmp_path))
if dl.session is None:
pytest.skip("requests 未安装,此分支由上一用例覆盖")
save = tmp_path / "y.pdf"
with patch.object(dl.session, "get", side_effect=ConnectionError("refused")), \
patch.object(hu, "download_file", return_value=str(save)) as m:
assert dl._download_file("http://a.cn/y.pdf", save) is True
m.assert_called_once()
# ════════════════ 5. 机构名单扩充 ════════════════
EXPANDED_CATEGORIES = ["payment", "wm_subsidiary", "local_amc",
"internet_loan", "fund_sales", "credit_service"]
class TestInstitutionRoster:
def test_curated_bundle_expanded_categories_present(self):
bundle = json.loads((DATA_DIR / "curated.json").read_text(encoding="utf-8"))
for cat in EXPANDED_CATEGORIES:
items = bundle.get(cat)
assert isinstance(items, list) and items, f"curated.json 缺少类别 {cat}"
def test_load_curated_path_fix(self):
"""v7.5 回归修复:load_curated 必须能加载 data/curated.json
(此前误指向已删除的 data/curated/ 目录导致返回空)"""
import institution_expander as exp
for cat in ("payment", "fund_sales", "local_amc"):
items = exp.load_curated(cat)
assert items and all(i.get("name") for i in items), \
f"load_curated({cat}) 为空 — 精选名录路径回归未修复"
def test_registry_total_after_expansion(self):
reg = json.loads((DATA_DIR / "institution_registry.json")
.read_text(encoding="utf-8"))
assert reg.get("_f") == "c"
assert len(reg["d"]) >= 2334, "registry 机构总数应 ≥2334(v7.5 扩充后)"
type_idx = reg["c"].index("type")
types = {r[type_idx] for r in reg["d"]}
assert len(types) >= 43