Initial commit: cnfinancialscraper v1.0.47

This commit is contained in:
MoBai
2026-09-13 23:43:57 +08:00
commit 126e43db6f
172 changed files with 76156 additions and 0 deletions
+370
View File
@@ -0,0 +1,370 @@
# -*- coding: utf-8 -*-
"""
数据完整性验证工具
验证机构名单、注册表等数据文件的完整性和格式正确性
使用方式:
python scripts/data_validator.py
功能:
1. 验证机构注册表(institution_registry.json)
2. 验证各类机构名单文件(*_list.json)
3. 输出详细的验证报告
"""
import json
import sys
from pathlib import Path
from typing import Dict, List, Tuple, Any
# 数据目录
SKILL_DATA_DIR = Path(__file__).parent.parent / "data"
def validate_institution_registry() -> Tuple[bool, str, Dict[str, Any]]:
"""
验证机构注册表
Returns:
(是否通过, 消息, 详细信息)
"""
registry_file = SKILL_DATA_DIR / "institution_registry.json"
if not registry_file.exists():
return False, "文件不存在", {"file": "institution_registry.json"}
try:
with open(registry_file, 'r', encoding='utf-8') as f:
data = json.load(f)
# 检查格式
if '_f' not in data:
return False, "文件格式不正确,缺少 _f 字段", {"file": "institution_registry.json"}
if 'd' not in data:
return False, "文件格式不正确,缺少 d 字段", {"file": "institution_registry.json"}
# 检查列定义
expected_columns = ['id', 'name', 'code', 'type', 'data_source', 'update_time', 'website', 'url_source']
actual_columns = data.get('c', [])
if actual_columns != expected_columns:
return False, f"列定义不匹配", {
"expected": expected_columns,
"actual": actual_columns
}
# 检查数据行数
institutions = data['d']
if len(institutions) == 0:
return False, "机构数据为空", {"file": "institution_registry.json"}
# 统计各类型机构
type_counts = {}
for row in institutions:
if len(row) >= 4:
inst_type = row[3]
type_counts[inst_type] = type_counts.get(inst_type, 0) + 1
return True, f"验证通过,共 {len(institutions)} 家机构", {
"file": "institution_registry.json",
"total": len(institutions),
"types": type_counts
}
except json.JSONDecodeError as e:
return False, f"JSON 解析错误: {e}", {"file": "institution_registry.json"}
except Exception as e:
return False, f"验证失败: {e}", {"file": "institution_registry.json"}
def validate_list_file(filepath: Path) -> Tuple[bool, str, Dict[str, Any]]:
"""
验证单个机构名单文件
Args:
filepath: 文件路径
Returns:
(是否通过, 消息, 详细信息)
"""
filename = filepath.name
try:
with open(filepath, 'r', encoding='utf-8') as f:
data = json.load(f)
# 检查基本结构
if not isinstance(data, dict):
return False, "文件格式不正确,应为 JSON 对象", {"file": filename}
# 检查 type 字段
if 'type' not in data:
return False, "缺少 type 字段", {"file": filename}
# 检查 institutions 字段
if 'institutions' not in data:
return False, "缺少 institutions 字段", {"file": filename}
institutions = data['institutions']
if not isinstance(institutions, list):
return False, "institutions 字段应为数组", {"file": filename}
if len(institutions) == 0:
return False, "机构列表为空", {"file": filename}
# 检查每条记录
invalid_records = []
for i, inst in enumerate(institutions):
if not isinstance(inst, dict):
invalid_records.append(f"记录 {i}: 不是对象")
continue
if 'name' not in inst:
invalid_records.append(f"记录 {i}: 缺少 name 字段")
if invalid_records:
return False, f"数据格式错误", {
"file": filename,
"errors": invalid_records[:5] # 只显示前5个错误
}
# 提取元数据
metadata = {
"file": filename,
"type": data.get('type', '未知'),
"count": len(institutions),
"data_source": data.get('data_source', '未知'),
"update_time": data.get('update_time', '未知')
}
return True, f"验证通过,共 {len(institutions)} 家机构", metadata
except json.JSONDecodeError as e:
return False, f"JSON 解析错误: {e}", {"file": filename}
except Exception as e:
return False, f"验证失败: {e}", {"file": filename}
def validate_all_list_files() -> Dict[str, Tuple[bool, str, Dict[str, Any]]]:
"""
验证所有机构名单文件
Returns:
{文件名: (是否通过, 消息, 详细信息)}
"""
results = {}
# 查找所有 *_list.json 文件
list_files = sorted(SKILL_DATA_DIR.glob("*_list.json"))
if not list_files:
return {"_error": (False, "未找到 *_list.json 文件", {"dir": str(SKILL_DATA_DIR)})}
for filepath in list_files:
results[filepath.name] = validate_list_file(filepath)
return results
def run_full_validation() -> Dict[str, Any]:
"""
运行完整验证
Returns:
验证结果字典
"""
results = {
"registry": None,
"list_files": {},
"summary": {
"total": 0,
"passed": 0,
"failed": 0,
"total_institutions": 0,
"types": {}
}
}
# 验证注册表
registry_ok, registry_msg, registry_details = validate_institution_registry()
results["registry"] = {
"passed": registry_ok,
"message": registry_msg,
"details": registry_details
}
results["summary"]["total"] += 1
if registry_ok:
results["summary"]["passed"] += 1
results["summary"]["total_institutions"] += registry_details.get("total", 0)
else:
results["summary"]["failed"] += 1
# 验证名单文件
list_results = validate_all_list_files()
for filename, (passed, message, details) in list_results.items():
results["list_files"][filename] = {
"passed": passed,
"message": message,
"details": details
}
results["summary"]["total"] += 1
if passed:
results["summary"]["passed"] += 1
results["summary"]["total_institutions"] += details.get("count", 0)
inst_type = details.get("type", "未知")
results["summary"]["types"][inst_type] = details.get("count", 0)
else:
results["summary"]["failed"] += 1
return results
def print_validation_report(results: Dict[str, Any]):
"""
打印验证报告
Args:
results: 验证结果
"""
print("\n" + "=" * 60)
print(" 数据完整性验证报告")
print("=" * 60)
# 注册表
registry = results["registry"]
status = "✓" if registry["passed"] else "✗"
print(f"\n{status} institution_registry.json: {registry['message']}")
if registry["passed"] and "types" in registry.get("details", {}):
types = registry["details"]["types"]
for inst_type, count in sorted(types.items(), key=lambda x: -x[1])[:5]:
print(f" - {inst_type}: {count} 家")
# 名单文件
print("\n机构名单文件:")
for filename, result in sorted(results["list_files"].items()):
status = "✓" if result["passed"] else "✗"
print(f" {status} {filename}: {result['message']}")
# 汇总
summary = results["summary"]
print(f"\n{'=' * 60}")
print(f" 汇总")
print(f"{'=' * 60}")
print(f" 文件验证: {summary['passed']}/{summary['total']} 通过")
print(f" 机构总数: {summary['total_institutions']} 家")
if summary.get("types"):
print(f"\n 机构类型分布:")
for inst_type, count in sorted(summary["types"].items(), key=lambda x: -x[1]):
print(f" - {inst_type}: {count} 家")
# 结论
print(f"\n{'=' * 60}")
if summary["failed"] == 0:
print(" ✓ 所有数据文件验证通过!")
else:
print(f" ⚠ 有 {summary['failed']} 个文件验证失败")
print(" 请检查 data/ 目录下的 JSON 文件是否完整")
print("=" * 60)
def main():
"""主函数"""
print("开始数据完整性验证...")
# 运行验证
results = run_full_validation()
# 打印报告
print_validation_report(results)
# 返回状态码
if results["summary"]["failed"] == 0:
sys.exit(0)
else:
sys.exit(1)
# ── v4.6: 日期窗口核验 ──────────────────────────────────────
def validate_date_window(articles: List[Dict[str, Any]],
requested_days: int = 7,
cutoff_date: str = "") -> Dict[str, Any]:
"""验证文章列表是否在请求的日期窗口内。
Args:
articles: 文章列表 [{"title": ..., "publish_time": "2026-07-28", ...}, ...]
requested_days: 请求的时间窗口(天)
cutoff_date: 截止日期 "YYYY-MM-DD"(留空=按 requested_days 自动计算)
Returns:
{
"total": 50,
"in_window": 48,
"out_of_window": 2,
"unparseable": 0,
"out_articles": [{"title": "...", "publish_time": "2026-03-15", "age_days": 137}, ...],
"date_range_actual": {"earliest": "2026-07-24", "latest": "2026-07-30"},
"requested_window": "2026-07-23 ~ 2026-07-30",
"pass": True/False, # True 表示所有文章在窗内
}
"""
from datetime import datetime, timedelta
if cutoff_date:
cutoff = datetime.strptime(cutoff_date, "%Y-%m-%d")
else:
cutoff = datetime.now() - timedelta(days=requested_days)
total = len(articles)
in_window = 0
out_of_window = 0
unparseable = 0
out_articles: List[Dict[str, Any]] = []
dates_found: List[str] = []
for a in articles:
pub = a.get("publish_time", "")
if not pub:
unparseable += 1
continue
try:
pub_str = str(pub)[:10]
pub_dt = datetime.strptime(pub_str, "%Y-%m-%d")
dates_found.append(pub_str)
if pub_dt >= cutoff:
in_window += 1
else:
out_of_window += 1
out_articles.append({
"title": (a.get("title", "") or "")[:60],
"publish_time": pub,
"age_days": (datetime.now() - pub_dt).days,
"source": a.get("source", ""),
})
except (ValueError, IndexError):
unparseable += 1
actual_range = {}
if dates_found:
dates_found.sort()
actual_range = {"earliest": dates_found[0], "latest": dates_found[-1]}
return {
"total": total,
"in_window": in_window,
"out_of_window": out_of_window,
"unparseable": unparseable,
"out_articles": out_articles[:10],
"date_range_actual": actual_range,
"requested_window": f"{cutoff.strftime('%Y-%m-%d')} ~ {datetime.now().strftime('%Y-%m-%d')}",
"pass": out_of_window == 0,
}
if __name__ == "__main__":
main()