Initial commit: cnfinancialscraper v1.0.47

This commit is contained in:
MoBai
2026-09-13 23:43:57 +08:00
commit 126e43db6f
172 changed files with 76156 additions and 0 deletions
+398
View File
@@ -0,0 +1,398 @@
# -*- coding: utf-8 -*-
"""
文档解析模块
支持PDF、Word、Excel等金融文档解析
"""
import json
import re
import os
from pathlib import Path
from typing import Dict, Any, List, Optional
# 文档解析依赖检测
PDF_AVAILABLE = False
DOCX_AVAILABLE = False
XLSX_AVAILABLE = False
try:
import pypdf
PDF_AVAILABLE = True
except ImportError:
pass
try:
import PyPDF2
PDF_AVAILABLE = True
except ImportError:
pass
try:
from docx import Document as DocxDocument
DOCX_AVAILABLE = True
except ImportError:
pass
try:
import openpyxl
XLSX_AVAILABLE = True
except ImportError:
pass
SKILL_DATA_DIR = Path(__file__).parent.parent / "data"
def parse_pdf(file_path: str) -> Dict[str, Any]:
"""
解析PDF文档
Args:
file_path: PDF文件路径
Returns:
解析结果
"""
result = {
"file_type": "pdf",
"file_path": file_path,
"pages": 0,
"text_content": "",
"tables": [],
"financial_data": {}
}
if not PDF_AVAILABLE:
result["error"] = "PyPDF2未安装,请运行: pip install PyPDF2"
return result
try:
with open(file_path, 'rb') as f:
reader = PyPDF2.PdfReader(f)
result["pages"] = len(reader.pages)
# 提取文本
full_text = []
for page in reader.pages:
text = page.extract_text()
if text:
full_text.append(text)
result["text_content"] = "\n\n".join(full_text)
# 尝试提取财务数据
result["financial_data"] = extract_financial_numbers(result["text_content"])
except Exception as e:
result["error"] = f"PDF解析失败: {e}"
return result
def parse_docx(file_path: str) -> Dict[str, Any]:
"""
解析Word文档
Args:
file_path: DOCX文件路径
Returns:
解析结果
"""
result = {
"file_type": "docx",
"file_path": file_path,
"paragraphs": [],
"tables": [],
"text_content": "",
"financial_data": {}
}
if not DOCX_AVAILABLE:
result["error"] = "python-docx未安装,请运行: pip install python-docx"
return result
try:
doc = DocxDocument(file_path)
# 提取段落
paragraphs = []
for para in doc.paragraphs:
if para.text.strip():
paragraphs.append(para.text)
result["paragraphs"] = paragraphs
result["text_content"] = "\n\n".join(paragraphs)
# 提取表格
tables = []
for i, table in enumerate(doc.tables):
table_data = []
for row in table.rows:
row_data = [cell.text.strip() for cell in row.cells]
table_data.append(row_data)
tables.append(table_data)
result["tables"] = tables
# 财务数据
result["financial_data"] = extract_financial_numbers(result["text_content"])
except Exception as e:
result["error"] = f"DOCX解析失败: {e}"
return result
def parse_xlsx(file_path: str, sheet_name: Optional[str] = None) -> Dict[str, Any]:
"""
解析Excel文档
Args:
file_path: Excel文件路径
sheet_name: 指定工作表名称(默认第一个)
Returns:
解析结果
"""
result = {
"file_type": "xlsx",
"file_path": file_path,
"sheets": {},
"financial_data": {}
}
if not XLSX_AVAILABLE:
result["error"] = "openpyxl未安装,请运行: pip install openpyxl"
return result
try:
wb = openpyxl.load_workbook(file_path, data_only=True)
result["sheet_names"] = wb.sheetnames
# 读取指定sheet或全部
sheets_to_read = [sheet_name] if sheet_name else wb.sheetnames
for sheet in sheets_to_read:
ws = wb[sheet]
data = []
for row in ws.iter_rows(values_only=True):
row_data = [str(cell) if cell is not None else "" for cell in row]
if any(row_data):
data.append(row_data)
result["sheets"][sheet] = {
"rows": len(data),
"data": data[:100] # 限制前100行
}
# 财务数据
all_text = json.dumps(result["sheets"])
result["financial_data"] = extract_financial_numbers(all_text)
except Exception as e:
result["error"] = f"Excel解析失败: {e}"
return result
def parse_txt(file_path: str) -> Dict[str, Any]:
"""
解析文本文件
Args:
file_path: 文本文件路径
Returns:
解析结果
"""
result = {
"file_type": "txt",
"file_path": file_path,
"text_content": "",
"financial_data": {}
}
try:
with open(file_path, 'r', encoding='utf-8') as f:
result["text_content"] = f.read()
result["financial_data"] = extract_financial_numbers(result["text_content"])
except Exception as e:
try:
# 尝试GBK编码
with open(file_path, 'r', encoding='gbk') as f:
result["text_content"] = f.read()
result["financial_data"] = extract_financial_numbers(result["text_content"])
except:
result["error"] = f"文本解析失败: {e}"
return result
def extract_financial_numbers(text: str) -> Dict[str, Any]:
"""
从文本中提取财务数字
Args:
text: 文本内容
Returns:
财务数据字典
"""
data = {}
# 提取金额(亿/万)
amount_patterns = [
(r'([+-]?\d+\.?\d*)\s*亿元', 'amount_yi'),
(r'([+-]?\d+\.?\d*)\s*万元', 'amount_wan'),
(r'([+-]?\d+\.?\d*)\s*%,', 'percentage'),
]
amounts = []
percentages = []
for pattern, ptype in amount_patterns:
matches = re.findall(pattern, text)
for match in matches:
try:
value = float(match)
if ptype == 'amount_yi':
amounts.append(value)
elif ptype == 'percentage':
percentages.append(value)
except:
pass
if amounts:
data["amounts_yi"] = amounts[:10] # 取前10个
data["total_yi"] = sum(amounts)
if percentages:
data["percentages"] = percentages[:20]
# 提取常见财务指标
indicators = {
'净利润': r'净利润[::]\s*([+-]?\d+\.?\d*)',
'营业收入': r'营业收入[::]\s*([+-]?\d+\.?\d*)',
'总资产': r'总资产[::]\s*([+-]?\d+\.?\d*)',
'净资产': r'净资产[::]\s*([+-]?\d+\.?\d*)',
'每股收益': r'每股收益[((]EPS[))][::]\s*([+-]?\d+\.?\d*)',
'市盈率': r'市盈率[((]PE[))][::]\s*([+-]?\d+\.?\d*)',
'市净率': r'市净率[((]PB[))][::]\s*([+-]?\d+\.?\d*)',
}
for name, pattern in indicators.items():
match = re.search(pattern, text)
if match:
try:
data[name] = float(match.group(1))
except:
pass
return data
def parse_document(file_path: str, sheet_name: Optional[str] = None) -> Dict[str, Any]:
"""
通用文档解析接口
Args:
file_path: 文件路径
sheet_name: Excel指定工作表
Returns:
解析结果
"""
path = Path(file_path)
suffix = path.suffix.lower()
parsers = {
'.pdf': parse_pdf,
'.docx': parse_docx,
'.doc': parse_docx, # 尝试作为docx
'.xlsx': parse_xlsx,
'.xls': parse_xlsx,
'.txt': parse_txt,
'.md': parse_txt,
'.csv': parse_txt,
}
parser = parsers.get(suffix)
if not parser:
return {"error": f"不支持的文件类型: {suffix}"}
if suffix == '.xlsx' or suffix == '.xls':
return parser(file_path, sheet_name)
else:
return parser(file_path)
def extract_key_info(parsed_doc: Dict[str, Any]) -> str:
"""
从解析结果中提取关键信息摘要
Args:
parsed_doc: parse_document的返回结果
Returns:
关键信息摘要文本
"""
if "error" in parsed_doc:
return f"解析错误: {parsed_doc['error']}"
summary = f"【文档解析结果】\n\n"
summary += f"文件类型: {parsed_doc.get('file_type', 'unknown').upper()}\n"
if "pages" in parsed_doc:
summary += f"页数: {parsed_doc['pages']}\n"
if "paragraphs" in parsed_doc:
summary += f"段落数: {len(parsed_doc['paragraphs'])}\n"
if "tables" in parsed_doc:
summary += f"表格数: {len(parsed_doc['tables'])}\n"
if "sheets" in parsed_doc:
summary += f"工作表: {', '.join(parsed_doc.get('sheet_names', []))}\n"
# 财务数据摘要
fin_data = parsed_doc.get("financial_data", {})
if fin_data:
summary += "\n【提取的财务数据】\n"
for key, value in fin_data.items():
if key == "percentages":
summary += f"百分比数据: {value[:5]}...\n" if len(value) > 5 else f"百分比数据: {value}\n"
elif key == "amounts_yi":
summary += f"金额数据(亿元): {value[:5]}...\n" if len(value) > 5 else f"金额数据(亿元): {value}\n"
elif key == "total_yi":
summary += f"总金额: {value:.2f}亿元\n"
else:
summary += f"{key}: {value}\n"
# 文本预览
text = parsed_doc.get("text_content", "")
if text:
preview = text[:500].replace('\n', ' ')
summary += f"\n【内容预览】\n{preview}..."
return summary
if __name__ == "__main__":
import sys
if len(sys.argv) < 2:
print("用法: python document_parser.py <文件路径>")
print("示例: python document_parser.py ./report.pdf")
print("\n支持格式: PDF, DOCX, XLSX, TXT, MD, CSV")
sys.exit(1)
file_path = sys.argv[1]
if not os.path.exists(file_path):
print(f"文件不存在: {file_path}")
sys.exit(1)
print(f"正在解析: {file_path}\n")
result = parse_document(file_path)
print(extract_key_info(result))