Files
cnfinancialscraper/scripts/research_report_scraper.py

722 lines
25 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""
券商研报爬虫模块
支持爬取买方评级、估值模型、行业分析等研报
"""
import json
import re
import time
from pathlib import Path
from typing import Dict, Any, List, Optional, Callable
from datetime import datetime
from dataclasses import dataclass, field
try:
import requests
REQUESTS_AVAILABLE = True
except ImportError:
REQUESTS_AVAILABLE = False
try:
from scrapling.fetchers import StealthyFetcher
SCRAPLING_AVAILABLE = True
except ImportError:
SCRAPLING_AVAILABLE = False
SKILL_DATA_DIR = Path(__file__).parent.parent / "data"
BROKER_REPORT_DIR = SKILL_DATA_DIR / "broker_reports"
@dataclass
class BrokerReport:
"""券商研报信息"""
stock_code: str
stock_name: str
title: str
broker_name: str # 券商名称
analyst: str = "" # 分析师
report_type: str = "" # 行业分析/个股研报/策略报告/业绩点评
rating: str = "" # 买入/增持/中性/减持/卖出
target_price: float = 0.0
target_change_pct: float = 0.0 # 目标涨幅
publish_date: str = ""
url: str = ""
file_type: str = "pdf"
is_downloaded: bool = False
local_path: str = ""
@dataclass
class BrokerReportStats:
"""研报统计"""
total_count: int = 0
buy_count: int = 0 # 买入
increase_count: int = 0 # 增持
neutral_count: int = 0 # 中性
reduce_count: int = 0 # 减持
sell_count: int = 0 # 卖出
class EastMoneyBrokerReportAPI:
"""东方财富研报API"""
# 研报类型映射
REPORT_TYPE_MAP = {
"行业": "industry",
"个股": "stock",
"策略": "strategy",
"业绩": "performance",
"宏观": "macro",
"债券": "bond",
"海外": "overseas"
}
# 评级关键词
RATING_KEYWORDS = {
"买入": ["买入", "推荐", "强烈推荐", "增持", "超配", "推荐-A", "推荐-B"],
"中性": ["中性", "持有", "观望", "标配", "中性-A", "中性-B", "同步大市"],
"减持": ["减持", "回避", "卖出", "低配", "规避", "卖出-A", "卖出-B"]
}
def __init__(self):
self.session = None
if REQUESTS_AVAILABLE:
try:
from .http_utils import RatelimitedSession
except ImportError:
from http_utils import RatelimitedSession # type: ignore
self.session = RatelimitedSession(requests.Session())
self.session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Referer': 'https://www.eastmoney.com'
})
self.base_url = "https://datacenter-web.eastmoney.com/api/data/v1/get"
def get_stock_reports(self, stock_code: str,
start_date: str = "",
end_date: str = "",
max_results: int = 50) -> List[BrokerReport]:
"""
获取个股研报
Args:
stock_code: 股票代码
start_date: 开始日期 YYYY-MM-DD
end_date: 结束日期 YYYY-MM-DD
max_results: 最大结果数
Returns:
研报列表
"""
if not self.session:
return []
reports = []
code = stock_code.replace('.SH', '').replace('.SZ', '').replace('.BJ', '')
try:
# 东方财富研报API
url = "https://report.eastmoney.com/api/security/announcement"
params = {
"sr": -1,
"page_size": max_results,
"page_index": 1,
"ann_type": "NOTICE",
"client_source": "web",
"stock_list": code
}
if start_date:
params["begin_date"] = start_date.replace('-', '')
if end_date:
params["end_date"] = end_date.replace('-', '')
resp = self.session.get(url, params=params, timeout=30)
data = resp.json()
if data.get('data') and data['data'].get('list'):
for item in data['data']['list']:
report = self._parse_report_item(item, code)
if report:
reports.append(report)
except Exception as e:
print(f"[错误] 获取研报失败: {e}")
# 如果API失败,尝试备用方法:直接从搜索引擎数据
if not reports:
reports = self._search_reports_fallback(code, max_results)
return reports
def _search_reports_fallback(self, stock_code: str,
max_results: int = 50) -> List[BrokerReport]:
"""备用搜索方法:从东方财富研报页面获取"""
reports = []
try:
# 尝试东方财富研报中心API
url = self.base_url
params = {
"reportName": "RPT_BROKER_NEWS",
"columns": "ALL",
"filter": f"(SECUCODE%3D%22{stock_code}.SH%22)",
"pageNumber": 1,
"pageSize": max_results,
"source": "WEB",
"client": "WEB"
}
resp = self.session.get(url, params=params, timeout=30)
data = resp.json()
if data.get('result') and data['result'].get('data'):
for item in data['result']['data']:
report = self._parse_datacenter_item(item)
if report:
reports.append(report)
except Exception:
pass
return reports
def _parse_report_item(self, item: Dict, stock_code: str) -> Optional[BrokerReport]:
"""解析研报项"""
try:
title = item.get('title', '') or item.get('title_ch', '')
if not title:
return None
# 跳过非研报类型的公告
skip_keywords = ['上市', '发行', '审核', '批复', '决定', '通知']
if any(kw in title for kw in skip_keywords):
return None
# 判断研报类型
report_type = self._identify_report_type(title)
# 提取券商名称
broker_name = self._extract_broker_name(title)
# 提取评级
rating = self._extract_rating(title)
# 提取分析师
analyst = item.get('author', '') or item.get('analyst', '') or ""
# 提取目标价
target_price, target_change = self._extract_target_price(title)
# 公告时间
publish_date = item.get('notice_date', '') or item.get('display_time', '')
if len(publish_date) >= 10:
publish_date = publish_date[:10]
# URL
art_url = item.get('art_url', '') or item.get('globalId', '')
if art_url and not art_url.startswith('http'):
art_url = f"https://data.eastmoney.com{art_url}"
return BrokerReport(
stock_code=stock_code,
stock_name=item.get('stock_name', '') or item.get('secuName', ''),
title=title,
broker_name=broker_name,
analyst=analyst,
report_type=report_type,
rating=rating,
target_price=target_price,
target_change_pct=target_change,
publish_date=publish_date,
url=art_url,
file_type='pdf' if '.pdf' in art_url.lower() else 'html'
)
except Exception:
return None
def _parse_datacenter_item(self, item: Dict) -> Optional[BrokerReport]:
"""解析数据中心研报项"""
try:
title = item.get('NOTICETITLE', '') or item.get('title', '')
if not title or len(title) < 5:
return None
report_type = self._identify_report_type(title)
broker_name = self._extract_broker_name(title)
rating = self._extract_rating(title)
# 目标价
target_price = 0.0
target_change = 0.0
if 'TARGETPRICE' in item:
try:
target_price = float(item['TARGETPRICE'])
except (ValueError, TypeError):
pass
publish_date = item.get('NOTICEDATE', '') or item.get('publish_date', '')
if isinstance(publish_date, str) and len(publish_date) >= 10:
publish_date = publish_date[:10]
return BrokerReport(
stock_code=item.get('SECUCODE', '').replace('.SH', '').replace('.SZ', ''),
stock_name=item.get('SECUNAME', '') or item.get('stock_name', ''),
title=title,
broker_name=broker_name,
analyst=item.get('ANALYST', '') or item.get('author', ''),
report_type=report_type,
rating=rating,
target_price=target_price,
publish_date=publish_date,
url=item.get('URL', '') or item.get('art_url', ''),
file_type='pdf'
)
except Exception:
return None
def _identify_report_type(self, title: str) -> str:
"""识别研报类型"""
title_upper = title.upper()
type_patterns = [
(["行业", "INDUSTRY"], "行业分析"),
(["个股", "STOCK", "公司研究"], "个股研报"),
(["策略", "STRATEGY"], "策略报告"),
(["业绩", "PERFORMANCE", "财报"], "业绩点评"),
(["宏观", "MACRO", "经济"], "宏观研究"),
(["债券", "BOND", "信用"], "债券研究"),
]
for keywords, report_type in type_patterns:
for kw in keywords:
if kw in title_upper:
return report_type
return "个股研报" # 默认
def _extract_broker_name(self, title: str) -> str:
"""提取券商名称"""
# 常见券商简称模式
broker_patterns = [
r'^(.+?)\s*(?:研究|策略|行业|宏观)',
r'^(.+?)\s*(?:证券|投行|银行)',
r'\[(.+?)\]',
]
for pattern in broker_patterns:
match = re.search(pattern, title)
if match:
name = match.group(1).strip()
if len(name) >= 2 and len(name) <= 10:
return name
return "未知券商"
def _extract_rating(self, title: str) -> str:
"""提取评级"""
title_upper = title.upper()
for rating, keywords in self.RATING_KEYWORDS.items():
for kw in keywords:
if kw in title_upper:
return rating
return "未知"
def _extract_target_price(self, title: str) -> tuple:
"""提取目标价和目标涨幅"""
target_price = 0.0
target_change = 0.0
# 目标价模式:目标价XX元、target price XX
price_match = re.search(r'(?:目标价|target\s*price|目标价格)\s*[::]?\s*([\d.]+)\s*(?:元|RMB)?', title, re.I)
if price_match:
try:
target_price = float(price_match.group(1))
except ValueError:
pass
# 目标涨幅模式:上涨XX%、涨幅XX%
change_match = re.search(r'(?:上涨|涨幅|上升|提升)\s*([\d.]+)%', title, re.I)
if change_match:
try:
target_change = float(change_match.group(1))
except ValueError:
pass
return target_price, target_change
def search_reports_by_keyword(self, keyword: str,
start_date: str = "",
end_date: str = "",
max_results: int = 50) -> List[BrokerReport]:
"""按关键词搜索研报"""
if not self.session:
return []
reports = []
try:
# 使用东方财富搜索API
url = "https://search-api.eastmoney.com/search/jsonp"
params = {
"cb": "callback",
"param": json.dumps({
"uid": "",
"keyword": keyword,
"type": ["researchReport"],
"client": "web",
"param": {
"researchReport": {
"fields": ["title", "time", "url", "broker", "analyst", "rating"],
"pageSize": max_results,
"pageIndex": 1
}
}
}, ensure_ascii=False)
}
resp = self.session.get(url, params=params, timeout=30)
text = resp.text
# 解析JSONP
json_match = re.search(r'callback\((.*)\)', text, re.DOTALL)
if json_match:
data = json.loads(json_match.group(1))
if data.get('result') and data['result'].get('researchReport'):
for item in data['result']['researchReport']:
report = self._parse_search_result(item)
if report:
reports.append(report)
except Exception as e:
print(f"[错误] 搜索研报失败: {e}")
return reports
def _parse_search_result(self, item: Dict) -> Optional[BrokerReport]:
"""解析搜索结果"""
try:
title = item.get('title', '')
if not title:
return None
return BrokerReport(
stock_code="",
stock_name="",
title=title,
broker_name=item.get('broker', ''),
analyst=item.get('analyst', ''),
report_type=self._identify_report_type(title),
rating=item.get('rating', ''),
publish_date=item.get('time', '')[:10] if item.get('time') else '',
url=item.get('url', ''),
file_type='pdf'
)
except Exception:
return None
def get_top_reports(self, broker_name: str = "",
min_rating: str = "买入",
limit: int = 20) -> List[BrokerReport]:
"""
获取热门研报
Args:
broker_name: 券商名称(空则所有)
min_rating: 最低评级
limit: 返回数量
Returns:
研报列表
"""
if not self.session:
return []
reports = []
try:
url = self.base_url
params = {
"reportName": "RPT_BROKER_NEWS",
"columns": "ALL",
"pageNumber": 1,
"pageSize": limit,
"source": "WEB",
"client": "WEB",
"sortColumns": "NOTICEDATE",
"sortTypes": "-1"
}
resp = self.session.get(url, params=params, timeout=30)
data = resp.json()
if data.get('result') and data['result'].get('data'):
for item in data['result']['data']:
report = self._parse_datacenter_item(item)
if report:
# 过滤
# v4.4.0 修复:精确匹配而非子串匹配,避免 "中信" 误匹配 "中信建投"
if broker_name and broker_name != report.broker_name:
continue
if not self._rating_meets_threshold(report.rating, min_rating):
continue
reports.append(report)
except Exception as e:
print(f"[错误] 获取热门研报失败: {e}")
return reports[:limit]
def _rating_meets_threshold(self, rating: str, min_rating: str) -> bool:
"""判断评级是否满足阈值"""
rating_order = {"买入": 3, "增持": 2, "中性": 1, "减持": 0, "卖出": -1}
rating_val = rating_order.get(rating, 0)
min_val = rating_order.get(min_rating, 0)
return rating_val >= min_val
class BrokerReportDownloader:
"""研报下载器"""
def __init__(self, download_dir: str = None):
self.download_dir = Path(download_dir) if download_dir else BROKER_REPORT_DIR
self.download_dir.mkdir(parents=True, exist_ok=True)
self.session = None
if REQUESTS_AVAILABLE:
try:
from .http_utils import RatelimitedSession
except ImportError:
from http_utils import RatelimitedSession # type: ignore
self.session = RatelimitedSession(requests.Session())
self.session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
})
self.downloaded_count = 0
self.failed_count = 0
def download_report(self, report: BrokerReport) -> str:
"""下载单条研报"""
if not report.url:
return ""
save_dir = self.download_dir / report.stock_code
save_dir.mkdir(parents=True, exist_ok=True)
filename = self._generate_filename(report)
save_path = save_dir / filename
if save_path.exists():
self.downloaded_count += 1
report.is_downloaded = True
report.local_path = str(save_path)
return str(save_path)
try:
if self.session:
resp = self.session.get(report.url, timeout=60, stream=True)
if resp.status_code == 200:
content_type = resp.headers.get('Content-Type', '')
actual_name = self._get_filename_from_cd(resp.headers)
if actual_name:
save_path = save_dir / actual_name
with open(save_path, 'wb') as f:
for chunk in resp.iter_content(chunk_size=8192):
f.write(chunk)
self.downloaded_count += 1
report.is_downloaded = True
report.local_path = str(save_path)
return str(save_path)
except Exception as e:
print(f"[错误] 下载失败: {report.url}, {e}")
self.failed_count += 1
return ""
def _get_filename_from_cd(self, headers) -> Optional[str]:
"""从Content-Disposition获取文件名"""
cd = headers.get('Content-Disposition', '')
match = re.search(r'filename[^=]*=\s*"?([^";]+)"?', cd)
if match:
return match.group(1).strip('" ')
return None
def _generate_filename(self, report: BrokerReport) -> str:
"""生成文件名"""
safe_title = re.sub(r'[<>:"/\\|?*]', '', report.title)
safe_title = safe_title[:50]
date_str = report.publish_date.replace("-", "") if report.publish_date else ""
broker = re.sub(r'[<>:"/\\|?*]', '', report.broker_name)[:10]
ext = ".pdf" if report.file_type == "pdf" else ".html"
return f"{date_str}_{broker}_{safe_title}{ext}"
def batch_download(self, reports: List[BrokerReport],
progress_callback: Callable = None) -> Dict[str, Any]:
"""批量下载"""
self.downloaded_count = 0
self.failed_count = 0
results = []
total = len(reports)
for i, report in enumerate(reports):
local_path = self.download_report(report)
results.append({
"report": report,
"local_path": local_path,
"success": bool(local_path)
})
if progress_callback:
progress_callback(i + 1, total)
return {
"total": total,
"downloaded": self.downloaded_count,
"failed": self.failed_count,
"results": results
}
class BrokerReportManager:
"""研报管理器"""
def __init__(self):
self.api = EastMoneyBrokerReportAPI()
self.downloader = BrokerReportDownloader()
def get_reports_all(self, stock_code: str, max_results: int = 50) -> List[BrokerReport]:
"""获取个股所有研报"""
return self.api.get_stock_reports(stock_code, max_results=max_results)
def get_reports_by_broker(self, broker_name: str, limit: int = 20) -> List[BrokerReport]:
"""按券商筛选研报"""
return self.api.get_top_reports(broker_name=broker_name, limit=limit)
def download_reports(self, stock_code: str,
min_rating: str = "中性",
progress_callback: Callable = None) -> Dict[str, Any]:
"""下载研报"""
reports = self.get_reports_all(stock_code)
# 按评级过滤
if min_rating:
rating_order = {"买入": 3, "增持": 2, "中性": 1, "减持": 0, "卖出": -1}
min_val = rating_order.get(min_rating, 0)
reports = [r for r in reports if rating_order.get(r.rating, 0) >= min_val]
return self.downloader.batch_download(reports, progress_callback)
def generate_report_summary(self, stock_code: str) -> str:
"""生成研报摘要"""
reports = self.get_reports_all(stock_code)
if not reports:
return f"未找到 {stock_code} 的研报记录"
lines = []
lines.append(f"\n{'='*60}")
lines.append(f"【{stock_code} 券商研报】")
lines.append(f"{'='*60}")
# 统计
stats = BrokerReportStats()
stats.total_count = len(reports)
for r in reports:
if r.rating == "买入":
stats.buy_count += 1
elif r.rating == "增持":
stats.increase_count += 1
elif r.rating == "中性":
stats.neutral_count += 1
elif r.rating == "减持":
stats.reduce_count += 1
elif r.rating == "卖出":
stats.sell_count += 1
lines.append(f"\n总计: {stats.total_count}份")
lines.append(f" 买入: {stats.buy_count} | 增持: {stats.increase_count} | 中性: {stats.neutral_count} | 减持: {stats.reduce_count}")
# 按券商分组
by_broker = {}
for r in reports:
broker = r.broker_name or "未知券商"
if broker not in by_broker:
by_broker[broker] = []
by_broker[broker].append(r)
lines.append(f"\n按券商 ({len(by_broker)}家):")
for broker, broker_reports in sorted(by_broker.items(), key=lambda x: -len(x[1]))[:10]:
lines.append(f"\n {broker} ({len(broker_reports)}份):")
for r in broker_reports[:3]:
rating_icon = {"买入": "++", "增持": "+", "中性": "=", "减持": "-", "卖出": "--"}.get(r.rating, "?")
lines.append(f" [{rating_icon}] {r.publish_date[:10] if r.publish_date else 'N/A'} {r.title[:35]}")
lines.append(f"\n{'='*60}")
return "\n".join(lines)
# ============ CLI入口 ============
if __name__ == "__main__":
import sys
if len(sys.argv) < 2:
print("用法:")
print(" python research_report_scraper.py search <股票代码> # 搜索研报")
print(" python research_report_scraper.py top [券商] # 热门研报")
print(" python research_report_scraper.py broker <券商> # 按券商筛选")
sys.exit(1)
cmd = sys.argv[1]
if cmd == "search":
if len(sys.argv) < 3:
print("请提供股票代码")
sys.exit(1)
code = sys.argv[2]
manager = BrokerReportManager()
print(manager.generate_report_summary(code))
elif cmd == "top":
broker = sys.argv[2] if len(sys.argv) > 2 else ""
api = EastMoneyBrokerReportAPI()
reports = api.get_top_reports(broker_name=broker, limit=20)
print(f"\n{'='*60}")
print(f"【热门研报】" + (f" - {broker}" if broker else ""))
print(f"{'='*60}")
for r in reports:
rating_icon = {"买入": "++", "增持": "+", "中性": "=", "减持": "-", "卖出": "--"}.get(r.rating, "?")
print(f"{rating_icon} {r.broker_name} | {r.publish_date[:10] if r.publish_date else 'N/A'} | {r.title[:40]}")
if r.stock_code:
print(f" 股票: {r.stock_code} {r.stock_name}")
print(f"{'='*60}")
elif cmd == "broker":
if len(sys.argv) < 3:
print("请提供券商名称")
sys.exit(1)
broker = sys.argv[2]
manager = BrokerReportManager()
reports = manager.get_reports_by_broker(broker, limit=20)
print(f"\n{'='*60}")
print(f"【{broker} 研报】")
print(f"{'='*60}")
for r in reports:
print(f" [{r.rating}] {r.publish_date[:10] if r.publish_date else 'N/A'} {r.title[:40]}")
print(f"{'='*60}")
else:
print(f"未知命令: {cmd}")