# -*- coding: utf-8 -*- """ 金融监管与政策资讯爬虫 v2.0 (regulatory_scraper.py) v6.0.0 起数据源全面切换为商业财经数据源(不依赖任何政府网站): 1. 东方财富资讯 API(宏观 / 政策 / 监管新闻) 2. 新浪财经滚动新闻 API(财经要闻) 3. 巨潮资讯公告查询 API(A股上市公司公告) 提取:政策动态、财经要闻、监管新闻、A股公告。 用法: from regulatory_scraper import RegulatoryScraper, get_regulatory_updates scraper = RegulatoryScraper() news = scraper.get_macro_policy_news(limit=10) """ import json import re import logging import time from datetime import datetime from typing import Dict, List, Optional, Any from pathlib import Path try: from bs4 import BeautifulSoup # type: ignore HAS_BS4 = True except ImportError: HAS_BS4 = False try: from .http_utils import get_session HAS_HTTP_UTILS = True except ImportError: try: from http_utils import get_session HAS_HTTP_UTILS = True except ImportError: HAS_HTTP_UTILS = False logger = logging.getLogger("regulatory_scraper") # ─── 商业数据源 API(v6.0.0 起)──────────────────────────────────────────── # 东方财富资讯列表(column 345=宏观/政策,344=财经导读) EM_NEWS_API = ("https://np-listapi.eastmoney.com/comm/web/getNewsByColumns" "?client=web&biz=web_news_col&column={column}&order=1" "&needInteractData=0&page_index=1&page_size={size}&req_trace={ts}") # 新浪财经滚动新闻(lid 2516=财经/宏观,2510=要闻) SINA_ROLL_API = ("https://feed.mix.sina.com.cn/api/roll/get?pageid=153" "&lid={lid}&k=&num={size}&page=1") # 巨潮资讯公告查询(POST) CNINFO_QUERY_API = "https://www.cninfo.com.cn/new/hisAnnouncement/query" class RegulatoryScraper: """金融监管与政策资讯爬虫 — 东财 / 新浪 / 巨潮(商业数据源)""" def __init__(self): self.session = None self._init_session() def _init_session(self): """初始化请求会话""" if HAS_HTTP_UTILS: try: self.session = get_session() except Exception: pass if self.session is None: import requests self.session = requests.Session() self.session.headers.update({ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "Accept": "application/json,text/html,*/*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", }) def _fetch_html(self, url: str, timeout: int = 30) -> Optional[str]: """获取页面内容,自动检测编码""" try: resp = self.session.get(url, timeout=timeout) # 兼容 requests 和 stdlib response 的编码处理 if hasattr(resp, 'apparent_encoding'): resp.encoding = resp.apparent_encoding or resp.encoding or "utf-8" elif hasattr(resp, 'encoding') and resp.encoding: try: resp.encoding = resp.encoding except Exception: pass # stdlib response 可能不支持设置 encoding return resp.text if hasattr(resp, 'text') else resp.content.decode("utf-8", errors="replace") except Exception as e: logger.warning(f"获取页面失败: {url}, {e}") return None def _fetch_json(self, url: str, timeout: int = 30) -> Optional[Any]: """GET 请求并解析 JSON""" try: resp = self.session.get(url, timeout=timeout) if hasattr(resp, "json"): return resp.json() return json.loads(resp.content.decode("utf-8", errors="replace")) except Exception as e: logger.warning(f"JSON 获取失败: {url}, {e}") return None def _fetch_json_post(self, url: str, payload: Dict[str, Any], timeout: int = 30) -> Optional[Any]: """POST 请求并解析 JSON(巨潮公告接口需要)""" headers = { "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", "Referer": "http://www.cninfo.com.cn/new/commonUrl/pageOfSearch?url=disclosure/list/search", "Origin": "http://www.cninfo.com.cn", "X-Requested-With": "XMLHttpRequest", } try: if hasattr(self.session, "post"): resp = self.session.post(url, data=payload, headers=headers, timeout=timeout) else: from urllib import request as ur req = ur.Request(url, data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST") with ur.urlopen(req, timeout=timeout) as r: return json.loads(r.read().decode("utf-8", errors="replace")) if hasattr(resp, "json"): return resp.json() return json.loads(resp.content.decode("utf-8", errors="replace")) except Exception as e: logger.warning(f"POST JSON 失败: {url}, {e}") return None def _parse_html_links(self, html: str, base_url: str, list_selector: str = "a") -> List[Dict[str, str]]: """从 HTML 中提取链接列表""" if not html or not HAS_BS4: return [] soup = BeautifulSoup(html, "html.parser") results = [] for link in soup.select(list_selector): title = link.get_text(strip=True) href = link.get("href", "") if not title or len(title) < 4: continue if href and not href.startswith("http"): if href.startswith("/"): href = base_url.rstrip("/") + href elif href.startswith("./"): href = base_url.rstrip("/") + href[1:] else: href = base_url.rstrip("/") + "/" + href results.append({"title": title, "url": href, "date": ""}) return results # ==================== 东方财富资讯(宏观 / 政策) ==================== def get_macro_policy_news(self, limit: int = 20) -> List[Dict[str, str]]: """获取宏观政策 / 财经监管资讯(东方财富 column=345)。 返回 [{title, url, source, date}, ...],失败时返回空列表。 """ results: List[Dict[str, str]] = [] url = EM_NEWS_API.format(column=345, size=limit, ts=int(time.time() * 1000)) data = self._fetch_json(url) try: items = (data or {}).get("data", {}).get("list", []) or [] except AttributeError: items = [] for it in items: title = (it.get("title") or "").strip() href = it.get("url") or "" date = (it.get("showTime") or "")[:10] if title and href: results.append({"title": title, "url": href, "source": "东方财富-宏观政策", "date": date}) if len(results) >= limit: break return results[:limit] def get_market_news(self, limit: int = 20) -> List[Dict[str, str]]: """获取财经要闻(东方财富 column=344)。""" results: List[Dict[str, str]] = [] url = EM_NEWS_API.format(column=344, size=limit, ts=int(time.time() * 1000)) data = self._fetch_json(url) try: items = (data or {}).get("data", {}).get("list", []) or [] except AttributeError: items = [] for it in items: title = (it.get("title") or "").strip() href = it.get("url") or "" date = (it.get("showTime") or "")[:10] if title and href: results.append({"title": title, "url": href, "source": "东方财富-财经要闻", "date": date}) if len(results) >= limit: break return results[:limit] # ==================== 新浪财经滚动新闻 ==================== def get_sina_finance_news(self, limit: int = 20, lid: int = 2516) -> List[Dict[str, str]]: """获取新浪财经滚动新闻(lid=2516 财经/宏观,2510 要闻)。""" results: List[Dict[str, str]] = [] url = SINA_ROLL_API.format(lid=lid, size=limit) data = self._fetch_json(url) try: items = (data or {}).get("result", {}).get("data", []) or [] except AttributeError: items = [] for it in items: title = (it.get("title") or "").strip() href = it.get("url") or "" ctime = it.get("ctime", "") date = "" if ctime: try: date = datetime.fromtimestamp(int(ctime)).strftime("%Y-%m-%d") except (ValueError, OSError, OverflowError): date = "" if title and href: results.append({"title": title, "url": href, "source": "新浪财经-滚动", "date": date}) if len(results) >= limit: break return results[:limit] # ==================== 巨潮资讯公告 ==================== def get_announcements(self, limit: int = 20, market: str = "szse") -> List[Dict[str, str]]: """获取 A股上市公司公告(巨潮资讯查询 API)。""" results: List[Dict[str, str]] = [] payload = { "pageNum": 1, "pageSize": min(limit, 30), "column": "szse", "tabName": "fulltext", "plate": "", "stock": "", "searchkey": "", "secid": "", "category": "", "trade": "", "seDate": "", "sortName": "", "sortType": "", "isHLtitle": "true", } data = self._fetch_json_post(CNINFO_QUERY_API, payload) try: items = (data or {}).get("announcements") or [] except AttributeError: items = [] for it in items: title = (it.get("announcementTitle") or "").strip() adj = (it.get("adjunctUrl") or "").strip() href = f"http://static.cninfo.com.cn/{adj}" if adj else "" ts = it.get("announcementTime") or 0 date = "" if ts: try: date = datetime.fromtimestamp(int(ts) / 1000).strftime("%Y-%m-%d") except (ValueError, OSError, OverflowError): date = "" if title and href: results.append({"title": title, "url": href, "source": "巨潮资讯-公告", "date": date}) if len(results) >= limit: break return results[:limit] # ==================== 跨源搜索 ==================== def search_all(self, keyword: str, limit: int = 30) -> List[Dict[str, str]]: """跨东财 / 新浪 / 巨潮搜索关键词。""" results = [] for source, func in [ ("EM", self.get_macro_policy_news), ("SINA", self.get_sina_finance_news), ("CNINFO", self.get_announcements), ]: try: items = func(limit=limit) for item in items: if keyword.lower() in item.get("title", "").lower(): item["source"] = f"{source} - {item.get('source', source)}" results.append(item) except Exception as e: logger.warning(f"{source} 搜索失败: {e}") return results[:limit] # ==================== 政策文件下载与分析(保留) ==================== def download_document(self, url: str, save_dir: Optional[str] = None) -> str: """下载政策文件/公告到本地。 支持 PDF / DOC / DOCX / HTML / 纯文本附件。 Args: url: 文件链接 save_dir: 保存目录,默认 data/regulatory_docs/ Returns: 保存的文件路径;失败返回空字符串 """ if not url: return "" import os save_dir = save_dir or str(Path(__file__).resolve().parent.parent / "data" / "regulatory_docs") os.makedirs(save_dir, exist_ok=True) # 推断文件名 path_part = url.split("?")[0].rstrip("/") fname = path_part.split("/")[-1] or f"doc_{int(time.time())}.html" fname = re.sub(r'[\\/:*?"<>|]', "_", fname) if "." not in fname: fname += ".html" save_path = os.path.join(save_dir, fname) try: resp = self.session.get(url, timeout=30, stream=True) content_type = (resp.headers.get("Content-Type", "") if hasattr(resp, "headers") else "") or "" # 若文件名无扩展名,根据 Content-Type 推断 if fname.endswith((".pdf", ".doc", ".docx", ".xls", ".xlsx", ".html", ".htm")): pass elif "pdf" in content_type.lower(): save_path = os.path.splitext(save_path)[0] + ".pdf" elif "word" in content_type.lower() or "doc" in content_type.lower(): save_path = os.path.splitext(save_path)[0] + ".doc" elif "html" in content_type.lower(): save_path = os.path.splitext(save_path)[0] + ".html" if hasattr(resp, "iter_content"): with open(save_path, "wb") as f: for chunk in resp.iter_content(chunk_size=8192): if chunk: f.write(chunk) else: data = resp.content if hasattr(resp, "content") else resp.read() with open(save_path, "wb") as f: f.write(data) logger.info(f"政策文件已下载: {save_path}") return save_path except Exception as e: logger.warning(f"政策文件下载失败: {url}, {e}") return "" def analyze_policy(self, url_or_path: str, keyword: str = "") -> Dict[str, Any]: """下载并分析政策文件/公告,提取关键信息。 支持 PDF(pypdf)/ HTML / 纯文本。返回结构化分析结果, 包括标题、发布时间、正文文本、关键词命中、内容摘要。 Args: url_or_path: 政策文件 URL 或本地文件路径 keyword: 关注关键词(如 "利率"、"风险"),命中时高亮 Returns: { "title": str, "date": str, "source_url": str, "content_length": int, "content_preview": str, "keywords_hit": [...], "summary": str, "saved_path": str, "warning": str | None, } """ import os # 判断是 URL 还是本地路径 is_url = url_or_path.startswith("http://") or url_or_path.startswith("https://") local_path = "" html = "" text = "" if is_url: # 先尝试直接抓取 HTML 文本 html = self._fetch_html(url_or_path, timeout=30) or "" # 下载附件(若页面是 PDF 链接则下载) local_path = self.download_document(url_or_path) file_text = self._extract_file_text(local_path) if local_path else "" if file_text and not html: text = file_text else: local_path = url_or_path if os.path.isfile(url_or_path): text = self._extract_file_text(url_or_path) # 从 HTML 提取正文 if html and not text: text = self._html_to_text(html) if not text: return { "error": "无法提取文档内容", "source_url": url_or_path, "warning": "内容为空或文件格式不支持", } # 结构化 title = "" if is_url and html: m = re.search(r"