#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ cn-financial-scraper 环境安装脚本 一键搭建环境: python setup_env.py 功能: 1. 创建虚拟环境 .venv(可选) 2. pip install 依赖 3. 验证核心模块可导入 【环境强制】运行时禁止生成 .pyc,避免污染仓库(项目根 .gitignore 已同步屏蔽二进制) """ import sys sys.dont_write_bytecode = True # 关闭 .pyc 生成(全局 -B 标志同等效果) import os os.environ.setdefault("PYTHONDONTWRITEBYTECODE", "1") import subprocess from pathlib import Path PROJECT_DIR = Path(__file__).parent.resolve() REQUIREMENTS = PROJECT_DIR / "requirements.txt" def print_header(text: str): print(f"\n{'=' * 60}") print(f" {text}") print(f"{'=' * 60}") def print_step(text: str): print(f"\n>>> {text}") def print_ok(text: str): print(f" ✅ {text}") def print_warn(text: str): print(f" ⚠️ {text}") def print_fail(text: str): print(f" ❌ {text}") # ─── 1. Python 版本检查 ──────────────────────────────────────────────────── def check_python_version() -> bool: print_step("检查 Python 版本") v = sys.version_info print(f" 当前: Python {v.major}.{v.minor}.{v.micro}") if v < (3, 8): print_fail(f"需要 Python 3.8+,当前为 {v.major}.{v.minor}") return False print_ok(f"Python {v.major}.{v.minor} 满足要求") return True # ─── 2. 安装依赖 ──────────────────────────────────────────────────────────── TIER_PACKAGES = { "core": [], # 纯标准库,无需 pip 安装 "recommended": ["beautifulsoup4", "lxml", "python-docx", "openpyxl"], "full": None, # None = 安装 requirements.txt 全部 } def install_dependencies(tier: str = "core") -> bool: if tier == "core": print_step("核心模式 - 零外部依赖") print(" cn-financial-scraper 的核心舆情功能使用 Python 标准库") print(" 无需安装任何 pip 包,可直接使用:") print(" python run_sentiment.py") print() print(" 如需增强功能:") print(" python setup_env.py --recommended (HTML解析 + Word/Excel导出)") print(" python setup_env.py --full (全部功能)") return True if not REQUIREMENTS.exists(): print_fail(f"未找到 {REQUIREMENTS}") return False mirrors = [ ("https://pypi.tuna.tsinghua.edu.cn/simple", "pypi.tuna.tsinghua.edu.cn"), ("https://mirrors.aliyun.com/pypi/simple", "mirrors.aliyun.com"), ] if tier == "recommended": print_step(f"推荐模式 - 安装核心增强包") packages = TIER_PACKAGES["recommended"] else: print_step(f"全功能模式 - 安装全部依赖") packages = None # 用 requirements.txt for mirror_url, host in mirrors: if packages: cmd = [sys.executable, "-m", "pip", "install"] + packages else: cmd = [sys.executable, "-m", "pip", "install", "-r", str(REQUIREMENTS)] cmd_full = cmd + ["-i", mirror_url, "--trusted-host", host] print(f" 执行: pip install {' '.join(packages) if packages else 'requirements.txt'}") try: result = subprocess.run(cmd_full, capture_output=True, text=True, timeout=300) if result.returncode == 0: print_ok(f"依赖安装成功 (镜像: {host})") return True else: print_warn(f"镜像 {host} 安装失败,尝试下一个...") except subprocess.TimeoutExpired: print_warn(f"镜像 {host} 超时") except Exception as e: print_warn(f"镜像 {host} 异常: {e}") print_fail("所有镜像安装失败") return False # ─── 3. 核心模块导入验证 ───────────────────────────────────────────────────── def check_core_modules() -> bool: print_step("验证核心模块导入") sys.path.insert(0, str(PROJECT_DIR / "scripts")) modules = [ ("http_utils", "HTTP 基础设施"), ("scraper", "页面爬取器"), ("web_parser", "网页解析器"), ("institution_scraper", "机构爬虫"), ("scrapable_registry", "机构注册表"), ("news_scraper", "新闻爬虫"), ("announcement_scraper", "公告爬虫"), ("research_report_scraper", "券商研报"), ("comprehensive_report_scraper", "综合报告"), ("document_parser", "文档解析"), ("document_analyzer", "文档分析"), ("report_exporter", "报告导出"), ("batch_institution_crawler", "批量爬虫"), ("sina_scraper", "新浪财经"), ("cls_scraper", "财联社"), ("jisilu_scraper", "集思录"), ("exchange_scraper", "交易所"), ("report_stdlib_fallbacks", "标准库文档生成"), # v4.0 新增模块 ("enhanced_parser", "增强文件解析"), ("content_compressor", "内容智能压缩"), ("crawl_scheduler", "定时爬取调度"), ("crawl_packager", "批量打包ZIP"), ("report_templates", "报告模板库"), ("financial_writer", "金融写作引擎"), ("research_report_generator", "研究报告生成"), # v4.1 新增模块 ("overseas_scraper", "海外金融机构爬取"), ("translate_utils", "金融术语翻译"), # v4.2 新增模块 ("browser_scraper", "浏览器自动化爬虫"), # v4.3 新增模块 ("sentiment_crawler", "全网舆情爬虫"), ("sentiment_exporter", "舆情导出器"), ("sentiment_chat", "对话式NLU入口"), ("sentiment_keywords", "舆情关键词库"), # v4.4 新增数据源 ("eastmoney_scraper", "东方财富数据爬虫"), ("cninfo_scraper", "巨潮资讯公告爬虫"), ("regulatory_scraper", "金融监管机构爬虫"), # v4.5 新增模块 ("fullpage_archiver", "全页内容归档器"), ("search_engine", "多搜索引擎聚合"), ("crawl_backtester", "数据回测工具"), # v4.7 新增模块 ("market_data_scraper", "市场数据爬虫"), ] all_ok = True for mod, desc in modules: try: __import__(mod) print_ok(f"{desc} ({mod})") except SyntaxError as e: print_fail(f"{desc}: SyntaxError {e}") all_ok = False except ImportError as e: print_warn(f"{desc}: ImportError - {e}") except Exception as e: print_warn(f"{desc}: {type(e).__name__} - {e}") return all_ok # ─── 4. 可选增强检查 ──────────────────────────────────────────────────────── def check_optional(): print_step("可选增强功能") for pkg, desc in [("scrapling", "动态 JS 渲染"), ("playwright", "浏览器自动化")]: try: __import__(pkg.split(".")[0]) print_ok(f"{pkg} 已安装({desc}可用)") except ImportError: print_warn(f"{pkg} 未安装({desc}不可用,不影响核心功能)") # ─── 主函数 ──────────────────────────────────────────────────────────────── def main(): import argparse ap = argparse.ArgumentParser(description="cn-financial-scraper 环境安装") ap.add_argument("--recommended", action="store_true", help="安装推荐包 (HTML解析+导出)") ap.add_argument("--full", action="store_true", help="安装全部依赖") args, _ = ap.parse_known_args() tier = "full" if args.full else ("recommended" if args.recommended else "core") tier_label = {"core": "核心(零依赖)", "recommended": "推荐", "full": "全功能"}[tier] print_header(f"cn-financial-scraper 环境安装向导 v5.0.0 - {tier_label}") print(f"\n 版本: 5.0.0") print(f" 模式: {tier_label}") print(f" 目录: {PROJECT_DIR}") results = {} results["Python 版本"] = check_python_version() if not results["Python 版本"]: print_fail("Python 版本不满足要求,继续验证(结果仅供参考)") results["依赖安装"] = install_dependencies(tier) results["核心模块"] = check_core_modules() check_optional() print_header("验证结果汇总") for name, ok in results.items(): icon = "✅" if ok else "❌" print(f" {icon} {name}: {'通过' if ok else '失败'}") print("\n" + "=" * 60) all_ok = all(isinstance(v, bool) and v for v in results.values()) if all_ok: print(" 🎉 所有检查通过!skill 已就绪。") print("\n 快速开始:") print(" python -c \"from scripts import search_institution; print(search_institution('华夏基金'))\"") print("\n MCP Server 模式:") print(" python mcp_server.py") else: print(" ⚠️ 部分检查失败,请检查上述错误。") print("=" * 60) return 0 if all_ok else 1 if __name__ == "__main__": sys.exit(main())