Files
cnfinancialscraper/scripts/report_indexer.py
T

775 lines
28 KiB
Python

# -*- coding: utf-8 -*-
"""
全量扫描索引器
扫描所有A股上市公司,建立报告/公告索引
支持增量更新和断点续扫
"""
import json
import time
import sqlite3
from pathlib import Path
from typing import Dict, Any, List, Optional, Callable
from datetime import datetime, timedelta
from dataclasses import dataclass, field
from contextlib import contextmanager
SKILL_DATA_DIR = Path(__file__).parent.parent / "data"
INDEX_DB_PATH = SKILL_DATA_DIR / "report_index.db"
SCAN_STATE_FILE = SKILL_DATA_DIR / "scan_state.json"
@dataclass
class ReportIndex:
"""索引记录"""
stock_code: str
stock_name: str
report_type: str # periodic/broker/announcement
report_title: str
publish_date: str
url: str
is_available: bool = True
indexed_at: str = ""
broker_name: str = ""
rating: str = ""
def to_dict(self) -> Dict:
return {
"stock_code": self.stock_code,
"stock_name": self.stock_name,
"report_type": self.report_type,
"report_title": self.report_title,
"publish_date": self.publish_date,
"url": self.url,
"is_available": self.is_available,
"indexed_at": self.indexed_at,
"broker_name": self.broker_name,
"rating": self.rating
}
@dataclass
class ScanProgress:
"""扫描进度"""
total_stocks: int = 0
scanned: int = 0
failed: int = 0
current_stock: str = ""
current_index: int = 0
start_time: str = ""
last_checkpoint: str = ""
status: str = "idle" # idle/running/completed/error
class StockIndexDatabase:
"""股票索引数据库 (SQLite)"""
def __init__(self, db_path: str = None):
self.db_path = Path(db_path) if db_path else INDEX_DB_PATH
self.db_path.parent.mkdir(parents=True, exist_ok=True)
self._init_db()
@contextmanager
def get_connection(self):
"""获取数据库连接"""
conn = sqlite3.connect(str(self.db_path))
conn.row_factory = sqlite3.Row
try:
yield conn
finally:
conn.close()
def _init_db(self):
"""初始化数据库表"""
with self.get_connection() as conn:
cursor = conn.cursor()
# 股票列表表
cursor.execute("""
CREATE TABLE IF NOT EXISTS stocks (
stock_code TEXT PRIMARY KEY,
stock_name TEXT NOT NULL,
market TEXT DEFAULT "",
listed_date TEXT DEFAULT "",
last_indexed TEXT DEFAULT ""
)
""")
# 报告索引表
cursor.execute("""
CREATE TABLE IF NOT EXISTS report_index (
id INTEGER PRIMARY KEY AUTOINCREMENT,
stock_code TEXT NOT NULL,
stock_name TEXT DEFAULT "",
report_type TEXT NOT NULL,
report_title TEXT NOT NULL,
publish_date TEXT DEFAULT "",
url TEXT DEFAULT "",
is_available INTEGER DEFAULT 1,
indexed_at TEXT DEFAULT "",
broker_name TEXT DEFAULT "",
rating TEXT DEFAULT "",
UNIQUE(stock_code, report_type, report_title, publish_date)
)
""")
# 扫描状态表
cursor.execute("""
CREATE TABLE IF NOT EXISTS scan_state (
id INTEGER PRIMARY KEY,
last_stock_index INTEGER DEFAULT 0,
last_scan_time TEXT DEFAULT "",
status TEXT DEFAULT 'idle',
total_stocks INTEGER DEFAULT 0
)
""")
# 创建索引
cursor.execute("CREATE INDEX IF NOT EXISTS idx_report_stock ON report_index(stock_code)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_report_type ON report_index(report_type)")
cursor.execute("CREATE INDEX IF NOT EXISTS idx_report_date ON report_index(publish_date)")
conn.commit()
def add_stock(self, stock_code: str, stock_name: str, market: str = ""):
"""添加股票"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
INSERT OR REPLACE INTO stocks (stock_code, stock_name, market, last_indexed)
VALUES (?, ?, ?, ?)
""", (stock_code, stock_name, market, ""))
conn.commit()
def add_stock_batch(self, stocks: List[Dict]):
"""批量添加股票"""
with self.get_connection() as conn:
cursor = conn.cursor()
data = [(s['code'], s['name'], s.get('market', '')) for s in stocks]
cursor.executemany("""
INSERT OR REPLACE INTO stocks (stock_code, stock_name, market)
VALUES (?, ?, ?)
""", data)
conn.commit()
def index_report(self, report: ReportIndex):
"""索引单条报告"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
INSERT OR IGNORE INTO report_index
(stock_code, stock_name, report_type, report_title, publish_date,
url, is_available, indexed_at, broker_name, rating)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
report.stock_code, report.stock_name, report.report_type,
report.report_title, report.publish_date, report.url,
1 if report.is_available else 0, report.indexed_at,
report.broker_name, report.rating
))
conn.commit()
def index_reports_batch(self, reports: List[ReportIndex]):
"""批量索引报告"""
with self.get_connection() as conn:
cursor = conn.cursor()
data = [
(r.stock_code, r.stock_name, r.report_type, r.report_title,
r.publish_date, r.url, 1 if r.is_available else 0,
r.indexed_at, r.broker_name, r.rating)
for r in reports
]
cursor.executemany("""
INSERT OR IGNORE INTO report_index
(stock_code, stock_name, report_type, report_title, publish_date,
url, is_available, indexed_at, broker_name, rating)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", data)
conn.commit()
def get_reports_by_stock(self, stock_code: str,
report_type: str = None) -> List[ReportIndex]:
"""获取个股报告索引"""
with self.get_connection() as conn:
cursor = conn.cursor()
if report_type:
cursor.execute("""
SELECT * FROM report_index
WHERE stock_code = ? AND report_type = ?
ORDER BY publish_date DESC
""", (stock_code, report_type))
else:
cursor.execute("""
SELECT * FROM report_index
WHERE stock_code = ?
ORDER BY publish_date DESC
""", (stock_code,))
rows = cursor.fetchall()
return [self._row_to_index(rows[0])] if rows else []
def _row_to_index(self, row) -> ReportIndex:
return ReportIndex(
stock_code=row['stock_code'],
stock_name=row['stock_name'],
report_type=row['report_type'],
report_title=row['report_title'],
publish_date=row['publish_date'],
url=row['url'],
is_available=bool(row['is_available']),
indexed_at=row['indexed_at'],
broker_name=row.get('broker_name', ''),
rating=row.get('rating', '')
)
def search_reports(self, keyword: str = "",
report_type: str = None,
date_from: str = None,
date_to: str = None,
stock_codes: List[str] = None,
limit: int = 100) -> List[ReportIndex]:
"""全文搜索索引"""
with self.get_connection() as conn:
cursor = conn.cursor()
conditions = []
params = []
if keyword:
conditions.append("(report_title LIKE ? OR stock_name LIKE ?)")
params.extend([f"%{keyword}%", f"%{keyword}%"])
if report_type:
conditions.append("report_type = ?")
params.append(report_type)
if date_from:
conditions.append("publish_date >= ?")
params.append(date_from)
if date_to:
conditions.append("publish_date <= ?")
params.append(date_to)
if stock_codes:
placeholders = ','.join(['?' for _ in stock_codes])
conditions.append(f"stock_code IN ({placeholders})")
params.extend(stock_codes)
where_clause = " AND ".join(conditions) if conditions else "1=1"
query = f"""
SELECT * FROM report_index
WHERE {where_clause}
ORDER BY publish_date DESC
LIMIT ?
"""
params.append(limit)
cursor.execute(query, params)
rows = cursor.fetchall()
return [self._row_to_index(r) for r in rows]
def get_stock_count(self) -> int:
"""获取股票总数"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("SELECT COUNT(*) FROM stocks")
return cursor.fetchone()[0]
def get_report_count(self, report_type: str = None) -> int:
"""获取报告总数"""
with self.get_connection() as conn:
cursor = conn.cursor()
if report_type:
cursor.execute("SELECT COUNT(*) FROM report_index WHERE report_type = ?", (report_type,))
else:
cursor.execute("SELECT COUNT(*) FROM report_index")
return cursor.fetchone()[0]
def get_scan_state(self) -> Optional[ScanProgress]:
"""获取扫描状态"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("SELECT * FROM scan_state WHERE id = 1")
row = cursor.fetchone()
if row:
return ScanProgress(
total_stocks=row['total_stocks'],
scanned=row['last_stock_index'],
last_checkpoint=row['last_scan_time'],
status=row['status']
)
return None
def save_scan_state(self, progress: ScanProgress):
"""保存扫描状态"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
INSERT OR REPLACE INTO scan_state (id, last_stock_index, last_scan_time, status, total_stocks)
VALUES (1, ?, ?, ?, ?)
""", (progress.scanned, progress.last_checkpoint, progress.status, progress.total_stocks))
conn.commit()
def update_stock_last_indexed(self, stock_code: str):
"""更新股票最后索引时间"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
UPDATE stocks SET last_indexed = ? WHERE stock_code = ?
""", (datetime.now().isoformat(), stock_code))
conn.commit()
def get_stats(self) -> Dict[str, Any]:
"""获取统计信息"""
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("SELECT COUNT(*) FROM stocks")
stock_count = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM report_index")
total_reports = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM report_index WHERE report_type = 'periodic'")
periodic_count = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM report_index WHERE report_type = 'broker'")
broker_count = cursor.fetchone()[0]
cursor.execute("SELECT COUNT(*) FROM report_index WHERE report_type = 'announcement'")
announcement_count = cursor.fetchone()[0]
return {
"total_stocks": stock_count,
"total_reports": total_reports,
"periodic_reports": periodic_count,
"broker_reports": broker_count,
"announcements": announcement_count
}
class StockIndexer:
"""
全量扫描索引器
扫描模式:
1. 初始化: 从EastMoney获取全量股票列表 (~5000只)
2. 全量扫描: 遍历每只股票,索引其所有报告/公告
3. 增量扫描: 仅扫描新发布或变化的报告
4. 断点续扫: 记录扫描进度,异常中断后可恢复
"""
def __init__(self, db_path: str = None):
self.db = StockIndexDatabase(db_path)
self.session = None
try:
import requests
self.session = requests.Session()
self.session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
})
except ImportError:
pass
# 延迟导入API
self._periodic_api = None
self._broker_api = None
self._announcement_api = None
# Rate limiting
self.request_delay = 0.5
self.max_retries = 3
# Progress tracking
self.progress = ScanProgress()
@property
def periodic_api(self):
if self._periodic_api is None:
from .company_report_scraper import EastMoneyReportAPI
self._periodic_api = EastMoneyReportAPI()
return self._periodic_api
@property
def broker_api(self):
if self._broker_api is None:
from .research_report_scraper import EastMoneyBrokerReportAPI
self._broker_api = EastMoneyBrokerReportAPI()
return self._broker_api
@property
def announcement_api(self):
if self._announcement_api is None:
from .announcement_scraper import AnnouncementSearcher
self._announcement_api = AnnouncementSearcher()
return self._announcement_api
def init_stock_list(self, market: str = "all") -> int:
"""
初始化股票列表
从EastMoney获取全量A股列表并写入数据库
"""
if not self.session:
print("[错误] requests库未安装")
return 0
try:
url = "https://80.push2.eastmoney.com/api/qt/clist/get"
params = {
"pn": 1,
"pz": 5000,
"po": 1,
"np": 1,
"ut": "bd1d9ddb04089700cf9c27f6f7426281",
"fltt": 2,
"invt": 2,
"fid": "f3",
"fs": "m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23,m:0+t:81+s:2048",
"fields": "f12,f14,f3"
}
resp = self.session.get(url, params=params, timeout=30)
data = resp.json()
stocks = []
if data.get('data') and data['data'].get('diff'):
for item in data['data']['diff']:
code = item.get('f12', '')
name = item.get('f14', '')
if code:
market_type = "SH" if code.startswith('6') or code.startswith('9') else "SZ"
stocks.append({
"code": code,
"name": name,
"market": market_type
})
self.db.add_stock_batch(stocks)
print(f"[完成] 已加载 {len(stocks)} 只股票到索引库")
return len(stocks)
except Exception as e:
print(f"[错误] 初始化股票列表失败: {e}")
return 0
def full_scan(self, report_types: List[str] = None,
progress_callback: Callable = None,
checkpoint_interval: int = 100,
limit: int = 0) -> Dict[str, Any]:
"""
全量扫描
Args:
report_types: 扫描类型 ["periodic", "broker", "announcement"]
progress_callback: 进度回调 (scanned, total, current_stock)
checkpoint_interval: 每N只股票保存一次进度
limit: 限制扫描数量(0=全部)
Returns:
{"scanned": N, "failed": M, "duration": seconds}
"""
if report_types is None:
report_types = ["periodic", "broker", "announcement"]
start_time = datetime.now()
# 获取股票列表
with self.db.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("SELECT stock_code, stock_name FROM stocks ORDER BY stock_code")
stocks = cursor.fetchall()
if limit > 0:
stocks = stocks[:limit]
total = len(stocks)
self.progress = ScanProgress(
total_stocks=total,
start_time=start_time.isoformat(),
status="running"
)
print(f"[开始] 全量扫描 {total} 只股票...")
scanned = 0
failed = 0
for i, stock in enumerate(stocks):
code = stock['stock_code']
name = stock['stock_name']
self.progress.current_stock = code
self.progress.current_index = i
try:
self._scan_single_stock(code, name, report_types)
scanned += 1
self.db.update_stock_last_indexed(code)
except Exception as e:
failed += 1
print(f"[警告] 扫描 {code} 失败: {e}")
# 进度回调
if progress_callback:
progress_callback(i + 1, total, code)
# 保存检查点
if (i + 1) % checkpoint_interval == 0:
self.progress.scanned = scanned
self.progress.last_checkpoint = datetime.now().isoformat()
self.db.save_scan_state(self.progress)
print(f"[进度] 已扫描 {scanned}/{total}, 失败 {failed}")
# 避免请求过快
time.sleep(self.request_delay)
self.progress.scanned = scanned
self.progress.status = "completed"
self.progress.last_checkpoint = datetime.now().isoformat()
self.db.save_scan_state(self.progress)
duration = (datetime.now() - start_time).total_seconds()
print(f"[完成] 扫描完成: 成功 {scanned}, 失败 {failed}, 耗时 {duration:.1f}秒")
return {
"scanned": scanned,
"failed": failed,
"duration": duration
}
def _scan_single_stock(self, stock_code: str, stock_name: str,
report_types: List[str]) -> List[ReportIndex]:
"""扫描单只股票的所有报告"""
reports = []
now = datetime.now().isoformat()
# 定期报告
if "periodic" in report_types:
try:
periodic_reports = self.periodic_api.search_reports(
stock_code, max_results=50
)
for p in periodic_reports:
from .company_report_scraper import CompanyReport
if isinstance(p, CompanyReport):
idx = ReportIndex(
stock_code=stock_code,
stock_name=stock_name,
report_type="periodic",
report_title=getattr(p, 'title', ''),
publish_date=getattr(p, 'publish_date', ''),
url=getattr(p, 'url', ''),
indexed_at=now
)
reports.append(idx)
except Exception as e:
print(f"[警告] 获取 {stock_code} 定期报告失败: {e}")
# 券商研报
if "broker" in report_types:
try:
broker_reports = self.broker_api.get_stock_reports(
stock_code, max_results=50
)
for b in broker_reports:
from .research_report_scraper import BrokerReport
if isinstance(b, BrokerReport):
idx = ReportIndex(
stock_code=stock_code,
stock_name=stock_name,
report_type="broker",
report_title=getattr(b, 'title', ''),
publish_date=getattr(b, 'publish_date', ''),
url=getattr(b, 'url', ''),
indexed_at=now,
broker_name=getattr(b, 'broker_name', ''),
rating=getattr(b, 'rating', '')
)
reports.append(idx)
except Exception as e:
print(f"[警告] 获取 {stock_code} 券商研报失败: {e}")
# 公告
if "announcement" in report_types:
try:
announcements = self.announcement_api.search_announcements(
"", fund_code=stock_code, max_results=50
)
for a in announcements:
from .announcement_scraper import Announcement
if isinstance(a, Announcement):
idx = ReportIndex(
stock_code=stock_code,
stock_name=stock_name,
report_type="announcement",
report_title=getattr(a, 'title', ''),
publish_date=getattr(a, 'publish_date', ''),
url=getattr(a, 'url', ''),
indexed_at=now
)
reports.append(idx)
except Exception as e:
print(f"[警告] 获取 {stock_code} 公告失败: {e}")
# 批量写入索引
if reports:
self.db.index_reports_batch(reports)
return reports
def incremental_scan(self, days: int = 7) -> Dict[str, Any]:
"""
增量扫描 - 仅扫描最近N天内发布的报告
"""
from datetime import timedelta
end_date = datetime.now()
start_date = end_date - timedelta(days=days)
start_date_str = start_date.strftime('%Y-%m-%d')
end_date_str = end_date.strftime('%Y-%m-%d')
print(f"[增量扫描] {start_date_str} 至 {end_date_str}")
# 获取最近N天有更新的股票
# 直接对所有股票扫描(简化版)
return self.full_scan(limit=500) # 限制增量扫描数量
def resume_scan(self) -> Dict[str, Any]:
"""
断点续扫 - 从上次中断处继续
"""
state = self.db.get_scan_state()
if not state or state.status != "running":
print("[信息] 没有可恢复的扫描任务")
return {"scanned": 0, "message": "No task to resume"}
print(f"[恢复] 从进度 {state.scanned}/{state.total_stocks} 继续扫描...")
# 获取剩余股票
with self.db.get_connection() as conn:
cursor = conn.cursor()
cursor.execute("""
SELECT stock_code, stock_name FROM stocks
ORDER BY stock_code
LIMIT ? OFFSET ?
""", (state.total_stocks - state.scanned, state.scanned))
stocks = cursor.fetchall()
total = len(stocks)
scanned = 0
failed = 0
for i, stock in enumerate(stocks):
code = stock['stock_code']
name = stock['stock_name']
try:
self._scan_single_stock(code, name, ["periodic", "broker", "announcement"])
scanned += 1
except:
failed += 1
if (i + 1) % 100 == 0:
print(f"[进度] 已恢复 {scanned}/{total}")
time.sleep(self.request_delay)
# 更新状态
state.scanned += scanned
state.status = "completed"
state.last_checkpoint = datetime.now().isoformat()
self.db.save_scan_state(state)
return {"scanned": scanned, "failed": failed}
def search_reports(self, keyword: str = "",
report_type: str = None,
date_from: str = None,
date_to: str = None,
limit: int = 100) -> List[ReportIndex]:
"""搜索索引"""
return self.db.search_reports(keyword, report_type, date_from, date_to, limit=limit)
def get_stats(self) -> Dict[str, Any]:
"""获取扫描统计"""
return self.db.get_stats()
# ============ CLI入口 ============
if __name__ == "__main__":
import sys
if len(sys.argv) < 2:
print("用法:")
print(" python report_indexer.py init # 初始化股票列表")
print(" python report_indexer.py scan --full # 全量扫描")
print(" python report_indexer.py scan --incr # 增量扫描")
print(" python report_indexer.py scan --resume # 断点续扫")
print(" python report_indexer.py search <关键词> # 搜索")
print(" python report_indexer.py stats # 查看统计")
sys.exit(1)
cmd = sys.argv[1]
indexer = StockIndexer()
if cmd == "init":
count = indexer.init_stock_list()
print(f"初始化完成: {count} 只股票")
elif cmd == "scan":
if len(sys.argv) < 3:
print("请指定扫描模式: --full, --incr, --resume")
sys.exit(1)
mode = sys.argv[2]
if mode == "--full":
limit = 0
for i, arg in enumerate(sys.argv):
if arg == "--limit" and i + 1 < len(sys.argv):
limit = int(sys.argv[i + 1])
result = indexer.full_scan(limit=limit)
print(f"全量扫描完成: 成功 {result['scanned']}, 失败 {result['failed']}, 耗时 {result['duration']:.1f}秒")
elif mode == "--incr":
days = 7
for i, arg in enumerate(sys.argv):
if arg == "--days" and i + 1 < len(sys.argv):
days = int(sys.argv[i + 1])
result = indexer.incremental_scan(days=days)
print(f"增量扫描完成")
elif mode == "--resume":
result = indexer.resume_scan()
print(f"断点续扫完成: {result}")
elif cmd == "search":
keyword = sys.argv[2] if len(sys.argv) > 2 else ""
report_type = None
for i, arg in enumerate(sys.argv):
if arg == "--type" and i + 1 < len(sys.argv):
report_type = sys.argv[i + 1]
results = indexer.search_reports(keyword=keyword, report_type=report_type)
print(f"找到 {len(results)} 条索引:")
for r in results[:20]:
print(f" [{r.report_type}] {r.stock_code} {r.stock_name} | {r.publish_date[:10]} | {r.report_title[:40]}")
elif cmd == "stats":
stats = indexer.get_stats()
print(f"\n索引统计:")
print(f" 股票总数: {stats['total_stocks']}")
print(f" 报告总数: {stats['total_reports']}")
print(f" - 定期报告: {stats['periodic_reports']}")
print(f" - 券商研报: {stats['broker_reports']}")
print(f" - 公告: {stats['announcements']}")
else:
print(f"未知命令: {cmd}")