# -*- coding: utf-8 -*- """ 金融机构可爬取名单注册表 维护所有中国大陆金融机构的URL映射关系,支持增量更新 """ import json import re from pathlib import Path from typing import Dict, List, Optional SKILL_DIR = Path(__file__).parent.parent REGISTRY_FILE = SKILL_DIR / "data" / "institution_registry.json" PREDEFINED_URLS: Dict[str, str] = { # 国有大型商业银行 "中国工商银行": "https://www.icbc.com.cn", "中国建设银行": "https://www.ccb.com", "中国农业银行": "https://www.abchina.com", "中国银行": "https://www.boc.cn", "交通银行": "https://www.bankcomm.com", "中国邮政储蓄银行": "https://www.psbc.com", # 股份制商业银行 "招商银行": "https://www.cmbchina.com", "浦发银行": "https://www.spdb.com.cn", "中信银行": "https://www.citicbank.com", "中国光大银行": "https://www.cebbank.com", "华夏银行": "https://www.hxb.com.cn", "中国民生银行": "https://www.cmbc.com.cn", "广发银行": "https://www.cgbchina.com.cn", "兴业银行": "https://www.cib.com.cn", "平安银行": "https://bank.pingan.com", "浙商银行": "https://www.czbank.com", "恒丰银行": "https://www.hfbank.com.cn", "渤海银行": "https://www.cbhb.com.cn", # 政策性银行 "国家开发银行": "https://www.cdb.com.cn", # 中国进出口银行(官网为政府域名,已移除) "中国农业发展银行": "https://www.adbc.com.cn", # 证券公司 "中信证券": "https://www.cs.ecitic.com", "中信建投证券": "https://www.csc108.com", "国泰君安证券": "https://www.gtja.com", "华泰证券": "https://www.htsc.com.cn", "广发证券": "https://www.gf.com.cn", "招商证券": "https://www.newone.com.cn", "海通证券": "https://www.htsec.com", "国信证券": "https://www.guosen.com.cn", "东方证券": "https://www.dfzq.com.cn", "兴业证券": "https://www.xyzq.com.cn", "中国银河证券": "https://www.chinastock.com.cn", "长江证券": "https://www.95579.com", "中金公司": "https://www.cicc.com", "光大证券": "https://www.ebscn.com", "平安证券": "https://stock.pingan.com", "方正证券": "https://www.foundersc.com", "中泰证券": "https://www.zts.com.cn", "申万宏源证券": "https://www.swhysc.com", "国元证券": "https://www.gyzq.com.cn", "安信证券": "https://www.essence.com.cn", "开源证券": "https://www.kysec.cn", "华西证券": "https://www.hx168.com.cn", "民生证券": "https://www.mszq.com", "东北证券": "https://www.nesc.cn", "南京证券": "https://www.njzq.com", "长城证券": "https://www.cgws.com", "中银证券": "https://www.bocichina.com", "华安证券": "https://www.hazq.com", "财达证券": "https://www.s10000.com", "万联证券": "https://www.wlzq.com", "山西证券": "https://www.i618.com.cn", "西部证券": "https://www.westsecu.com", "浙商证券": "https://www.stocke.com.cn", "东海证券": "https://www.longone.com.cn", "国海证券": "https://www.ghzq.com.cn", "广州证券": "https://www.gzs.com.cn", "大同证券": "https://www.dtsbc.com.cn", "华融证券": "https://www.china-huarong.com", "中天证券": "https://www.izq.com.cn", # 基金管理公司 "易方达基金": "https://www.efunds.com.cn", "华夏基金": "https://www.chinaamc.com", "广发基金": "https://www.gffunds.com.cn", "嘉实基金": "https://www.jsfund.cn", "南方基金": "https://www.nffund.com.cn", "博时基金": "https://www.bosera.com", "招商基金": "https://www.cmfchina.com", "工银基金": "https://www.icbcfs.com", "建信基金": "https://www.ccbfund.cn", "富国基金": "https://www.fullgoal.com.cn", "鹏华基金": "https://www.phfund.com.cn", "汇添富基金": "https://www.htffund.com", "中欧基金": "https://www.zofund.com", "兴证全球基金": "https://www.xqglobal.com", "景林资产": "https://www.greenpeakgroup.com", "高毅资产": "https://www.gao-yi.com", "淡水泉投资": "https://www.downspring.com", "明汯投资": "https://www.minghonginvestment.com", "幻方量化": "https://www.quantizedcapital.com", "九坤投资": "https://www.jiukuninvest.com", "灵均投资": "https://www.lingjun-invest.com", "天演资本": "https://www.tianyan-capital.com", "东方港湾": "https://www.eastport.com.cn", "汉和资本": "https://www.hanhecap.com", "同犇投资": "https://www.tongben-invest.com", "石锋资产": "https://www.shifengam.com", "趣时资产": "https://www.quishi.com", "聚鸣投资": "https://www.jumingsh.com", "泰旸资产": "https://www.taiyangam.com", # 保险公司 "中国人寿": "https://www.chinalife.com.cn", "中国平安": "https://www.pingan.com.cn", "中国太平洋保险": "https://www.cpic.com.cn", "中国人保": "https://www.picc.com.cn", "新华保险": "https://www.newchina-life.com", "泰康保险": "https://www.taikang.com", "友邦保险": "https://www.aia.com.cn", "阳光保险": "https://www.sinosig.com", "合众人寿": "https://www.reformlife.com", "华夏保险": "https://www.huaxia-insurance.com", "富德生命人寿": "https://www.sino-life.com", "天安保险": "https://www.tiananinsurance.com", "中华保险": "https://www.cai-insurance.com", "大地保险": "https://www.95590.com", "中华联合保险": "https://www.cic.cn", "华泰保险": "https://www.ehuatai.com", "安华农业保险": "https://www.anohua.com", # 信托公司 "中信信托": "https://www.zxxt.com.cn", "平安信托": "https://www.paxt.com.cn", "中融信托": "https://www.zhongrongtrust.com", "华润信托": "https://www.crhtrust.com.cn", "外贸信托": "https://www.fotacn.com", "上海信托": "https://www.shanghaitrust.com", "中建投信托": "https://www.cctrust.com", "兴业信托": "https://www.ciit.com.cn", "华宝信托": "https://www.hwabaotrust.com", "交银信托": "https://www.bankcommtrust.com", "光大信托": "https://www.ebtrust.com", "中粮信托": "https://www.cofco-trust.com", "昆仑信托": "https://www.kunluntrust.com", "五矿信托": "https://www.minmetals-trust.com", "中铁信托": "https://www.crec-trust.com", "江苏信托": "https://www.js-trust.com", "粤财信托": "https://www.gdyc-trust.com", "重庆信托": "https://www.cqitic.com", "华融信托": "https://www.huarongtrust.com.cn", # 银行理财子公司 "工银理财": "https://www.icbc.com.cn/ICBC/FinancialManagement", "建信理财": "https://www.ccbfund.com.cn", "农银理财": "https://www.abchina.com/wealth", "中银理财": "https://www.boc.cn/financial", "交银理财": "https://www.bankcomm.com/wealth", "光大理财": "https://www.cebbank.com/finance", "招银理财": "https://www.cmbc.com.cn/wealth", "兴银理财": "https://www.cib.com.cn/wealth", "平安理财": "https://bank.pingan.com/wealth", "浦银理财": "https://www.spdb.com.cn/wealth", # 第三方销售机构 "天天基金": "https://fund.eastmoney.com", "蚂蚁基金": "https://www.fund123.cn", "腾安基金": "https://www.tan'anfund.com", "雪球基金": "https://danjuanfunds.com", "京东基金": "https://jd.jrj.com", "理财通": "https://www.lct.com", # 外资金融机构 "汇丰银行": "https://www.hsbc.com.cn", "渣打银行": "https://www.standardchartered.com.cn", "花旗银行": "https://www.citibank.com.cn", "摩根大通银行": "https://www.jpmorganchina.com.cn", "高盛集团": "https://www.goldmansachs.com.cn", "瑞银证券": "https://www.ubs.com/cn/zh.html", "野村证券": "https://www.nomura.com", "安盛保险": "https://www.axa.com.cn", "安联保险": "https://www.allianz.com.cn", "贝莱德资产管理": "https://www.blackrock.com.cn", "瑞银资产管理": "https://www.ubs.com/global/en/asset-management.html", "摩根士丹利": "https://www.morganstanley.com.cn", "美林证券": "https://www.ml.com", "德意志银行": "https://www.db.com.cn", "星展银行": "https://www.dbs.com.cn", "东亚银行": "https://www.hkbea.com.cn", "恒生银行": "https://www.hangseng.com", "华侨银行": "https://www.ocbc.com.cn", # ── 城市商业银行(补充) ── "北京银行": "https://www.bankofbeijing.com.cn", "上海银行": "https://www.bosc.cn", "江苏银行": "https://www.jsbchina.cn", "南京银行": "https://www.njcb.com.cn", "宁波银行": "https://www.nbcb.com.cn", "杭州银行": "https://www.hzbank.com.cn", "成都银行": "https://www.bocd.com.cn", "长沙银行": "https://www.bankofchangsha.com", "郑州银行": "https://www.zzbank.cn", "青岛银行": "https://www.qdccb.com", "西安银行": "https://www.xacbank.com", "贵阳银行": "https://www.bankgy.cn", "兰州银行": "https://www.lzbank.com", "厦门国际银行": "https://www.xib.com.cn", # ── 农商行 ── "重庆农商行": "https://www.cqrcb.com", "广州农商行": "https://www.grcbank.com", "上海农商行": "https://www.shrcb.com", "北京农商行": "https://www.bjrcb.com", "深圳农商行": "https://www.szrcb.com", # ── 金融科技 ── "蚂蚁集团": "https://www.antgroup.com", "京东科技": "https://www.jdcloud.com", "度小满金融": "https://www.duxiaoman.com", "陆金所": "https://www.lu.com", "众安保险": "https://www.zhongan.com", "微众银行": "https://www.webank.com", "网商银行": "https://www.mybank.cn", # ── 更多私募基金 ── "重阳投资": "https://www.chongyang.net", "凯丰投资": "https://www.kffund.com.cn", "敦和资管": "https://www.dunhefund.com", "乐瑞资产": "https://www.lowrisk.com.cn", "千合资本": "https://www.qhcapital.com.cn", "林园投资": "https://www.linyuaninvest.com", "星石投资": "https://www.starrockinvest.com", "望正资产": "https://www.wangzhengcapital.com", # ── 另类数据/评价平台 ── "好买基金": "https://www.howbuy.com", "私募排排网": "https://www.simuwang.com", "雪球": "https://xueqiu.com", "集思录": "https://www.jisilu.cn", "理杏仁": "https://www.lixinger.com", "韭圈儿": "https://www.funddb.cn", # ── 交易所 ── "上海证券交易所": "https://www.sse.com.cn", "深圳证券交易所": "https://www.szse.cn", "北京证券交易所": "https://www.bse.cn", "香港交易所": "https://www.hkex.com.hk", # ── 期货交易所 ── "上海期货交易所": "https://www.shfe.com.cn", "大连商品交易所": "https://www.dce.com.cn", "郑州商品交易所": "https://www.czce.com.cn", "中国金融期货交易所": "https://www.cffex.com.cn", "上海国际能源交易中心": "https://www.ine.cn", "广州期货交易所": "https://www.gfex.com.cn", # ── 行业协会 ── "中国银行业协会": "https://www.china-cba.net", "中国证券业协会": "https://www.sac.net.cn", "中国保险行业协会": "https://www.iachina.cn", # ── 同业市场 ── "中国外汇交易中心": "https://www.chinamoney.com.cn", "上海银行间同业拆放利率": "https://www.shibor.org", "中国债券信息网(中央国债登记结算公司)": "https://www.chinabond.com.cn", # ── 海外交易所 ── "纳斯达克": "https://www.nasdaq.com", "纽约证券交易所": "https://www.nyse.com", "芝加哥商品交易所": "https://www.cmegroup.com", "伦敦证券交易所": "https://www.londonstockexchange.com", "东京证券交易所": "https://www.jpx.co.jp", "新加坡交易所": "https://www.sgx.com", # ── 其他金融机构补充 ── "中国再保险": "https://www.chinare.com.cn", "中投公司": "https://www.china-inv.cn", "国家开发投资集团": "https://www.sdic.com.cn", "中国银河资产管理": "https://www.china-galaxyam.com", } class ScrapableRegistry: def __init__(self, registry_file: str = None): self.registry_file = Path(registry_file) if registry_file else REGISTRY_FILE self._registry = None self._load() def _load(self): if self.registry_file.exists(): with open(self.registry_file, encoding="utf-8") as f: self._registry = json.load(f) # 解码列式格式 if self._registry.get("_f") == "c": self._registry = self._decode_columnar(self._registry) else: self._registry = {} @staticmethod def _decode_columnar(data: dict) -> dict: """将列式格式还原为标准 dict""" columns = data.get("c", []) rows = data.get("d", []) meta = data.get("m", data.get("_meta", {})) items = [] for row in rows: item = {} for i, col_name in enumerate(columns): if i < len(row): item[col_name] = row[i] items.append(item) result = {"institutions": items} if meta: result["_meta"] = meta return result @property def institutions(self) -> List[Dict]: return self._registry.get("institutions", []) @property def total(self) -> int: return self._registry.get("_meta", {}).get("total", len(self.institutions)) def is_scrapable(self, name: str, inst_type: str = "") -> bool: if name in PREDEFINED_URLS: return True return False def get(self, name: str, inst_type: str = "") -> Optional[Dict]: for inst in self.institutions: if inst["name"] == name: result = dict(inst) website = inst.get("website", "") if not website and name in PREDEFINED_URLS: website = PREDEFINED_URLS[name] result["website"] = website result["scrapable"] = bool(website) return result return None def search(self, keyword: str) -> List[Dict]: results = [] kw = keyword.lower() for inst in self.institutions: if kw in inst["name"].lower() or kw in inst.get("code", "").lower(): results.append(self.get(inst["name"])) return results def list_scrapable(self, inst_type: str = "") -> List[Dict]: results = [] for inst in self.institutions: if inst_type and inst["type"] != inst_type: continue info = self.get(inst["name"]) if info and info["scrapable"]: results.append(info) return results def list_by_type(self, inst_type: str) -> List[Dict]: return [self.get(i["name"]) for i in self.institutions if i["type"] == inst_type] def add_url(self, name: str, url: str) -> bool: for inst in self.institutions: if inst["name"] == name: inst["website"] = url self._registry["_meta"]["with_website"] = sum( 1 for i in self.institutions if i.get("website") ) self.save() return True return False def get_statistics(self) -> Dict: summary = self._registry.get("_type_summary", {}) stats = { "total": self.total, "scrapable": sum( 1 for i in self.institutions if self.is_scrapable(i["name"], i["type"]) ), "by_type": {}, } for inst_type, info in summary.items(): scrapable_count = sum( 1 for i in self.institutions if i["type"] == inst_type and self.is_scrapable(i["name"], inst_type) ) stats["by_type"][inst_type] = { "total": info["total"], "with_url": info["with_website"], "scrapable": scrapable_count, } return stats def save(self): with open(self.registry_file, "w", encoding="utf-8") as f: json.dump(self._registry, f, ensure_ascii=False, indent=2) def generate_report(self) -> str: stats = self.get_statistics() lines = [ "=" * 60, "金融机构可爬取名单报告", "=" * 60, f"机构总数:{stats['total']}", f"可爬取数:{stats['scrapable']} ({stats['scrapable']*100//stats['total']}%)", "", "按类型明细:", ] for inst_type, info in sorted(stats["by_type"].items(), key=lambda x: -x[1]["total"]): lines.append( f" {inst_type}: {info['scrapable']}/{info['total']} 可爬取 " f"({info['with_url']} 有URL)" ) return "\n".join(lines) # ── v7.0.0: API 发现引擎桥接 ──────────────────────────────── def suggest_api(self, name_or_domain: str, limit: int = 10) -> List[Dict]: """查询 v7.0.0 网络拦截挖掘出的 API 注册表(data/api_registry.json)。 机构名会先解析出官网域名,再返回该域名下已发现的 JSON 接口, 供降级链直接 HTTP 直连(跳过浏览器渲染)。 Args: name_or_domain: 机构名(如 东方财富)或域名(如 eastmoney.com) limit: 最多返回条数 """ api_file = Path(self.registry_file).parent / "api_registry.json" if not api_file.exists(): return [] try: doc = json.loads(api_file.read_text(encoding="utf-8")) except Exception: return [] domain = name_or_domain if "." not in domain: inst = self.get(name_or_domain) if not inst or not inst.get("website"): return [] domain = inst["website"].split("//")[-1].split("/")[0] out = [ep for ep in doc.get("endpoints", []) if domain and domain in ep.get("url", "")] out.sort(key=lambda e: -e.get("hit_count", 1)) return out[:limit] def main(): import sys registry = ScrapableRegistry() args = sys.argv[1:] if not args: print(registry.generate_report()) return cmd = args[0] if cmd == "stat": print(registry.generate_report()) elif cmd == "list" and len(args) > 1: for info in registry.list_scrapable(args[1]): print(f" {info['name']} | {info['type']} | {info.get('website', 'N/A')}") elif cmd == "search" and len(args) > 1: for info in registry.search(args[1]): scrapable = "OK" if info["scrapable"] else "NO" print(f" [{scrapable}] {info['name']} | {info['type']} | {info.get('website', '')}") elif cmd == "add" and len(args) > 2: name, url = args[1], args[2] if registry.add_url(name, url): print(f"URL已更新: {name} -> {url}") else: print(f"机构未找到: {name}") elif cmd == "export": for inst in registry.institutions: info = registry.get(inst["name"]) if info["scrapable"]: print(f"{info['name']}\t{info['type']}\t{info.get('website', '')}") else: print("用法:") print(" python scrapable_registry.py stat") print(" python scrapable_registry.py list <类型>") print(" python scrapable_registry.py search <关键词>") print(" python scrapable_registry.py add <名称> ") print(" python scrapable_registry.py export") if __name__ == "__main__": main()