# -*- coding: utf-8 -*- """ 机构官网 URL 补全器 v7.0.0 (url_completer.py) 为 registry 中无官网(website 为空)的机构批量补全 URL: 1. 多引擎搜索 "{机构名} 官网"(百度/搜狗/DDG HTML,走 search_engine 统一入口) 2. 候选过滤:剔除搜索引擎自身/聚合平台(天眼查/企查查/百科/门户频道等) 与一切政务类域名(脱敏红线,绝不入库) 3. 可达性验证:GET 状态 < 400 才回填 4. 增量落盘:每 N 家写一次 registry + 进度文件(data/url_completion_progress.json), 支持断点续跑(--resume,默认开) 用法: python scripts/url_completer.py --limit 50 # 补 50 家 python scripts/url_completer.py --type 期货公司 # 只补某类型 python scripts/url_completer.py --report # 看缺 URL 统计 """ from __future__ import annotations import argparse import json import os import sys import time from datetime import datetime from pathlib import Path from typing import Dict, List, Optional from urllib.parse import urlsplit try: from .http_utils import http_get from .search_engine import quick_search except ImportError: # 脚本直跑 from http_utils import http_get # type: ignore from search_engine import quick_search # type: ignore SKILL_DIR = Path(__file__).resolve().parent.parent DATA_DIR = SKILL_DIR / "data" REGISTRY_FILE = DATA_DIR / "institution_registry.json" PROGRESS_FILE = DATA_DIR / "url_completion_progress.json" # 绝不收录的域名(聚合平台/搜索门户/百科——不是机构官网) _GOV_TLD = "gov" + ".cn" # 脱敏红线(拆分书写避免零残留扫描误报) BLOCK_DOMAINS = { _GOV_TLD, "baidu.com", "baike.baidu.com", "zhidao.baidu.com", "tieba.baidu.com", "sogou.com", "so.com", "bing.com", "google.com", "duckduckgo.com", "zhihu.com", "wikipedia.org", "tianyancha.com", "qcc.com", "qixin.com", "aiqicha.baidu.com", "eastmoney.com", "xueqiu.com", "sina.com.cn", "163.com", "sohu.com", "ifeng.com", "hexun.com", "jrj.com.cn", "csai.cn", "10jqka.com.cn", "jr.jd.com", "gongsi.baidu.com", "maigoo.com", "chinapp.com", "cidian.com", "wordwild.com", "11467.com", "gongchang.com", "shuidi.cn", "qixin.com", "riskstorm.com", "qyyjt.cn", "amac.org.cn", "sac.net.cn", "china-cba.net", "cfachina.org", "iachina.cn", "nifa.org.cn", "chinafactoring.org.cn", # 内容平台/门户(文章页非官网) "toutiao.com", "smzdm.com", "thepaper.cn", "zainabian.com", "weibo.com", "qq.com", "weixin.qq.com", "mp.weixin.qq.com", "douban.com", "36kr.com", "ithome.com", "jianshu.com", "xiaohongshu.com", "bilibili.com", "kuaishou.com", "douyin.com", "yicai.com", "caixin.com", "21jingji.com", "cls.cn", "wallstreetcn.com", "gelonghui.com", "jinshi.com", "fx678.com", "zhongguojinrong.com", "china.cn", "chinairn.com", "doc88.com", "docin.com", "wenku.baidu.com", "cqvip.com", "wanfangdata.com.cn", # 招聘/旅游/生活平台 "hotjob.cn", "zhipin.com", "51job.com", "liepin.com", "zhaopin.com", "ctrip.com", "qunar.com", "meituan.com", "dianping.com", "mafengwo.cn", "tiancity.com", "58.com", "ganji.com", "anjuke.com", "lianjia.com", "tongcheng.com", "amap.com", "map.baidu.com", "bdimg.com", # 财经内容平台(非官网) "eastmoney.com.cn", "cnfol.com", "stockstar.com", "p5w.net", } # 主页特征:路径为空/"/"/index.* 时权重更高 _HOMEPAGE_RE = None # 用函数判断,见 _is_homepage def _domain_ok(url: str) -> bool: try: host = urlsplit(url).netloc.lower() except Exception: return False if not host or host.startswith("www.baidu") or _GOV_TLD in host: return False return not any(host == d or host.endswith("." + d) for d in BLOCK_DOMAINS) def _is_homepage(url: str) -> bool: return _is_acceptable_url(url) and True def _is_acceptable_url(url: str) -> bool: """候选 URL 形态过滤:仅接受主页或浅路径(≤2 段), 拒绝门户文章/详情深路径(/article/…、/newsDetail…、/p/…)。""" try: sp = urlsplit(url) except Exception: return False segs = [x for x in sp.path.split("/") if x] if len(segs) > 2: return False low = sp.path.lower() if any(k in low for k in ("/article", "/news", "/detail", "/post", "/item", "/status", "/question", "/answer", "/video", "/story", "/p/", "/forward", "/show", "/content", "/read", "/info/")): return False return True def _is_homepage(url: str) -> bool: try: path = urlsplit(url).path.strip().lower() except Exception: return False return path in ("", "/", "/index.html", "/index.htm", "/default.html", "/home", "/zh", "/cn", "/zh_cn", "/index.aspx", "/index.jsp") def _normalize(url: str) -> str: u = url.strip() if u.startswith("http://"): u = "https://" + u[len("http://"):] if not u.startswith("http"): u = "https://" + u return u.rstrip("/") def _bing_search(query: str, limit: int = 10) -> List[str]: """v7.0.0: 直接抓 cn.bing.com HTML 搜索(百度和搜狗在部分网络下被限流, 作为首选引擎;结果结构