为 JSON。"""
tables: List[Dict] = []
for i, m in enumerate(_TABLE_RE.finditer(html)):
table_html = m.group(0)
try:
table_data = self._parse_html_table(table_html)
if not table_data:
continue
table_info = {
"page_num": page_num,
"table_index": i,
"rows": len(table_data),
"cols": max(len(row) for row in table_data) if table_data else 0,
"data": table_data,
}
# 保存为 JSON
json_path = os.path.join(
tables_dir,
f"table_p{page_num:02d}_{i:02d}.json",
)
Path(json_path).write_text(
json.dumps(table_info, ensure_ascii=False, indent=2),
encoding="utf-8",
)
tables.append(table_info)
except Exception:
continue
return tables
def _parse_html_table(self, table_html: str) -> List[List[str]]:
"""简易 HTML 表格解析(不依赖 BeautifulSoup)。"""
# 提取所有行
rows: List[List[str]] = []
tr_pattern = re.compile(r'][\s\S]*?
', re.IGNORECASE)
td_pattern = re.compile(r'<(?:td|th)[^>]*>([\s\S]*?)(?:td|th)>', re.IGNORECASE)
tag_strip = re.compile(r'<[^>]+>')
for tr_m in tr_pattern.finditer(table_html):
row_html = tr_m.group(0)
cells = []
for td_m in td_pattern.finditer(row_html):
cell_text = tag_strip.sub('', td_m.group(1)).strip()
cell_text = re.sub(r'\s+', ' ', cell_text)
cells.append(cell_text)
if cells:
rows.append(cells)
return rows
# ── 内部: 输出生成 ────────────────────────────────────────────────
def _write_inline_html(self, pages_data: List[Dict],
image_map: Dict[str, str],
canvas_paths: List[str],
title: str, output_path: str):
"""生成 base64 内嵌图片的单文件 HTML。"""
# 合并所有页面
parts = [
'',
'',
'',
f'',
f'{self._escape_html(title)}',
'',
'',
'',
f'{self._escape_html(title)}
',
f'归档时间: {datetime.now().strftime("%Y-%m-%d %H:%M")} '
f'| 页数: {len(pages_data)}
',
]
for i, pdata in enumerate(pages_data):
if i > 0:
parts.append(
f'📄 第 {pdata["page_num"]} 页 '
f'— {self._escape_html(pdata.get("url", "")[:80])}
'
)
html = pdata["html"]
# 替换图片 src 为 base64
html = self._inline_images_in_html(html, image_map, pdata["url"])
# 注入到 body
body_content = self._extract_body_content(html)
parts.append(body_content)
# 附加 Canvas 截图
if canvas_paths:
parts.append('
📊 图表 (Canvas 截图)
')
for cp in canvas_paths:
b64 = self._file_to_base64(cp)
if b64:
ext = os.path.splitext(cp)[1].lower()
mime = "image/png" if ext == ".png" else "image/jpeg"
parts.append(
f'
{os.path.basename(cp)}'
f''
)
parts.extend(['', ''])
Path(output_path).write_text("\n".join(parts), encoding="utf-8")
def _write_assets_html(self, pages_data: List[Dict],
image_map: Dict[str, str],
canvas_paths: List[str],
title: str, output_path: str):
"""生成引用本地资源的目录版 index.html。"""
parts = [
'',
'',
'',
f'',
f'{self._escape_html(title)}',
'',
'',
'',
f'{self._escape_html(title)}
',
f'归档时间: {datetime.now().strftime("%Y-%m-%d %H:%M")} '
f'| 页数: {len(pages_data)}
',
]
for i, pdata in enumerate(pages_data):
if i > 0:
parts.append(
f'📄 第 {pdata["page_num"]} 页
'
)
html = pdata["html"]
# 替换图片 src 为相对路径
html = self._relink_images_in_html(html, image_map)
body_content = self._extract_body_content(html)
parts.append(body_content)
# 附加 Canvas
if canvas_paths:
parts.append('
📊 图表
')
for cp in canvas_paths:
rel_path = os.path.relpath(
cp, os.path.dirname(output_path)
).replace("\\", "/")
parts.append(
f'
'
f'{os.path.basename(cp)}'
)
parts.extend(['', ''])
Path(output_path).write_text("\n".join(parts), encoding="utf-8")
# ── 内部: HTML 处理 ───────────────────────────────────────────────
def _inline_images_in_html(self, html: str, image_map: Dict[str, str],
base_url: str) -> str:
"""将 HTML 中的图片 src 替换为 base64 data URI。"""
def _replace_img(m: re.Match) -> str:
tag = m.group(0)
src_m = re.search(r'src\s*=\s*["\']([^"\']+)["\']', tag, re.IGNORECASE)
if not src_m:
return tag
src = src_m.group(1)
full_src = urljoin(base_url, src)
local = image_map.get(full_src) or image_map.get(src)
if local and os.path.exists(local):
b64 = self._file_to_base64(local)
if b64:
ext = os.path.splitext(local)[1].lower()
mime_map = {".png": "image/png", ".jpg": "image/jpeg",
".jpeg": "image/jpeg", ".gif": "image/gif",
".webp": "image/webp", ".svg": "image/svg+xml",
".bmp": "image/bmp"}
mime = mime_map.get(ext, "image/png")
tag = tag.replace(
src_m.group(1),
f"data:{mime};base64,{b64}",
)
return tag
# 还有 CSS url() 引用
def _replace_css_url(m: re.Match) -> str:
full = m.group(0)
inner = m.group(1)
if inner.startswith("data:"):
return full
full_url = urljoin(base_url, inner)
local = image_map.get(full_url) or image_map.get(inner)
if local and os.path.exists(local):
b64 = self._file_to_base64(local)
if b64:
ext = os.path.splitext(local)[1].lower()
mime = "image/png" if ext == ".png" else "image/jpeg"
return f"url(data:{mime};base64,{b64})"
return full
html = _IMG_SRC_RE.sub(_replace_img, html)
html = _CSS_URL_RE.sub(_replace_css_url, html)
return html
def _relink_images_in_html(self, html: str,
image_map: Dict[str, str]) -> str:
"""将图片 src 替换为本地相对路径。"""
def _replace(m: re.Match) -> str:
tag = m.group(0)
src_m = re.search(r'src\s*=\s*["\']([^"\']+)["\']', tag, re.IGNORECASE)
if not src_m:
return tag
src = src_m.group(1)
local = image_map.get(src)
if local:
# 使用 assets/images/ 相对路径
fname = os.path.basename(local)
tag = tag.replace(src_m.group(1), f"assets/images/{fname}")
return tag
return re.sub(r'
]+src\s*=\s*["\']([^"\']+)["\'][^>]*>',
_replace, html, flags=re.IGNORECASE)
def _extract_body_content(self, html: str) -> str:
"""抽取 内容(无 body 标签时返回全文)。"""
m = re.search(r']*>([\s\S]*?)', html, re.IGNORECASE)
if m:
return m.group(1)
return html
# ── 内部: 工具函数 ────────────────────────────────────────────────
@staticmethod
def _extract_title(html: str) -> str:
m = re.search(r']*>([\s\S]*?)', html, re.IGNORECASE)
if m:
return re.sub(r'\s+', ' ', m.group(1)).strip()[:200]
return ""
@staticmethod
def _extract_domain(url: str) -> str:
try:
return urlparse(url).netloc.replace("www.", "")[:40]
except Exception:
return "unknown"
@staticmethod
def _guess_image_ext(url: str) -> str:
path = urlparse(url).path.lower()
for ext in (".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg"):
if path.endswith(ext):
return ".jpg" if ext == ".jpeg" else ext
return ".jpg"
@staticmethod
def _escape_html(text: str) -> str:
return (text.replace("&", "&").replace("<", "<")
.replace(">", ">").replace('"', """))
@staticmethod
def _file_to_base64(file_path: str) -> str:
"""文件转 base64 字符串。"""
try:
with open(file_path, "rb") as f:
return base64.b64encode(f.read()).decode("ascii")
except Exception:
return ""
# ── 便捷函数 ────────────────────────────────────────────────────────────────
def quick_archive(url: str, **kwargs) -> ArchiveResult:
"""一行归档:默认同时生成内嵌版 + 目录版。"""
archiver = FullPageArchiver()
return archiver.archive(url, **kwargs)
# ── CLI ──────────────────────────────────────────────────────────────────────
def main():
import argparse
ap = argparse.ArgumentParser(description="全页内容归档器 v1.0")
ap.add_argument("url", help="目标 URL")
ap.add_argument("--paginate", action="store_true", help="启用翻页")
ap.add_argument("--next-selector", default=".next,a[rel=next]",
help="下一页选择器")
ap.add_argument("--max-pages", type=int, default=5, help="最大页数")
ap.add_argument("--output-dir", help="输出目录")
ap.add_argument("--no-inline", action="store_true", help="不生成内嵌版")
ap.add_argument("--no-assets", action="store_true", help="不生成目录版")
ap.add_argument("--json", action="store_true", help="输出 JSON 格式结果")
args = ap.parse_args()
print(f"📥 正在归档: {args.url}")
result = quick_archive(
args.url,
paginate=args.paginate,
next_selector=args.next_selector if args.paginate else None,
max_pages=args.max_pages,
output_dir=args.output_dir,
inline_images=not args.no_inline,
save_assets=not args.no_assets,
)
if args.json:
print(json.dumps(result.to_dict(), ensure_ascii=False, indent=2))
else:
print(result.summary)
if __name__ == "__main__":
main()