# -*- coding: utf-8 -*-
"""
增强文件/网页解析模块 v4.0
支持 PPT、HTML、Markdown、CSV、JSON 等格式的深度解析，
与现有 document_parser.py 无缝集成。

新增能力：
- PPTX 解析：文本 + 表格 + 备注 + 图片alt文本
- HTML 解析：清洗标签、保留表格结构、提取 meta/JSON-LD
- Markdown 解析：章节结构、表格、代码块
- MultiFormatParser：自动检测格式，统一入口
"""

import re
import os
import json
import csv
import io
import html as html_module
from pathlib import Path
from typing import Dict, Any, List, Optional, Tuple
from datetime import datetime

# ==================== 依赖检测 ====================

PPTX_AVAILABLE = False
try:
    from pptx import Presentation  # type: ignore
    PPTX_AVAILABLE = True
except ImportError:
    pass

HTML_PARSER_AVAILABLE = False
try:
    from html.parser import HTMLParser as StdHTMLParser
    HTML_PARSER_AVAILABLE = True
except ImportError:
    pass

LXML_AVAILABLE = False
try:
    import lxml.html  # type: ignore
    LXML_AVAILABLE = True
except ImportError:
    pass

BS4_AVAILABLE = False
try:
    from bs4 import BeautifulSoup  # type: ignore
    BS4_AVAILABLE = True
except ImportError:
    pass

SKILL_DATA_DIR = Path(__file__).parent.parent / "data"
SKILL_DATA_DIR.mkdir(parents=True, exist_ok=True)


# ==================== PPTX 解析器 ====================

class PPTXParser:
    """PowerPoint 文件解析器，支持文本/表格/备注提取。"""

    def __init__(self):
        self.available = PPTX_AVAILABLE

    def parse(self, file_path: str) -> Dict[str, Any]:
        """
        解析 PPTX 文件。

        返回: {
            file_type, file_path, slide_count,
            slides: [{slide_num, title, texts, tables, notes, images}],
            all_text, all_notes, all_tables_summary
        }
        """
        result: Dict[str, Any] = {
            "file_type": "pptx",
            "file_path": file_path,
            "slide_count": 0,
            "slides": [],
            "all_text": "",
            "all_notes": "",
            "all_tables_summary": [],
            "metadata": {},
        }

        if not self.available:
            result["error"] = "python-pptx 未安装，请运行: pip install python-pptx"
            return result

        try:
            prs = Presentation(file_path)

            # 提取文档属性
            props = prs.core_properties
            result["metadata"] = {
                "title": getattr(props, 'title', '') or '',
                "author": getattr(props, 'author', '') or '',
                "created": str(getattr(props, 'created', '')) if getattr(props, 'created', None) else '',
                "modified": str(getattr(props, 'modified', '')) if getattr(props, 'modified', None) else '',
                "slide_count": len(prs.slides),
            }

            result["slide_count"] = len(prs.slides)
            all_text_parts = []
            all_notes_parts = []

            for slide_num, slide in enumerate(prs.slides, 1):
                slide_data = {
                    "slide_num": slide_num,
                    "title": "",
                    "texts": [],
                    "tables": [],
                    "notes": "",
                    "image_count": 0,
                }

                # 提取标题（第一个非空文本框视为标题）
                first_text_found = False

                for shape in slide.shapes:
                    # 文本框
                    if shape.has_text_frame:
                        text = shape.text_frame.text.strip()
                        if text:
                            if not first_text_found:
                                slide_data["title"] = text[:200]
                                first_text_found = True
                            slide_data["texts"].append(text)
                            all_text_parts.append(text)

                    # 表格
                    if shape.has_table:
                        table = shape.table
                        table_data = {
                            "rows": table.rows.__len__(),
                            "cols": len(table.columns),
                            "headers": [],
                            "data": [],
                        }
                        for ri, row in enumerate(table.rows):
                            row_data = [cell.text.strip() for cell in row.cells]
                            if ri == 0:
                                table_data["headers"] = row_data
                            else:
                                table_data["data"].append(row_data)
                        slide_data["tables"].append(table_data)
                        result["all_tables_summary"].append({
                            "slide": slide_num,
                            "rows": table_data["rows"],
                            "cols": table_data["cols"],
                            "headers": table_data["headers"][:8],
                        })

                    # 图片计数
                    if shape.shape_type == 13:  # MSO_SHAPE_TYPE.PICTURE
                        slide_data["image_count"] += 1

                # 提取备注
                if slide.has_notes_slide:
                    notes_slide = slide.notes_slide
                    notes_text = notes_slide.notes_text_frame.text.strip() if notes_slide.notes_text_frame else ""
                    slide_data["notes"] = notes_text
                    if notes_text:
                        all_notes_parts.append(f"[Slide {slide_num}]\n{notes_text}")

                result["slides"].append(slide_data)

            result["all_text"] = "\n\n".join(all_text_parts)
            result["all_notes"] = "\n\n".join(all_notes_parts)
            result["text_content"] = result["all_text"]  # 兼容 document_parser 接口

        except Exception as e:
            result["error"] = f"PPTX 解析失败: {str(e)}"

        return result


# ==================== HTML 解析器 ====================

class HTMLParser:
    """HTML 网页/文件解析器，支持清洗标签、提取表格/结构化数据。"""

    def __init__(self):
        self.available = True

    def parse(self, source: str, is_url: bool = False) -> Dict[str, Any]:
        """
        解析 HTML 内容。

        Args:
            source: HTML 文本字符串或文件路径/URL
            is_url: 是否为 URL（需额外请求）

        返回: {
            file_type, title, meta, text_content, tables, links,
            json_ld, headings, financial_data
        }
        """
        result: Dict[str, Any] = {
            "file_type": "html",
            "title": "",
            "meta": {},
            "text_content": "",
            "tables": [],
            "links": [],
            "json_ld": [],
            "headings": [],
            "financial_data": {},
        }

        html_text = source

        # 如果是文件路径，读取文件
        if not is_url and os.path.isfile(source):
            try:
                with open(source, 'r', encoding='utf-8') as f:
                    html_text = f.read()
                result["file_path"] = source
            except UnicodeDecodeError:
                try:
                    with open(source, 'r', encoding='gbk') as f:
                        html_text = f.read()
                    result["file_path"] = source
                except Exception as e:
                    result["error"] = f"文件读取失败: {str(e)}"
                    return result

        if not html_text or len(html_text.strip()) < 10:
            result["error"] = "HTML 内容为空"
            return result

        try:
            # 提取 title
            title_match = re.search(r'<title[^>]*>(.*?)</title>', html_text, re.IGNORECASE | re.DOTALL)
            if title_match:
                result["title"] = self._clean_text(title_match.group(1))

            # 提取 meta 标签
            for meta_match in re.finditer(
                r'<meta\s+([^>]+)>', html_text, re.IGNORECASE
            ):
                attrs = meta_match.group(1)
                name_match = re.search(r'(?:name|property|http-equiv)\s*=\s*["\']([^"\']+)["\']', attrs, re.IGNORECASE)
                content_match = re.search(r'content\s*=\s*["\']([^"\']+)["\']', attrs, re.IGNORECASE)
                if name_match and content_match:
                    result["meta"][name_match.group(1).lower()] = content_match.group(1)

            # 提取 JSON-LD 结构化数据
            for ld_match in re.finditer(
                r'<script[^>]*type\s*=\s*["\']application/ld\+json["\'][^>]*>(.*?)</script>',
                html_text, re.IGNORECASE | re.DOTALL
            ):
                try:
                    ld_data = json.loads(ld_match.group(1))
                    result["json_ld"].append(ld_data)
                except json.JSONDecodeError:
                    pass

            # 提取表格
            result["tables"] = self._extract_tables(html_text)

            # 提取标题层级
            for level in range(1, 7):
                for h_match in re.finditer(
                    f'<h{level}[^>]*>(.*?)</h{level}>', html_text, re.IGNORECASE | re.DOTALL
                ):
                    text = self._clean_text(h_match.group(1))
                    if text:
                        result["headings"].append({"level": level, "text": text[:200]})

            # 提取链接
            for a_match in re.finditer(
                r'<a\s+[^>]*href\s*=\s*["\']([^"\']+)["\'][^>]*>(.*?)</a>',
                html_text, re.IGNORECASE | re.DOTALL
            ):
                href = a_match.group(1).strip()
                text = self._clean_text(a_match.group(2))
                if href and not href.startswith('#'):
                    result["links"].append({"url": href, "text": text[:200]})

            # 提取纯文本（去除标签）
            result["text_content"] = self._extract_text(html_text)

            # 尝试提取财务数据
            result["financial_data"] = self._extract_financial_from_html(html_text)

            result["char_count"] = len(result["text_content"])

        except Exception as e:
            result["error"] = f"HTML 解析异常: {str(e)}"

        return result

    def _clean_text(self, text: str) -> str:
        """清洗 HTML 实体和多余空白。"""
        text = re.sub(r'<[^>]+>', '', text)  # 去掉嵌套标签
        text = html_module.unescape(text)
        text = re.sub(r'\s+', ' ', text)
        return text.strip()

    def _extract_text(self, html_text: str) -> str:
        """从 HTML 提取纯文本（保留段落结构）。"""
        # 先处理换行标签
        text = re.sub(r'<(?:br|BR)\s*/?>', '\n', html_text)
        text = re.sub(r'<(?:p|P|div|DIV|li|LI|tr|TR|h\d|H\d)[^>]*>', '\n', text)
        # 去除所有标签
        text = re.sub(r'<[^>]+>', '', text)
        # 处理注释
        text = re.sub(r'<!--.*?-->', '', text, flags=re.DOTALL)
        # 处理 script/style
        text = re.sub(r'<(?:script|style|SCRIPT|STYLE)[^>]*>.*?</(?:script|style|SCRIPT|STYLE)>',
                      '', text, flags=re.DOTALL)
        # 解码 HTML 实体
        text = html_module.unescape(text)
        # 压缩空行
        text = re.sub(r'\n\s*\n', '\n\n', text)
        text = re.sub(r'[ \t]{2,}', ' ', text)
        return text.strip()

    def _extract_tables(self, html_text: str) -> List[Dict[str, Any]]:
        """提取 HTML 中的表格。"""
        tables = []
        for table_match in re.finditer(r'<table[^>]*>(.*?)</table>', html_text, re.IGNORECASE | re.DOTALL):
            table_html = table_match.group(1)
            rows = []
            for tr_match in re.finditer(r'<tr[^>]*>(.*?)</tr>', table_html, re.IGNORECASE | re.DOTALL):
                row_html = tr_match.group(1)
                cells = []
                for cell_match in re.finditer(
                    r'<t[dh][^>]*>(.*?)</t[dh]>', row_html, re.IGNORECASE | re.DOTALL
                ):
                    cells.append(self._clean_text(cell_match.group(1)))
                if cells:
                    rows.append(cells)
            if rows:
                tables.append({
                    "rows": len(rows),
                    "cols": max(len(r) for r in rows) if rows else 0,
                    "headers": rows[0] if rows else [],
                    "data": rows[1:] if len(rows) > 1 else [],
                })
        return tables

    def _extract_financial_from_html(self, html_text: str) -> Dict[str, Any]:
        """从 HTML 文本中提取财务数据。"""
        fin_data: Dict[str, Any] = {}
        text = self._extract_text(html_text)

        patterns = {
            "营业收入": r'营业(?:总)?收入[：:]\s*([\d,.]+\s*(?:亿|万|元)?)',
            "净利润": r'(?:归属.*)?净利润[：:]\s*([\d,.]+\s*(?:亿|万|元)?)',
            "总资产": r'总资产[：:]\s*([\d,.]+\s*(?:亿|万|元)?)',
            "每股收益": r'(?:基本)?每股收益[：:]\s*([\d,.]+)',
            "净资产收益率": r'净资产收益率[：:]\s*([\d,.]+)\s*%?',
        }
        for key, pat in patterns.items():
            m = re.search(pat, text)
            if m:
                fin_data[key] = m.group(1).strip()

        return fin_data


# ==================== Markdown 解析器 ====================

class MarkdownParser:
    """Markdown 文档解析器，支持章节/表格/代码块提取。"""

    def parse(self, source: str) -> Dict[str, Any]:
        """
        解析 Markdown 内容。

        返回: {file_type, title, sections, tables, code_blocks, text_content, metadata}
        """
        result: Dict[str, Any] = {
            "file_type": "markdown",
            "title": "",
            "sections": [],
            "tables": [],
            "code_blocks": [],
            "text_content": "",
            "metadata": {},
        }

        if os.path.isfile(source):
            try:
                with open(source, 'r', encoding='utf-8') as f:
                    text = f.read()
                result["file_path"] = source
            except Exception as e:
                result["error"] = f"文件读取失败: {str(e)}"
                return result
        else:
            text = source

        if not text.strip():
            result["error"] = "内容为空"
            return result

        lines = text.split('\n')
        result["text_content"] = text

        # 提取 frontmatter
        fm = self._extract_frontmatter(text)
        if fm:
            result["metadata"] = fm

        # 提取标题 (第一个 # 标题)
        for line in lines:
            h1 = re.match(r'^#\s+(.+)', line)
            if h1:
                result["title"] = h1.group(1).strip()[:200]
                break

        if not result["title"] and result["metadata"].get("title"):
            result["title"] = result["metadata"]["title"]

        # 提取章节结构
        current_section = None
        for line in lines:
            sec_match = re.match(r'^(#{1,6})\s+(.+)', line)
            if sec_match:
                level = len(sec_match.group(1))
                title = sec_match.group(2).strip()[:200]
                section = {"level": level, "title": title, "content_lines": []}
                result["sections"].append(section)
                current_section = section
            elif current_section is not None:
                current_section["content_lines"].append(line)

        # 为每个章节生成摘要
        for sec in result["sections"]:
            sec["preview"] = ' '.join(sec["content_lines"][:3])[:200]

        # 提取表格
        result["tables"] = self._extract_md_tables(text)

        # 提取代码块
        for code_match in re.finditer(r'```(\w*)\n(.*?)```', text, re.DOTALL):
            result["code_blocks"].append({
                "language": code_match.group(1) or "text",
                "content": code_match.group(2).strip(),
                "line_count": len(code_match.group(2).strip().split('\n')),
            })

        result["char_count"] = len(text)
        return result

    def _extract_frontmatter(self, text: str) -> Dict[str, Any]:
        """提取 YAML frontmatter。"""
        fm_match = re.match(r'^---\s*\n(.*?)\n---', text, re.DOTALL)
        if not fm_match:
            return {}
        fm = {}
        try:
            import yaml
            return yaml.safe_load(fm_match.group(1)) or {}
        except ImportError:
            pass

        # 手动解析简单 YAML
        for line in fm_match.group(1).split('\n'):
            kv = re.match(r'^(\w[\w_-]*)\s*:\s*(.+)', line.strip())
            if kv:
                key, val = kv.group(1), kv.group(2).strip().strip('"\'')
                fm[key] = val
        return fm

    def _extract_md_tables(self, text: str) -> List[Dict[str, Any]]:
        """提取 Markdown 表格。"""
        tables = []
        lines = text.split('\n')
        i = 0
        while i < len(lines):
            line = lines[i].strip()
            if '|' in line and line.count('|') >= 1:
                # 检查下一行是否是分隔行
                if i + 1 < len(lines) and re.match(r'^[\|\s\-:]+$', lines[i + 1].strip()):
                    # 这是一个表格
                    table_lines = [line]
                    header = [c.strip() for c in line.split('|') if c.strip()]
                    i += 2  # 跳过分隔行
                    data_rows = []
                    while i < len(lines) and '|' in lines[i]:
                        row = [c.strip() for c in lines[i].split('|') if c.strip()]
                        if row:
                            data_rows.append(row)
                        i += 1
                    tables.append({
                        "rows": 1 + len(data_rows),
                        "cols": len(header),
                        "headers": header,
                        "data": data_rows,
                    })
                    continue
            i += 1
        return tables


# ==================== CSV 解析器 ====================

class CSVParser:
    """CSV 文件解析器，支持多种编码自动检测。"""

    def parse(self, file_path: str) -> Dict[str, Any]:
        """解析 CSV 文件。"""
        result: Dict[str, Any] = {
            "file_type": "csv",
            "file_path": file_path,
            "headers": [],
            "rows": [],
            "row_count": 0,
            "col_count": 0,
            "text_content": "",
        }

        if not os.path.isfile(file_path):
            result["error"] = f"文件不存在: {file_path}"
            return result

        encodings = ['utf-8', 'utf-8-sig', 'gbk', 'gb2312', 'gb18030']
        for enc in encodings:
            try:
                with open(file_path, 'r', encoding=enc) as f:
                    reader = csv.reader(f)
                    rows = list(reader)
                if rows:
                    result["headers"] = rows[0]
                    result["rows"] = rows[1:]
                    result["row_count"] = len(rows)
                    result["col_count"] = len(rows[0]) if rows else 0
                    result["text_content"] = '\n'.join([','.join(r) for r in rows])
                    result["encoding"] = enc
                    return result
            except (UnicodeDecodeError, Exception):
                continue

        result["error"] = "无法识别 CSV 编码"
        return result


# ==================== 多格式统一解析器 ====================

class MultiFormatParser:
    """
    多格式统一解析入口。
    自动检测文件格式，分发到对应解析器。
    """

    SUPPORTED_EXTENSIONS = {
        '.pptx': 'pptx',
        '.ppt': 'pptx',
        '.html': 'html',
        '.htm': 'html',
        '.md': 'markdown',
        '.markdown': 'markdown',
        '.csv': 'csv',
        '.json': 'json',
        '.pdf': 'pdf',
        '.docx': 'docx',
        '.doc': 'docx',
        '.xlsx': 'xlsx',
        '.xls': 'xlsx',
        '.txt': 'text',
    }

    def __init__(self):
        self.pptx_parser = PPTXParser()
        self.html_parser = HTMLParser()
        self.md_parser = MarkdownParser()
        self.csv_parser = CSVParser()

    def parse(self, source: str, fmt: str = "auto") -> Dict[str, Any]:
        """
        统一解析入口。

        Args:
            source: 文件路径、URL 或文本内容
            fmt: 格式 — auto(自动检测) / pptx / html / markdown / csv

        返回: 解析结果字典
        """
        # 自动检测
        if fmt == "auto":
            if source.startswith(('http://', 'https://')):
                fmt = "html_url"
            elif os.path.isfile(source):
                ext = Path(source).suffix.lower()
                fmt = self.SUPPORTED_EXTENSIONS.get(ext, "text")
            else:
                # 启发式检测：是否为 HTML/Markdown 文本
                if re.search(r'<(!DOCTYPE|html|head|body|div|table)', source[:500], re.IGNORECASE):
                    fmt = "html"
                elif re.search(r'^#{1,6}\s', source[:500], re.MULTILINE):
                    fmt = "markdown"
                else:
                    fmt = "text"

        # 分发解析
        if fmt == "pptx":
            return self.pptx_parser.parse(source)
        elif fmt in ("html", "html_url"):
            return self.html_parser.parse(source, is_url=(fmt == "html_url"))
        elif fmt == "markdown":
            return self.md_parser.parse(source)
        elif fmt == "csv":
            return self.csv_parser.parse(source)
        elif fmt == "json":
            return self._parse_json(source)
        elif fmt in ("pdf", "docx", "doc", "xlsx", "xls", "text"):
            # 委托给现有 document_parser
            try:
                from document_parser import parse_document
                return parse_document(source)
            except ImportError:
                return {"error": "document_parser 不可用", "file_path": source}
        else:
            return {"error": f"不支持的格式: {fmt}", "file_path": source}

    def _parse_json(self, source: str) -> Dict[str, Any]:
        """解析 JSON 文件。"""
        if os.path.isfile(source):
            try:
                with open(source, 'r', encoding='utf-8') as f:
                    data = json.load(f)
            except Exception as e:
                return {"error": f"JSON 解析失败: {str(e)}", "file_path": source}
        else:
            try:
                data = json.loads(source)
            except json.JSONDecodeError as e:
                return {"error": f"JSON 解析失败: {str(e)}"}

        return {
            "file_type": "json",
            "file_path": source if os.path.isfile(source) else "",
            "data": data,
            "text_content": json.dumps(data, ensure_ascii=False, indent=2),
            "char_count": len(json.dumps(data, ensure_ascii=False)),
        }


# ==================== 便捷函数 ====================

def parse_file_enhanced(file_path: str, fmt: str = "auto") -> Dict[str, Any]:
    """一键解析任意文件/网页。"""
    parser = MultiFormatParser()
    return parser.parse(file_path, fmt)


def parse_pptx(file_path: str) -> Dict[str, Any]:
    """解析 PPTX 文件。"""
    return PPTXParser().parse(file_path)


def parse_html(source: str, is_url: bool = False) -> Dict[str, Any]:
    """解析 HTML 内容。"""
    return HTMLParser().parse(source, is_url)


def parse_markdown(source: str) -> Dict[str, Any]:
    """解析 Markdown 内容。"""
    return MarkdownParser().parse(source)


def extract_tables_from_html(html_text: str) -> List[Dict[str, Any]]:
    """从 HTML 中提取所有表格。"""
    parser = HTMLParser()
    return parser._extract_tables(html_text)


# ==================== CLI 入口 ====================

if __name__ == "__main__":
    import sys

    if len(sys.argv) < 2:
        print("用法: python enhanced_parser.py <文件路径> [格式]")
        print("格式: auto(默认) / pptx / html / markdown / csv / json")
        sys.exit(1)

    source = sys.argv[1]
    fmt = sys.argv[2] if len(sys.argv) > 2 else "auto"

    parser = MultiFormatParser()
    result = parser.parse(source, fmt)

    if result.get("error"):
        print(f"❌ {result['error']}")
    else:
        print(f"✅ 解析成功: {result.get('file_type', '?').upper()}")
        print(f"   字符数: {result.get('char_count', 0):,}")
        if result.get("title"):
            print(f"   标题: {result['title']}")
        if result.get("slide_count"):
            print(f"   幻灯片: {result['slide_count']} 页")
        if result.get("tables"):
            print(f"   表格: {len(result['tables'])} 个")
        if result.get("headings"):
            print(f"   章节: {len(result['headings'])} 个")
        print(f"\n--- 文本预览 (前500字) ---")
        preview = result.get("text_content", "")[:500]
        print(preview)