my_bookmate/backend/app/epub/cleaner.py

import re
from bs4 import BeautifulSoup, Tag


BLACKLIST_PATTERNS = [
    (re.compile(r"^\s*目\s*录\s*$"), True),
    (re.compile(r"^\s*Contents\b", re.I), True),
    (re.compile(r"^\s*Table of Contents", re.I), True),
    (re.compile(r"版权所有"), True),
    (re.compile(r"Copyright\s*\d{4}", re.I), True),
    (re.compile(r"ISBN[\s:\-]*\d"), True),
    (re.compile(r"版\s*权\s*页"), True),
    (re.compile(r"^\s*[·\-\—\*_=]{3,}\s*$"), True),
    (re.compile(r"^\s*序\s*言\s*$"), True),
    (re.compile(r"^\s*前\s*言\s*$"), True),
    (re.compile(r"^\s*致\s*谢\s*$"), True),
    (re.compile(r"^\s*后\s*记\s*$"), True),
    (re.compile(r"^\s*译者序\s*$"), True),
]

SKIP_TAGS = {"script", "style", "nav", "head", "meta", "link", "br"}

SENTENCE_ENDERS = re.compile(r"[。！？；…）」』】>）\)\"\"]$")
ENGLISH_SENTENCE_ENDERS = re.compile(r"[.!?;\"]$")


def clean_html(html_bytes: bytes, fix_wraps: bool = True) -> str:
    soup = BeautifulSoup(html_bytes, "lxml")

    for tag_name in SKIP_TAGS:
        for tag in soup.find_all(tag_name):
            tag.decompose()

    body = soup.find("body") or soup
    text = body.get_text("\n", strip=False)
    lines = [line.strip() for line in text.split("\n") if line.strip()]

    lines = _filter_boilerplate(lines)
    if fix_wraps:
        lines = _fix_chinese_wraps(lines)

    return "\n\n".join(lines)


def extract_paragraphs(html_bytes: bytes, fix_wraps: bool = True) -> list[dict]:
    """从 HTML 中提取段落列表，返回 [{text, tag, has_children}]"""
    soup = BeautifulSoup(html_bytes, "lxml")
    for tag_name in SKIP_TAGS:
        for tag in soup.find_all(tag_name):
            tag.decompose()

    body = soup.find("body") or soup
    paragraphs = []

    for el in body.descendants:
        if not isinstance(el, Tag):
            continue
        if el.name in SKIP_TAGS:
            continue
        if not el.name or el.name in ("html", "head", "body", "div", "span", "section", "article", "main"):
            continue

        text = el.get_text(" ", strip=True)
        if not text or len(text) < 2:
            continue

        if el.name in ("table",):
            text = _table_to_markdown(el)
        elif el.name in ("blockquote",):
            text = "> " + text.replace("\n", "\n> ")

        paragraphs.append({
            "text": text,
            "tag": el.name,
            "has_children": bool(el.find_all()),
        })

    if fix_wraps:
        paragraphs = _fix_paragraph_wraps(paragraphs)

    return paragraphs


def _filter_boilerplate(lines: list[str]) -> list[str]:
    result = []
    for line in lines:
        stripped = re.sub(r"\s+", "", line)
        if len(stripped) < 15:
            continue
        if any(pattern.search(stripped) for pattern, _ in BLACKLIST_PATTERNS):
            continue
        result.append(line)
    return result


def _fix_chinese_wraps(lines: list[str]) -> list[str]:
    """中文断行修复：末字非句末标点时，与下一行合并"""
    if not lines:
        return lines
    result = []
    buf = ""
    for line in lines:
        line = line.strip()
        if not line:
            if buf:
                result.append(buf)
                buf = ""
            continue
        if not buf:
            buf = line
            continue
        if SENTENCE_ENDERS.search(buf[-1]):
            result.append(buf)
            buf = line
        else:
            buf += line
    if buf:
        result.append(buf)
    return [r for r in result if r]


def _fix_paragraph_wraps(paragraphs: list[dict]) -> list[dict]:
    """段落级别的断行修复，合并被拆分的段落"""
    if not paragraphs:
        return paragraphs
    merged = []
    buf = paragraphs[0]
    for p in paragraphs[1:]:
        if not buf["text"]:
            buf = p
            continue
        if SENTENCE_ENDERS.search(buf["text"][-1:]):
            merged.append(buf)
            buf = p
        elif len(buf["text"]) < 50 and not SENTENCE_ENDERS.search(buf["text"][-1:]):
            buf["text"] = buf["text"] + p["text"]
        else:
            merged.append(buf)
            buf = p
    if buf["text"]:
        merged.append(buf)
    return merged


def _table_to_markdown(table_tag) -> str:
    rows = []
    for tr in table_tag.find_all("tr"):
        cells = [c.get_text(" ", strip=True) for c in tr.find_all(["td", "th"])]
        rows.append(cells)
    if not rows:
        return ""
    col_count = max(len(r) for r in rows)
    for r in rows:
        while len(r) < col_count:
            r.append("")
    col_widths = [max(len(r[i]) for r in rows) for i in range(col_count)]
    lines = []
    for i, row in enumerate(rows):
        line = "| " + " | ".join(cell.ljust(w) for cell, w in zip(row, col_widths)) + " |"
        lines.append(line)
        if i == 0:
            sep = "|" + "|".join("-" * (w + 2) for w in col_widths) + "|"
            lines.append(sep)
    return "\n".join(lines)


def detect_quality(text: str) -> tuple[int, list[str]]:
    """检测文本提取质量，返回 (score 0-100, issues)"""
    score = 100
    issues = []
    lines = [l for l in text.split("\n") if l.strip()]
    short_count = sum(1 for l in lines if len(l) < 10 and len(l) > 0)
    if short_count > len(lines) * 0.3:
        score -= 30
        issues.append("大量短行，可能是伪 EPUB")

    fullwidth = len(re.findall(r"[\u4e00-\u9fff]", text))
    ascii_chars = len(re.findall(r"[a-zA-Z]", text))
    if fullwidth + ascii_chars == 0:
        score -= 50
        issues.append("无有效文本内容")

    if len(text) < 100 and fullwidth == 0:
        score -= 40
        issues.append("文本过短")

    return max(score, 0), issues