import zipfile import hashlib import os from lxml import etree from sqlalchemy.ext.asyncio import AsyncSession from app.models.book import Book, Chapter, Segment from app.epub.toc import ( parse_toc_tree, flatten_headings, build_heading_paths, href_to_fragment, NS, _read_opf, _resolve_path, _dirname, ) from app.epub.cleaner import extract_paragraphs, clean_html, detect_quality from app.epub.chunker import chapter_to_chunks from app.epub.cfi import make_cfi, make_element_id async def parse_epub(file_path: str, db: AsyncSession) -> Book: """完整解析 EPUB，写入数据库，返回 Book 对象""" file_hash = _hash_file(file_path) existing = await db.execute( __import__("sqlalchemy").select(Book).where(Book.file_hash == file_hash) ) if existing.scalar_one_or_none(): raise ValueError("EPUB 已存在（根据文件哈希去重）") filename = os.path.basename(file_path) with zipfile.ZipFile(file_path) as zf: opf_path, opf_root = _read_opf(zf) opf_dir = _dirname(opf_path) title, author, language = _extract_metadata(opf_root) spine_items = _get_spine_items(zf, opf_root, opf_dir) headings = parse_toc_tree(file_path) flat = flatten_headings(headings) flat_with_paths = build_heading_paths(flat, 0) book = Book( filename=filename, title=title, author=author, language=language, file_path=file_path, file_hash=file_hash, status="parsing", ) db.add(book) await db.flush() total_chars = 0 spine_index_map = {} with zipfile.ZipFile(file_path) as zf: _, opf_root = _read_opf(zf) manifest = {item.attrib.get("id"): item.attrib.get("href", "") for item in opf_root.findall("opf:manifest/opf:item", NS)} spine_ids = [itemref.attrib.get("idref") for itemref in opf_root.findall("opf:spine/opf:itemref", NS)] for idx, idref in enumerate(spine_ids): href = manifest.get(idref, "") if href: spine_index_map[href] = idx with zipfile.ZipFile(file_path) as zf: chapters_to_insert = _match_chapters_to_spine( flat_with_paths, spine_items, zf, opf_dir, ) chapter_order = 0 for ch_info in chapters_to_insert: paragraphs = extract_paragraphs(ch_info["html"], fix_wraps=True) chapter_text = "\n\n".join(p["text"] for p in paragraphs) chapter_chars = len(chapter_text) total_chars += chapter_chars href = ch_info["href"] spine_idx = spine_index_map.get(href, chapter_order) cfi_start = make_cfi(spine_idx, href, ch_info.get("fragment", "")) chapter = Chapter( book_id=book.id, order_idx=chapter_order, title=ch_info["title"], level=ch_info.get("level", 1), heading_path=ch_info.get("heading_path", ""), cfi_start=cfi_start, cfi_end="", char_count=chapter_chars, ) db.add(chapter) await db.flush() chunks = chapter_to_chunks( paragraphs, heading_path=ch_info.get("heading_path", ""), cfi_start=cfi_start, book_char_total=total_chars, ) for seg_order, chunk in enumerate(chunks): segment = Segment( chapter_id=chapter.id, book_id=book.id, order_idx=seg_order, text=chunk["text"], char_count=chunk["char_count"], cfi=chunk["cfi"], ) db.add(segment) chapter_order += 1 book.char_count = total_chars book.status = "ready" quality_score, quality_issues = _assess_quality(zf=None, chapters=chapters_to_insert, file_path=file_path) await db.flush() return book def _hash_file(file_path: str) -> str: h = hashlib.sha256() with open(file_path, "rb") as f: for chunk in iter(lambda: f.read(8192), b""): h.update(chunk) return h.hexdigest() def _extract_metadata(opf: etree._Element) -> tuple[str, str, str]: ns = {"dc": "http://purl.org/dc/elements/1.1/"} title_el = opf.find(".//dc:title", ns) creator_el = opf.find(".//dc:creator", ns) lang_el = opf.find(".//dc:language", ns) title = title_el.text.strip() if title_el is not None and title_el.text else "未知书名" author = creator_el.text.strip() if creator_el is not None and creator_el.text else "未知作者" language = lang_el.text.strip() if lang_el is not None and lang_el.text else "zh" return title, author, language def _get_spine_items(zf: zipfile.ZipFile, opf: etree._Element, opf_dir: str) -> list[dict]: """获取 spine 中所有文件的 HTML 内容""" manifest = {item.attrib.get("id"): item.attrib.get("href", "") for item in opf.findall("opf:manifest/opf:item", NS)} spine_ids = [itemref.attrib.get("idref") for itemref in opf.findall("opf:spine/opf:itemref", NS)] items = [] for idref in spine_ids: href = manifest.get(idref, "") if not href: continue path = _resolve_path(opf_dir, href) try: html = zf.read(path) items.append({"href": href, "html": html, "path": path}) except KeyError: continue return items def _match_chapters_to_spine( flat_headings: list[dict], spine_items: list[dict], zf: zipfile.ZipFile, opf_dir: str, ) -> list[dict]: """将目录条目与 spine 文件匹配，按 href 对齐""" chapter_map = {} for h in flat_headings: href = h.get("href", "") base = href.split("#")[0] if "#" in href else href frag = href.split("#", 1)[1] if "#" in href else "" if base not in chapter_map: chapter_map[base] = [] chapter_map[base].append({**h, "fragment": frag}) result = [] seen = set() for item in spine_items: href = item["href"] base = href.split("#")[0] if "#" in href else href if base in chapter_map: for ch in chapter_map[base]: if ch.get("title", "") not in seen: seen.add(ch.get("title", "")) result.append({**ch, "html": item["html"]}) else: result.append({ "title": f"Section {len(result) + 1}", "level": 1, "href": href, "heading_path": "", "fragment": "", "html": item["html"], }) return result def _assess_quality(zf, chapters: list[dict], file_path: str) -> tuple[int, list[str]]: """评估解析质量""" if not chapters: return 0, ["无有效章节"] sample_text = "\n".join( clean_html(ch["html"], fix_wraps=False) for ch in chapters[:5] if ch.get("html") ) if not sample_text: return 0, ["无法提取文本"] return detect_quality(sample_text)