207 lines
7.0 KiB
Python
207 lines
7.0 KiB
Python
import zipfile
|
|
import hashlib
|
|
import os
|
|
from lxml import etree
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.models.book import Book, Chapter, Segment
|
|
from app.epub.toc import (
|
|
parse_toc_tree, flatten_headings, build_heading_paths,
|
|
href_to_fragment, NS, _read_opf, _resolve_path, _dirname,
|
|
)
|
|
from app.epub.cleaner import extract_paragraphs, clean_html, detect_quality
|
|
from app.epub.chunker import chapter_to_chunks
|
|
from app.epub.cfi import make_cfi, make_element_id
|
|
|
|
|
|
async def parse_epub(file_path: str, db: AsyncSession) -> Book:
|
|
"""完整解析 EPUB，写入数据库，返回 Book 对象"""
|
|
file_hash = _hash_file(file_path)
|
|
|
|
existing = await db.execute(
|
|
__import__("sqlalchemy").select(Book).where(Book.file_hash == file_hash)
|
|
)
|
|
if existing.scalar_one_or_none():
|
|
raise ValueError("EPUB 已存在（根据文件哈希去重）")
|
|
|
|
filename = os.path.basename(file_path)
|
|
|
|
with zipfile.ZipFile(file_path) as zf:
|
|
opf_path, opf_root = _read_opf(zf)
|
|
opf_dir = _dirname(opf_path)
|
|
|
|
title, author, language = _extract_metadata(opf_root)
|
|
spine_items = _get_spine_items(zf, opf_root, opf_dir)
|
|
|
|
headings = parse_toc_tree(file_path)
|
|
flat = flatten_headings(headings)
|
|
flat_with_paths = build_heading_paths(flat, 0)
|
|
|
|
book = Book(
|
|
filename=filename,
|
|
title=title,
|
|
author=author,
|
|
language=language,
|
|
file_path=file_path,
|
|
file_hash=file_hash,
|
|
status="parsing",
|
|
)
|
|
db.add(book)
|
|
await db.flush()
|
|
|
|
total_chars = 0
|
|
|
|
spine_index_map = {}
|
|
with zipfile.ZipFile(file_path) as zf:
|
|
_, opf_root = _read_opf(zf)
|
|
manifest = {item.attrib.get("id"): item.attrib.get("href", "") for item in opf_root.findall("opf:manifest/opf:item", NS)}
|
|
spine_ids = [itemref.attrib.get("idref") for itemref in opf_root.findall("opf:spine/opf:itemref", NS)]
|
|
for idx, idref in enumerate(spine_ids):
|
|
href = manifest.get(idref, "")
|
|
if href:
|
|
spine_index_map[href] = idx
|
|
|
|
with zipfile.ZipFile(file_path) as zf:
|
|
chapters_to_insert = _match_chapters_to_spine(
|
|
flat_with_paths, spine_items, zf, opf_dir,
|
|
)
|
|
|
|
chapter_order = 0
|
|
for ch_info in chapters_to_insert:
|
|
paragraphs = extract_paragraphs(ch_info["html"], fix_wraps=True)
|
|
chapter_text = "\n\n".join(p["text"] for p in paragraphs)
|
|
chapter_chars = len(chapter_text)
|
|
total_chars += chapter_chars
|
|
|
|
href = ch_info["href"]
|
|
spine_idx = spine_index_map.get(href, chapter_order)
|
|
cfi_start = make_cfi(spine_idx, href, ch_info.get("fragment", ""))
|
|
|
|
chapter = Chapter(
|
|
book_id=book.id,
|
|
order_idx=chapter_order,
|
|
title=ch_info["title"],
|
|
level=ch_info.get("level", 1),
|
|
heading_path=ch_info.get("heading_path", ""),
|
|
cfi_start=cfi_start,
|
|
cfi_end="",
|
|
char_count=chapter_chars,
|
|
)
|
|
db.add(chapter)
|
|
await db.flush()
|
|
|
|
chunks = chapter_to_chunks(
|
|
paragraphs,
|
|
heading_path=ch_info.get("heading_path", ""),
|
|
cfi_start=cfi_start,
|
|
book_char_total=total_chars,
|
|
)
|
|
|
|
for seg_order, chunk in enumerate(chunks):
|
|
segment = Segment(
|
|
chapter_id=chapter.id,
|
|
book_id=book.id,
|
|
order_idx=seg_order,
|
|
text=chunk["text"],
|
|
char_count=chunk["char_count"],
|
|
cfi=chunk["cfi"],
|
|
)
|
|
db.add(segment)
|
|
|
|
chapter_order += 1
|
|
|
|
book.char_count = total_chars
|
|
book.status = "ready"
|
|
quality_score, quality_issues = _assess_quality(zf=None, chapters=chapters_to_insert, file_path=file_path)
|
|
await db.flush()
|
|
|
|
return book
|
|
|
|
|
|
def _hash_file(file_path: str) -> str:
|
|
h = hashlib.sha256()
|
|
with open(file_path, "rb") as f:
|
|
for chunk in iter(lambda: f.read(8192), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def _extract_metadata(opf: etree._Element) -> tuple[str, str, str]:
|
|
ns = {"dc": "http://purl.org/dc/elements/1.1/"}
|
|
title_el = opf.find(".//dc:title", ns)
|
|
creator_el = opf.find(".//dc:creator", ns)
|
|
lang_el = opf.find(".//dc:language", ns)
|
|
title = title_el.text.strip() if title_el is not None and title_el.text else "未知书名"
|
|
author = creator_el.text.strip() if creator_el is not None and creator_el.text else "未知作者"
|
|
language = lang_el.text.strip() if lang_el is not None and lang_el.text else "zh"
|
|
return title, author, language
|
|
|
|
|
|
def _get_spine_items(zf: zipfile.ZipFile, opf: etree._Element, opf_dir: str) -> list[dict]:
|
|
"""获取 spine 中所有文件的 HTML 内容"""
|
|
manifest = {item.attrib.get("id"): item.attrib.get("href", "") for item in opf.findall("opf:manifest/opf:item", NS)}
|
|
spine_ids = [itemref.attrib.get("idref") for itemref in opf.findall("opf:spine/opf:itemref", NS)]
|
|
items = []
|
|
for idref in spine_ids:
|
|
href = manifest.get(idref, "")
|
|
if not href:
|
|
continue
|
|
path = _resolve_path(opf_dir, href)
|
|
try:
|
|
html = zf.read(path)
|
|
items.append({"href": href, "html": html, "path": path})
|
|
except KeyError:
|
|
continue
|
|
return items
|
|
|
|
|
|
def _match_chapters_to_spine(
|
|
flat_headings: list[dict],
|
|
spine_items: list[dict],
|
|
zf: zipfile.ZipFile,
|
|
opf_dir: str,
|
|
) -> list[dict]:
|
|
"""将目录条目与 spine 文件匹配，按 href 对齐"""
|
|
chapter_map = {}
|
|
for h in flat_headings:
|
|
href = h.get("href", "")
|
|
base = href.split("#")[0] if "#" in href else href
|
|
frag = href.split("#", 1)[1] if "#" in href else ""
|
|
if base not in chapter_map:
|
|
chapter_map[base] = []
|
|
chapter_map[base].append({**h, "fragment": frag})
|
|
|
|
result = []
|
|
seen = set()
|
|
for item in spine_items:
|
|
href = item["href"]
|
|
base = href.split("#")[0] if "#" in href else href
|
|
if base in chapter_map:
|
|
for ch in chapter_map[base]:
|
|
if ch.get("title", "") not in seen:
|
|
seen.add(ch.get("title", ""))
|
|
result.append({**ch, "html": item["html"]})
|
|
else:
|
|
result.append({
|
|
"title": f"Section {len(result) + 1}",
|
|
"level": 1,
|
|
"href": href,
|
|
"heading_path": "",
|
|
"fragment": "",
|
|
"html": item["html"],
|
|
})
|
|
return result
|
|
|
|
|
|
def _assess_quality(zf, chapters: list[dict], file_path: str) -> tuple[int, list[str]]:
|
|
"""评估解析质量"""
|
|
if not chapters:
|
|
return 0, ["无有效章节"]
|
|
sample_text = "\n".join(
|
|
clean_html(ch["html"], fix_wraps=False)
|
|
for ch in chapters[:5]
|
|
if ch.get("html")
|
|
)
|
|
if not sample_text:
|
|
return 0, ["无法提取文本"]
|
|
return detect_quality(sample_text) |