97 lines
2.5 KiB
Python
97 lines
2.5 KiB
Python
import re
|
|
|
|
|
|
def chunk_paragraphs(
|
|
paragraphs: list[dict],
|
|
heading_path: str,
|
|
cfi_base: str,
|
|
target_chars: int = 1500,
|
|
max_chars: int = 4000,
|
|
min_chars: int = 200,
|
|
) -> list[dict]:
|
|
"""将段落列表按语义边界聚类，不按固定字符数切。
|
|
|
|
策略：
|
|
- 章节总字数 < target_chars: 整章一块
|
|
- otherwise: 按段落聚类，目标 target_chars/块，允许±50%浮动
|
|
"""
|
|
total_chars = sum(len(p["text"]) for p in paragraphs)
|
|
if total_chars <= target_chars:
|
|
merged_text = "\n\n".join(p["text"] for p in paragraphs)
|
|
prefix = f"《heading_path》" if heading_path else ""
|
|
return [{
|
|
"text": f"{prefix}\n{merged_text}" if prefix else merged_text,
|
|
"char_count": len(merged_text),
|
|
"cfi": cfi_base,
|
|
"heading_path": heading_path,
|
|
}]
|
|
|
|
chunks = []
|
|
buf = []
|
|
buf_chars = 0
|
|
|
|
def flush():
|
|
nonlocal buf_chars
|
|
if not buf:
|
|
return
|
|
merged = "\n\n".join(p["text"] for p in buf)
|
|
prefix = f"《heading_path》" if heading_path else ""
|
|
text = f"{prefix}\n{merged}" if prefix else merged
|
|
chunks.append({
|
|
"text": text,
|
|
"char_count": len(merged),
|
|
"cfi": cfi_base,
|
|
"heading_path": heading_path,
|
|
})
|
|
buf.clear()
|
|
buf_chars = 0
|
|
|
|
for i, p in enumerate(paragraphs):
|
|
p_chars = len(p["text"])
|
|
|
|
if p["tag"] in ("h1", "h2", "h3", "h4"):
|
|
flush()
|
|
prefix = f"《heading_path》" if heading_path else ""
|
|
text = f"{prefix}\n{p['text']}" if prefix else p["text"]
|
|
chunks.append({
|
|
"text": text,
|
|
"char_count": p_chars,
|
|
"cfi": cfi_base,
|
|
"heading_path": heading_path,
|
|
})
|
|
continue
|
|
|
|
if buf_chars + p_chars > target_chars * 1.5:
|
|
flush()
|
|
|
|
buf.append(p)
|
|
buf_chars += p_chars
|
|
|
|
flush()
|
|
|
|
for i in range(len(chunks)):
|
|
chunks[i]["cfi"] = f"{cfi_base}!chunk-{i:04d}"
|
|
|
|
return chunks
|
|
|
|
|
|
def chapter_to_chunks(
|
|
paragraphs: list[dict],
|
|
heading_path: str,
|
|
cfi_start: str,
|
|
book_char_total: int,
|
|
) -> list[dict]:
|
|
"""根据书的总规模决定切块策略"""
|
|
if book_char_total < 50_000:
|
|
target = 2000
|
|
elif book_char_total < 500_000:
|
|
target = 1500
|
|
else:
|
|
target = 1000
|
|
|
|
return chunk_paragraphs(
|
|
paragraphs,
|
|
heading_path=heading_path,
|
|
cfi_base=cfi_start,
|
|
target_chars=target,
|
|
) |