jerryandClaude Fable 5
4749f77544
第一批：上传去限制+进度展示、模板HTML预览+按需导出、检索TAB实时更新
第二批：edit护栏移除、导出按钮纯化+file_name落库、编辑失焦还原、
二次预览白底根治（防竞态+渲染兜底）
模板体系：7套公文模板(templates/)+templating渲染器、websearch联网
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
141 lines
4.6 KiB
Python
141 lines
4.6 KiB
Python
"""test_parsing.py —— docx/pdf/文本解析 + 大纲派生。"""
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
|
|
import pytest
|
|
|
|
from vstore.parsing import (
|
|
ParsingError,
|
|
extract,
|
|
extract_docx,
|
|
extract_pdf,
|
|
extract_text_file,
|
|
outline_to_embedding_chunks,
|
|
outline_to_markdown,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# docx
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _make_docx_bytes() -> bytes:
|
|
from docx import Document
|
|
|
|
doc = Document()
|
|
doc.add_heading("第一章 总则", level=1)
|
|
doc.add_paragraph("这是正文第一段，介绍项目背景。")
|
|
doc.add_heading("1.1 适用范围", level=2)
|
|
doc.add_paragraph("本制度适用于全体人员。")
|
|
table = doc.add_table(rows=2, cols=2)
|
|
table.cell(0, 0).text = "字段"
|
|
table.cell(0, 1).text = "说明"
|
|
table.cell(1, 0).text = "名称"
|
|
table.cell(1, 1).text = "项目名称"
|
|
buf = io.BytesIO()
|
|
doc.save(buf)
|
|
return buf.getvalue()
|
|
|
|
|
|
def test_extract_docx_outline_paragraphs_tables():
|
|
parsed = extract_docx(_make_docx_bytes())
|
|
assert [o["text"] for o in parsed["outline"]] == ["第一章 总则", "1.1 适用范围"]
|
|
assert parsed["outline"][0]["level"] == 1
|
|
assert parsed["outline"][1]["level"] == 2
|
|
assert any("正文第一段" in p for p in parsed["paragraphs"])
|
|
assert len(parsed["tables"]) == 1
|
|
assert parsed["tables"][0][0] == ["字段", "说明"]
|
|
assert parsed["title"] # 首个非标题段落作为标题
|
|
assert "used_styles" in parsed["style_hints"]
|
|
|
|
|
|
def test_extract_docx_bad_bytes():
|
|
with pytest.raises(ParsingError):
|
|
extract_docx(b"this is not a docx file at all")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# pdf
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _make_pdf_bytes(pages: int = 2) -> bytes:
|
|
import pymupdf
|
|
|
|
doc = pymupdf.open()
|
|
for i in range(pages):
|
|
page = doc.new_page()
|
|
page.insert_text((72, 72), f"Page {i + 1} content line")
|
|
doc.set_toc([[1, "第一章 概述", 1], [2, "第一节 背景", 1]])
|
|
data = doc.tobytes()
|
|
doc.close()
|
|
return data
|
|
|
|
|
|
def test_extract_pdf_outline_and_text():
|
|
parsed = extract_pdf(_make_pdf_bytes())
|
|
assert [o["text"] for o in parsed["outline"]] == ["第一章 概述", "第一节 背景"]
|
|
assert any("Page 1" in p for p in parsed["paragraphs"])
|
|
assert parsed["title"] == "第一章 概述"
|
|
|
|
|
|
def test_extract_pdf_no_page_limit():
|
|
# 2026-09-04 手工测试：PDF 不再设页数上限，3 页应正常解析
|
|
parsed = extract_pdf(_make_pdf_bytes(pages=3))
|
|
assert len(parsed["paragraphs"]) >= 1
|
|
|
|
|
|
def test_extract_pdf_bad_bytes():
|
|
with pytest.raises(ParsingError):
|
|
extract_pdf(b"not a pdf")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 文本文件
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_extract_text_utf8():
|
|
parsed = extract_text_file("第一段内容。\n\n第二段内容。".encode("utf-8"))
|
|
assert parsed["paragraphs"] == ["第一段内容。", "第二段内容。"]
|
|
assert parsed["title"] == "第一段内容。"
|
|
|
|
|
|
def test_extract_text_gb18030_fallback():
|
|
# "中文" 的 gb18030 编码
|
|
parsed = extract_text_file("中文内容测试".encode("gb18030"))
|
|
assert parsed["paragraphs"] == ["中文内容测试"]
|
|
|
|
|
|
def test_extract_dispatch_by_extension():
|
|
assert "outline" in extract(b"plain text", "a.txt")
|
|
assert "outline" in extract(b"plain text", "noext")
|
|
with pytest.raises(ParsingError):
|
|
extract(b"xx", "a.pdf")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 大纲派生
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def test_outline_to_markdown():
|
|
md = outline_to_markdown([{"level": 1, "text": "总则"}, {"level": 3, "text": "细则"}])
|
|
assert md == "# 总则\n### 细则"
|
|
|
|
|
|
def test_outline_chunks_short():
|
|
outline = [{"level": 1, "text": "总则"}]
|
|
chunks = outline_to_embedding_chunks(outline)
|
|
assert chunks == ["# 总则"]
|
|
|
|
|
|
def test_outline_chunks_long_split_limited():
|
|
# 构造超长大纲：10 个一级标题，每个 900 字符 → 全长 > 7500
|
|
outline = [{"level": 1, "text": f"第{i}章" + "x" * 900} for i in range(10)]
|
|
chunks = outline_to_embedding_chunks(outline)
|
|
assert 1 <= len(chunks) <= 3 # 不超过 MAX_OUTLINE_CHUNKS
|
|
assert all(len(c) <= 7600 for c in chunks)
|
|
|
|
|
|
def test_outline_chunks_empty():
|
|
assert outline_to_embedding_chunks([]) == []
|