forked from admin/PlantAI-rag-new
Merge branch 'debug/backend1' into feature/bioflow-conversation-orchestration
This commit is contained in:
@@ -16,6 +16,7 @@ from src.rag.normalized_release import (
|
||||
)
|
||||
|
||||
from .schemas import EvidenceItem, RetrievalHypothesis
|
||||
from .score_gate import min_source_score
|
||||
|
||||
ALLOWED_QDRANT_COLLECTIONS = {
|
||||
"plant_srt_direct_v1_bge_m3_20260702",
|
||||
@@ -203,11 +204,14 @@ class DirectFlowRetriever:
|
||||
def qdrant_search(self, query: str, limit: int) -> List[EvidenceItem]:
|
||||
try:
|
||||
vector = self.embed(query)
|
||||
# 缺陷 #28（2026-09-11）：相关性下限双保险（与 pipeline
|
||||
# _select_graph_balanced 同源同值，见 score_gate.py）。
|
||||
response = self.qdrant.query_points(
|
||||
collection_name=self.collection,
|
||||
query=vector,
|
||||
limit=limit,
|
||||
with_payload=True,
|
||||
score_threshold=min_source_score(),
|
||||
)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
@@ -43,6 +43,7 @@ from .question_decomposition import QuestionDecompositionPlanner
|
||||
from .query_builder import QueryBuilder
|
||||
from .reranker import BGEReranker
|
||||
from .schemas import EvidenceItem, HypothesisEntity, RetrievalHypothesis
|
||||
from .score_gate import min_source_score
|
||||
from .soft_gate_policy import (
|
||||
LEGACY,
|
||||
doi_from_paper_id,
|
||||
@@ -3317,6 +3318,14 @@ class QAV1Pipeline:
|
||||
top_k_for_llm: int,
|
||||
graph_min_evidence: int,
|
||||
) -> tuple[List[EvidenceItem], Dict[str, Any]]:
|
||||
# 缺陷 #28（2026-09-11）：相关性下限。低于下限的候选不入选；
|
||||
# 全部低于下限 -> 返回空（前端呈现未命中，与 Qdrant
|
||||
# score_threshold 双保险）。阈值见 score_gate.py（env 可覆盖）。
|
||||
_min_score = min_source_score()
|
||||
_below_min = len(ranked) - sum(
|
||||
1 for item in ranked if item.final_score >= _min_score
|
||||
) # 被下限过滤的候选数（审计用，见 return 处）
|
||||
ranked = [item for item in ranked if item.final_score >= _min_score]
|
||||
graph_items = [item for item in ranked if cls._is_graph_item(item)]
|
||||
other_items = [item for item in ranked if not cls._is_graph_item(item)]
|
||||
selected: List[EvidenceItem] = []
|
||||
@@ -3365,6 +3374,8 @@ class QAV1Pipeline:
|
||||
"graph_selected": graph_selected,
|
||||
"non_graph_selected": len(selected) - graph_selected,
|
||||
"graph_quota_satisfied": graph_selected >= graph_min_evidence,
|
||||
"min_source_score": _min_score,
|
||||
"below_min_source_score": _below_min,
|
||||
"note": "Graph quota is enforced after BGE rerank. If fewer graph candidates exist, all available graph candidates are retained and the deficit is reported.",
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""score_gate.py —— 检索相关性下限（缺陷 #28，2026-09-11）。
|
||||
|
||||
语义：候选证据的最终得分（EvidenceItem.final_score）低于下限即视为
|
||||
「与问题无关」，不得进入 evidence_top30 / sources。
|
||||
|
||||
证据与阈值依据（2026-09-11 实测）：
|
||||
- 无关问题「景甜是谁」实测 30 条 final_score 全部 0.19-0.20（纯植物学
|
||||
无关文献）；正常命中（DISCOVERY.md §4.1 金标样本）vector_score=0.714、
|
||||
rerank_score=0.989、final_score=0.982。
|
||||
- BGE rerank 路径 final_score = 0.8*bge(sigmoid) + 0.2*rrf/max_rrf；
|
||||
bge 是 sigmoid 概率，0.20 垃圾分对应 raw logit ≈ -1.4，0.25 对应 -1.1，
|
||||
0.30 对应 -0.85，正常命中的 sigmoid 通常 >0.9。
|
||||
- Qdrant collection distance=Cosine（score ∈ [-1,1]；bge-m3 文本嵌入对
|
||||
随机中文问题的 top-k 得分通常 <0.2）。
|
||||
|
||||
默认 0.25：与垃圾实测带（0.19-0.20）留 ~25% 余量，又显著低于正常命中
|
||||
（>0.7），同时为 0.2~0.3 边缘问题保留再排序机会（阈值过滤器只是预筛，
|
||||
被过滤者已低于下限，保留也无意义）。
|
||||
env `QA_MIN_SOURCE_SCORE` 可覆盖（0-1，非法值回落默认），供复测按
|
||||
acceptance-r4 §8.3 授权临时调参。
|
||||
|
||||
本模块无第三方依赖、无 IO，可被 v16_retriever / direct_flow_retriever /
|
||||
pipeline 直接 import（编译期与运行期均安全）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
# 相关性下限（与 Qdrant score_threshold 同源同值，双保险）
|
||||
DEFAULT_MIN_SOURCE_SCORE = 0.25
|
||||
_ENV_KEY = "QA_MIN_SOURCE_SCORE"
|
||||
|
||||
|
||||
def _env_override() -> float:
|
||||
"""env 覆盖：QA_MIN_SOURCE_SCORE ∈ [0,1] 才生效，否则回落默认。"""
|
||||
raw = os.environ.get(_ENV_KEY, "").strip()
|
||||
if not raw:
|
||||
return DEFAULT_MIN_SOURCE_SCORE
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return DEFAULT_MIN_SOURCE_SCORE
|
||||
if not (0.0 <= value <= 1.0):
|
||||
return DEFAULT_MIN_SOURCE_SCORE
|
||||
return value
|
||||
|
||||
|
||||
def min_source_score() -> float:
|
||||
"""当前生效的相关性下限（模块导入期缓存 env 值，与部署环境一致）。"""
|
||||
return min_source_score.value
|
||||
|
||||
|
||||
min_source_score.value = _env_override() # type: ignore[attr-defined]
|
||||
@@ -0,0 +1,98 @@
|
||||
"""test_score_gate.py —— 相关性下限选择单测（缺陷 #28，acceptance-r4 §2.4）。
|
||||
|
||||
运行（venv 内，源码树）：
|
||||
cd agent/qa/src/rag/qa_v1_1_73
|
||||
python -m pytest test_score_gate.py -v
|
||||
（编译期 compile_qa.sh 会把全部 test_*.py 一并编进 .so 树，部署时不冲突；
|
||||
正式单测都在源码树跑。）
|
||||
|
||||
覆盖：
|
||||
- 30 条全部低于下限 -> 选入 0（acceptance 桩）
|
||||
- 5 高 + 25 低 -> 仅选入 5（遵循现有配额逻辑）
|
||||
- graph 配额不受影响；低于下限的 graph 候选不入选
|
||||
- env 覆盖 QA_MIN_SOURCE_SCORE 生效与非法值回落
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# 本文件位于 agent/qa/src/rag/qa_v1_1_73/，parents[2] = agent/qa/src
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from rag.qa_v1_1_73 import score_gate # noqa: E402
|
||||
from rag.qa_v1_1_73.schemas import EvidenceItem # noqa: E402
|
||||
from rag.qa_v1_1_73.pipeline import QAV1Pipeline # noqa: E402
|
||||
|
||||
|
||||
def _item(idx: int, *, final: float, graph: bool = False, key: str = "") -> EvidenceItem:
|
||||
return EvidenceItem(
|
||||
evidence_id=f"E{idx}",
|
||||
kb="V16",
|
||||
source=f"src{idx}",
|
||||
relation="has",
|
||||
target=f"tgt{idx}",
|
||||
evidence=f"evidence body {idx}",
|
||||
relation_id=(key or f"rid-{idx}"),
|
||||
retrieval_methods=(["neo4j_bounded_expand"] if graph else ["qdrant_vector"]),
|
||||
final_score=final,
|
||||
)
|
||||
|
||||
|
||||
def _select(items, *, top_k=30, graph_min=0):
|
||||
selected, audit = QAV1Pipeline._select_graph_balanced(
|
||||
items, top_k_for_llm=top_k, graph_min_evidence=graph_min
|
||||
)
|
||||
return selected, audit
|
||||
|
||||
|
||||
def test_all_below_threshold_selects_zero():
|
||||
"""acceptance §2.4：30 条全部低于下限 -> 选入 0。"""
|
||||
items = [_item(i, final=0.20) for i in range(30)]
|
||||
selected, audit = _select(items)
|
||||
assert selected == []
|
||||
assert audit["policy"].startswith("graph_balanced_top30")
|
||||
assert audit["below_min_source_score"] == 30
|
||||
assert audit["min_source_score"] == score_gate.min_source_score()
|
||||
|
||||
|
||||
def test_five_high_twentyfive_low_selects_only_five():
|
||||
"""acceptance §2.4：5 高 + 25 低 -> 仅选入 5（沿用现有配额与 rerank 序）。"""
|
||||
items = [_item(i, final=0.90 - i * 0.01) for i in range(5)]
|
||||
items += [_item(5 + i, final=0.20) for i in range(25)]
|
||||
selected, audit = _select(items)
|
||||
assert len(selected) == 5
|
||||
assert {item.evidence_id for item in selected} == {f"E{i}" for i in range(5)}
|
||||
# 顺序保持 rerank 序
|
||||
assert [item.evidence_id for item in selected] == [f"E{i}" for i in range(5)]
|
||||
assert audit["below_min_source_score"] == 25
|
||||
|
||||
|
||||
def test_below_threshold_graph_candidates_excluded():
|
||||
"""低分 graph 候选不占用 graph 配额（配额逻辑语义不变）。"""
|
||||
items = [
|
||||
_item(1, final=0.30, graph=True),
|
||||
_item(2, final=0.21, graph=True),
|
||||
_item(3, final=0.95, graph=False),
|
||||
_item(4, final=0.19, graph=False),
|
||||
]
|
||||
selected, audit = _select(items, top_k=30, graph_min=1)
|
||||
assert [item.evidence_id for item in selected] == ["E1", "E3"]
|
||||
assert audit["graph_selected"] == 1
|
||||
assert audit["below_min_source_score"] == 2
|
||||
|
||||
|
||||
def test_env_override_and_invalid_fallback(monkeypatch):
|
||||
"""env 解析函数行为：合法值生效；非法值/越界回落默认。"""
|
||||
monkeypatch.setenv("QA_MIN_SOURCE_SCORE", "0.35")
|
||||
assert score_gate._env_override() == 0.35
|
||||
monkeypatch.setenv("QA_MIN_SOURCE_SCORE", "abc")
|
||||
assert score_gate._env_override() == score_gate.DEFAULT_MIN_SOURCE_SCORE
|
||||
monkeypatch.setenv("QA_MIN_SOURCE_SCORE", "1.5")
|
||||
assert score_gate._env_override() == score_gate.DEFAULT_MIN_SOURCE_SCORE
|
||||
monkeypatch.delenv("QA_MIN_SOURCE_SCORE", raising=False)
|
||||
# 生效值（value 属性）驱动 _select_graph_balanced 的过滤
|
||||
monkeypatch.setattr(score_gate.min_source_score, "value", 0.35)
|
||||
items = [_item(i, final=0.30) for i in range(5)]
|
||||
selected, _ = _select(items)
|
||||
assert selected == []
|
||||
@@ -9,6 +9,7 @@ from typing import Any, Dict, List
|
||||
import requests
|
||||
from neo4j import GraphDatabase
|
||||
from qdrant_client import QdrantClient
|
||||
from .score_gate import min_source_score
|
||||
|
||||
|
||||
TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9_.-]{2,}|[\u4e00-\u9fff]{2,}")
|
||||
@@ -100,11 +101,15 @@ class V16IndependentRetriever:
|
||||
self, question: str, limit: int, include_candidate: bool
|
||||
) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
# 缺陷 #28（2026-09-11）：相关性下限双保险（与 pipeline
|
||||
# _select_graph_balanced 同源同值）。collection distance=Cosine，
|
||||
# 阈值语义 = 余弦相似度下限；低于下限的点直接不返回。
|
||||
response = self.qdrant.query_points(
|
||||
collection_name=self.collection,
|
||||
query=self.embed(question),
|
||||
limit=limit * 2,
|
||||
with_payload=True,
|
||||
score_threshold=min_source_score(),
|
||||
)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
@@ -18,7 +18,11 @@ route(text, has_attachments, has_active_artifact) -> "qa" | "docgen" | "edit" |
|
||||
|
||||
### 3. docgen（写文档意图）
|
||||
条件：text 含任一 WRITE_VERB 且含任一 DOC_TYPE
|
||||
- WRITE_VERB = [写, 撰写, 拟, 起草, 编制, 生成, 编写, 拟定, 草拟, 制定]
|
||||
- WRITE_VERB = [写, 撰写, 拟, 起草, 编制, 生成, 编写, 拟定, 草拟, 制定, 总结, 概括, 归纳]（r4 #52 新增后三词）
|
||||
- 动词自指保护（r4 #52 新增）：规则 3 判定时，若 text 命中的 DOC_TYPE 恰好等于命中的 WRITE_VERB 本身
|
||||
（如「总结一下植物光合作用的过程」——「总结」既是动词又是文档类型词），不算写文档意图，
|
||||
继续向下判定。「写一篇总结」（动词=写，类型=总结）与「总结这篇论文」（动词=总结，类型=论文）
|
||||
不受影响。
|
||||
- DOC_TYPE = [论文, 报告, 通知, 通告, 公告, 请示, 批复, 函, 申请书, 申请, 申报书, 标书, 合同, 协议, 方案, 计划, 总结, 汇报, 讲话稿, 致辞, 证明, 纪要, 简报, 材料, 说明书, 指南, 文档, 申报材料, 工作总结, 调研报告, 可行性报告, 项目书, 策划书, 演讲稿, 新闻稿, 文案, 简历, 制度, 规定, 办法, 细则, 章程, 声明, 邀请函, 感谢信, 慰问信, 介绍信, 推荐信, 倡议书, 述职报告, 开题报告, 结题报告, 实验报告, 分析报告, 会议纪要]
|
||||
- 明确排除（不属于 DOC_TYPE，防误判）：介绍, 内容, 代码, 诗, 作业, 问题, 建议, 想法
|
||||
|
||||
@@ -28,7 +32,7 @@ route(text, has_attachments, has_active_artifact) -> "qa" | "docgen" | "edit" |
|
||||
|
||||
### 5. 其余一律 "qa"（漏判安全原则：不命中绝不进 docgen）
|
||||
|
||||
## 用例表（30 条，前后端实现与测试共用同一张表）
|
||||
## 用例表（34 条，前后端实现与测试共用同一张表）
|
||||
|
||||
| # | 输入 text | 附件 | 活动文档 | 期望 |
|
||||
|---|---|---|---|---|
|
||||
@@ -62,6 +66,10 @@ route(text, has_attachments, has_active_artifact) -> "qa" | "docgen" | "edit" |
|
||||
| 28 | 帮我写个会议通知模板 | 否 | 是 | docgen |
|
||||
| 29 | 给这个通知加个结尾 | 否 | 是 | edit |
|
||||
| 30 | 增加一段关于风险的描述 | 否 | 是 | edit |
|
||||
| 31 | 帮我总结论文 | 是 | 否 | docgen |
|
||||
| 32 | 总结一下植物光合作用的过程 | 否 | 否 | qa |
|
||||
| 33 | 概括一下这个通知 | 是 | 是 | docgen |
|
||||
| 34 | 归纳这篇文章 | 是 | 否 | docgen |
|
||||
|
||||
## 一致性要求
|
||||
- dev1 的 sdk/app/services/docgen/intent.py 与 dev3 的 ui/src/intent.js 必须按本表逐条通过
|
||||
|
||||
@@ -983,7 +983,11 @@ async def export_artifact(
|
||||
html, _rendered_css = render_html(
|
||||
markdown, title=title, key=key, letterhead=letterhead
|
||||
)
|
||||
data = await docx_from_html_document(html, title)
|
||||
# 缺陷 #47a（2026-09-11）：css 一并传给 DOCX 转换，
|
||||
# 红头/标题样式与预览/PDF 同源；frozen_css 优先（同 PDF 路径）
|
||||
data = await docx_from_html_document(
|
||||
html, title, css=frozen_css or _rendered_css
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"docgen: HTML→DOCX 转换失败（%s），回退 md2docx 旧路径", exc
|
||||
|
||||
@@ -108,7 +108,7 @@ def _reset_mocks(monkeypatch):
|
||||
async def rebuild_ok():
|
||||
return {"rebuilt": True}
|
||||
|
||||
async def docx_html_ok(html: str, title: str):
|
||||
async def docx_html_ok(html: str, title: str, css: str = ""):
|
||||
return b"DOCX-HTML-BYTES"
|
||||
|
||||
def render_ok(markdown, *, title, key="document", letterhead=""):
|
||||
|
||||
@@ -31,13 +31,17 @@ async def pdf_from_markdown(markdown: str, title: str) -> bytes:
|
||||
return await anyio.to_thread.run_sync(_pdf_bytes, markdown, title)
|
||||
|
||||
|
||||
async def docx_from_html_document(html: str, title: str) -> bytes:
|
||||
"""HTML → docx（缺陷 #47a，2026-09-10）：与 converter 其它函数同签名，
|
||||
html2docx 是 CPU 同步转换，to_thread 避免阻塞事件循环。"""
|
||||
async def docx_from_html_document(html: str, title: str, css: str = "") -> bytes:
|
||||
"""HTML → docx（缺陷 #47a，2026-09-10/11）：与 converter 其它函数同签名，
|
||||
html2docx 是 CPU 同步转换，to_thread 避免阻塞事件循环。
|
||||
|
||||
css：render_html 返回的模板 CSS（缺陷 #47a 同源解析，2026-09-11）。
|
||||
为空时 html2docx 只解析 html 内嵌 <style> 并回退常量（历史行为不变）。
|
||||
"""
|
||||
from .html2docx import docx_from_html_document as _sync
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
out = _sync(html, title=title, out_path=Path(tmp) / "document.docx")
|
||||
out = _sync(html, title=title, out_path=Path(tmp) / "document.docx", css=css)
|
||||
return out.read_bytes()
|
||||
|
||||
|
||||
|
||||
@@ -35,12 +35,16 @@ from docx.oxml import OxmlElement
|
||||
from docx.oxml.ns import qn
|
||||
from docx.shared import Inches, Pt, RGBColor
|
||||
|
||||
from . import html_css # 缺陷 #47a：模板 CSS 解析（预览同源）
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 字体：正文宋体 / 标题黑体（与 md2docx 一致；Windows 中文环境标准）
|
||||
_FONT_BODY = "宋体"
|
||||
_FONT_HEADING = "黑体"
|
||||
|
||||
# 缺陷 #47a（2026-09-11）：红头/标题样式从 render_html 产物（<style> 内嵌
|
||||
# CSS）解析，与预览/PDF 同源；以下仅为解析失败时的回退常量（保留 r3 行为）。
|
||||
_HEADING_SIZES = {1: 22, 2: 16, 3: 14}
|
||||
_LETTERHEAD_SIZE = 22.0
|
||||
_IMAGE_MAX_WIDTH_IN = 6.0 # A4 正文可用宽度约 6 英寸
|
||||
@@ -60,6 +64,9 @@ _LEAF_BLOCKS = {
|
||||
# 遍历时整体跳过的标签
|
||||
_SKIP_TAGS = {"script", "style", "meta", "link", "title", "head"}
|
||||
|
||||
# 当前文档解析出的标题字号（docx_from_html_document 设置；单线程转换，无并发问题）
|
||||
_ACTIVE_HEADING_SIZES: dict[int, float] = dict(_HEADING_SIZES)
|
||||
|
||||
|
||||
def _set_run_font(run, font_name: str, size_pt: float | None = None,
|
||||
bold: bool | None = None, italic: bool | None = None,
|
||||
@@ -192,7 +199,7 @@ def _add_rich_block(doc: Document, node: Tag, list_level: int = 0) -> None:
|
||||
if level:
|
||||
para = doc.add_heading("", level=level)
|
||||
run = para.add_run(node.get_text(" ", strip=True))
|
||||
_set_run_font(run, _FONT_HEADING, _HEADING_SIZES[level], bold=True)
|
||||
_set_run_font(run, _FONT_HEADING, _ACTIVE_HEADING_SIZES.get(level, _HEADING_SIZES[level]), bold=True)
|
||||
para.paragraph_format.space_before = Pt(10)
|
||||
return
|
||||
|
||||
@@ -334,16 +341,24 @@ def _add_image(doc: Document, img: Tag) -> None:
|
||||
_set_run_font(run, _FONT_BODY, 10.5)
|
||||
|
||||
|
||||
def docx_from_html_document(html: str, title: str = "", out_path: str | Path | None = None) -> Path:
|
||||
def docx_from_html_document(html: str, title: str = "", out_path: str | Path | None = None, *, css: str = "") -> Path:
|
||||
"""HTML → docx。返回写入路径（out_path 为 None 时按 title 命名到当前目录）。
|
||||
|
||||
- 红头：.doc-letterhead 红字（RGBColor(255,0,0)）居中 22pt 黑体
|
||||
- h1-h3：黑体 22/16/14pt 加粗
|
||||
- 红头：.doc-letterhead 颜色/字号从 render_html 产物解析（css 参数 +
|
||||
html 内嵌 <style>），解析失败回退历史常量（FF0000/22pt）
|
||||
- h1-h3：黑体，字号同样解析优先、回退 22/16/14pt
|
||||
- 表格 Table Grid + 表头灰底；figure/img 内嵌（失败落 alt）
|
||||
- 空 HTML 产出空文档（不抛）
|
||||
"""
|
||||
doc = Document()
|
||||
soup = BeautifulSoup(html or "", "html.parser")
|
||||
# 缺陷 #47a：红头/标题样式同源解析（render_html 产物 -> DOCX 与预览一致）
|
||||
styles = html_css.parse_css_styles(html)
|
||||
if css:
|
||||
styles = html_css.merge_css_text(styles, css)
|
||||
letterhead_color, letterhead_size = html_css.letterhead_style(styles)
|
||||
global _ACTIVE_HEADING_SIZES
|
||||
_ACTIVE_HEADING_SIZES = html_css.heading_sizes(styles)
|
||||
|
||||
# 标题：<title> 优先（render_html 已回填 {{TITLE}}）
|
||||
if not title:
|
||||
@@ -361,8 +376,8 @@ def docx_from_html_document(html: str, title: str = "", out_path: str | Path | N
|
||||
para = doc.add_paragraph()
|
||||
para.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||||
run = para.add_run(letterhead)
|
||||
_set_run_font(run, _FONT_HEADING, _LETTERHEAD_SIZE, bold=True,
|
||||
color=RGBColor(255, 0, 0))
|
||||
_set_run_font(run, _FONT_HEADING, letterhead_size, bold=True,
|
||||
color=RGBColor(*letterhead_color))
|
||||
letterhead_el.decompose()
|
||||
|
||||
# 主内容：main 优先；递归下钻包裹容器（doc-body 等），保文档序
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""services/docgen/html_css.py —— 自家 render_html 产物的 CSS 解析（缺陷 #47a，2026-09-11）。
|
||||
|
||||
render_html 返回 (html, css)：html 是模板 wrapper（css 不内联，
|
||||
css 由调用方单独持有——PDF 路径与预览拼装时注入）。html2docx 的输入
|
||||
原本只有 html，缺陷 #47a 后调用方（docgen 路由）把 css 一并传入
|
||||
（converter.docx_from_html_document(html, title, css)），本模块负责：
|
||||
- 解析 html 内嵌 <style> 块（预览注入后的产物形态）；
|
||||
- 合并调用方传入的模板 CSS 文本（后出现者优先，与调用方拼装语义一致）；
|
||||
- 解析不到声明时回退历史硬编码常量（红头 FF0000 22pt、h1/h2/h3 22/16/14pt），
|
||||
保证无样式 HTML / 解析失败的行为与 r3 一致。
|
||||
|
||||
解析只覆盖 CSS 安全子集（块流/字体/颜色，见 templates 各 css 文件头注释）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
# ---- 历史硬编码常量（解析不出声明时的最终回退，行为与 r3 相同）----
|
||||
_LEGACY_LETTERHEAD_COLOR = (255, 0, 0) # FF0000
|
||||
_LEGACY_LETTERHEAD_SIZE_PT = 22.0
|
||||
_LEGACY_HEADING_SIZES_PT = {1: 22.0, 2: 16.0, 3: 14.0}
|
||||
|
||||
_ColorRGB = tuple[int, int, int]
|
||||
|
||||
# rgb(r, g, b) / rgba(...) / #rgb / #rrggbb（容忍空白）
|
||||
_RGB_RE = re.compile(
|
||||
r"rgba?\(\s*(\d{1,3})\s*,\s*(\d{1,3})\s*,\s*(\d{1,3})", re.IGNORECASE
|
||||
)
|
||||
_HEX6_RE = re.compile(r"#([0-9a-fA-F]{6})\b")
|
||||
_HEX3_RE = re.compile(r"#([0-9a-fA-F]{3})\b")
|
||||
|
||||
# 1.2e3 / 12.5 / 12 后接 px|pt|em|rem（匹配声明值中的尺寸）
|
||||
_SIZE_RE = re.compile(
|
||||
r"(\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)\s*(px|pt|em|rem)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _declarations(style: Tag) -> list[dict[str, str]]:
|
||||
"""解析 <style> 文本为声明块列表：{selector, prop: value...}。
|
||||
|
||||
组合选择器（"h3, h4, h5, h6"）按逗号拆分为独立条目，
|
||||
让 h3/h4… 各自可被单独查取（模板 CSS 只含简单元素/类选择器）。
|
||||
"""
|
||||
out: list[dict[str, str]] = []
|
||||
for match in re.finditer(r"([^{}]+)\{([^{}]*)\}", style.get_text() or ""):
|
||||
selectors = [sel.strip() for sel in match.group(1).split(",") if sel.strip()]
|
||||
if not selectors:
|
||||
continue
|
||||
block: dict[str, str] = {}
|
||||
for decl in re.findall(r"([\w-]+)\s*:\s*([^;]+);", match.group(2)):
|
||||
block[decl[0].strip().lower()] = decl[1].strip()
|
||||
if not block:
|
||||
continue
|
||||
for selector in selectors:
|
||||
out.append({"selector": selector, "block": dict(block)})
|
||||
return out
|
||||
|
||||
|
||||
def parse_rgb(value: str) -> _ColorRGB | None:
|
||||
"""CSS 颜色 → (r,g,b)；支持 #rrggbb/#rgb/rgb()/rgba()，其余返回 None。"""
|
||||
if not value:
|
||||
return None
|
||||
m = _HEX6_RE.search(value)
|
||||
if m:
|
||||
h = m.group(1)
|
||||
return (int(h[0:2], 16), int(h[2:4], 16), int(h[4:6], 16))
|
||||
m = _HEX3_RE.search(value)
|
||||
if m:
|
||||
h = m.group(1)
|
||||
return (int(h[0], 16) * 17, int(h[1], 16) * 17, int(h[2], 16) * 17)
|
||||
m = _RGB_RE.search(value)
|
||||
if m:
|
||||
r, g, b = (max(0, min(255, int(c))) for c in m.groups())
|
||||
return (r, g, b)
|
||||
return None
|
||||
|
||||
|
||||
def parse_size_pt(value: str) -> float | None:
|
||||
"""CSS 尺寸 → pt（px 按 96dpi：1px = 0.75pt）；非尺寸返回 None。"""
|
||||
if not value:
|
||||
return None
|
||||
m = _SIZE_RE.search(value)
|
||||
if not m:
|
||||
return None
|
||||
number = float(m.group(1))
|
||||
unit = m.group(2).lower()
|
||||
if unit == "px":
|
||||
return number * 0.75
|
||||
if unit in ("pt", "em", "rem"):
|
||||
return number
|
||||
return None
|
||||
|
||||
|
||||
def parse_css_styles(html: str) -> dict[str, dict[str, str]]:
|
||||
"""HTML 内嵌 <style> 块 → {selector: {prop: value}}。
|
||||
|
||||
多个 <style> 按出现顺序合并，后出现者优先（与浏览器级联一致）。
|
||||
无 <style> 时返回空 dict（调用方回退常量）。
|
||||
"""
|
||||
merged: dict[str, dict[str, str]] = {}
|
||||
if not html:
|
||||
return merged
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for style in soup.find_all("style"):
|
||||
for decl in _declarations(style):
|
||||
block = merged.setdefault(decl["selector"], {})
|
||||
block.update(decl["block"])
|
||||
return merged
|
||||
|
||||
|
||||
def merge_css_text(
|
||||
styles: dict[str, dict[str, str]], css: str
|
||||
) -> dict[str, dict[str, str]]:
|
||||
"""把调用方持有的模板 CSS 文本（render_html 返回的 css）并入解析结果。
|
||||
|
||||
css 文本通常不含 <style> 标签，这里包一层再走同一解析器；
|
||||
后出现者优先（模板 CSS 覆盖 html 内嵌声明，与调用方拼装语义一致）。
|
||||
"""
|
||||
if not css:
|
||||
return styles
|
||||
merged = dict(styles)
|
||||
wrapped = BeautifulSoup(f"<style>{css}</style>", "html.parser")
|
||||
for decl in _declarations(wrapped.style):
|
||||
block = merged.setdefault(decl["selector"], {})
|
||||
block.update(decl["block"])
|
||||
return merged
|
||||
|
||||
|
||||
def letterhead_style(styles: dict[str, dict[str, str]]) -> tuple[_ColorRGB, float]:
|
||||
"""红头样式：(rgb 颜色, 字号 pt)。解析不到时回退历史常量。"""
|
||||
block = styles.get(".doc-letterhead") or {}
|
||||
color = parse_rgb(block.get("color", "")) or _LEGACY_LETTERHEAD_COLOR
|
||||
size = parse_size_pt(block.get("font-size", ""))
|
||||
if size is None:
|
||||
size = _LEGACY_LETTERHEAD_SIZE_PT
|
||||
return color, size
|
||||
|
||||
|
||||
def heading_sizes(styles: dict[str, dict[str, str]]) -> dict[int, float]:
|
||||
"""h1-h3 字号（pt）：解析 h1.doc-title、h2、h3 声明；缺失回落历史常量。"""
|
||||
h1 = styles.get("h1.doc-title") or styles.get("h1") or {}
|
||||
h2 = styles.get("h2") or {}
|
||||
h3 = styles.get("h3") or {}
|
||||
parsed = {
|
||||
1: parse_size_pt(h1.get("font-size", "")),
|
||||
2: parse_size_pt(h2.get("font-size", "")),
|
||||
3: parse_size_pt(h3.get("font-size", "")),
|
||||
}
|
||||
return {
|
||||
level: parsed[level] if parsed[level] is not None else _LEGACY_HEADING_SIZES_PT[level]
|
||||
for level in (1, 2, 3)
|
||||
}
|
||||
@@ -20,7 +20,8 @@ FORMAT_WORD = ["pdf", "docx", "doc", "word"] # 英文不区分大小写（conta
|
||||
EDIT_WORD = ["改", "修改", "调整", "删", "删除", "加一段", "补充", "润色",
|
||||
"缩写", "扩写", "重写", "改写", "优化", "加个", "增加一段"]
|
||||
|
||||
WRITE_VERB = ["写", "撰写", "拟", "起草", "编制", "生成", "编写", "拟定", "草拟", "制定"]
|
||||
WRITE_VERB = ["写", "撰写", "拟", "起草", "编制", "生成", "编写", "拟定", "草拟", "制定",
|
||||
"总结", "概括", "归纳"] # r4 #52（2026-09-11）
|
||||
|
||||
DOC_TYPE = [
|
||||
"论文", "报告", "通知", "通告", "公告", "请示", "批复", "函", "申请书",
|
||||
@@ -71,8 +72,16 @@ def route(text: str, has_attachments: bool, has_active_artifact: bool) -> str:
|
||||
return "edit"
|
||||
|
||||
# 3. docgen：写类动词 + 文档类型词
|
||||
if _contains_any(text, WRITE_VERB) and _has_doc_type(text):
|
||||
return "docgen"
|
||||
# r4 #52（2026-09-11）动词自指保护：当唯一命中的 DOC_TYPE 恰好等于
|
||||
# 命中的动词本身（如「总结一下植物光合作用的过程」——「总结」既是动词
|
||||
# 又是文档类型词），不视为写文档意图（继续向下判定）。「写一篇总结」
|
||||
# （动词=写，类型=总结）与「总结这篇论文」（动词=总结，类型=论文）
|
||||
# 不受影响。规则详见 docs/team/cases_intent.md 用例 31-34。
|
||||
_matched_verbs = [w for w in WRITE_VERB if w in text]
|
||||
_matched_types = [w for w in DOC_TYPE if w in text]
|
||||
if _matched_verbs and _matched_types and _has_doc_type(text):
|
||||
if set(_matched_types) != set(_matched_verbs):
|
||||
return "docgen"
|
||||
|
||||
# 4. docgen：附件信号（拖入文件 + 写类动词）
|
||||
if has_attachments and _contains_any(text, WRITE_VERB):
|
||||
|
||||
@@ -20,6 +20,16 @@ def pick(obj: dict, *names: str, default=None):
|
||||
# ---------------------------------------------------------------------------
|
||||
# §8 Source 归一化（缺失字段键必须存在，值为 null）
|
||||
# ---------------------------------------------------------------------------
|
||||
# title 回填优先级（缺陷 #27，2026-09-11）：title/paper_title 缺失时，
|
||||
# 依次从以下字段取可读文本（论文标题优先于实体名），全部缺失才保持 null。
|
||||
# 排除 content/evidence/text/snippet 正文字段——正文片段不是标题。
|
||||
_TITLE_FALLBACK_TOPLEVEL = (
|
||||
"publication_title", "article_title", "journal_title", "name",
|
||||
)
|
||||
_TITLE_FALLBACK_METADATA = (
|
||||
"paper_title", "publication_title", "article_title", "journal_title",
|
||||
"source_entity", "name", "source",
|
||||
)
|
||||
|
||||
def normalize_source(item: dict[str, Any] | None) -> dict[str, Any]:
|
||||
"""单个 source item → API.md §8 形态（所有键存在，缺失为 null）。
|
||||
@@ -41,6 +51,23 @@ def normalize_source(item: dict[str, Any] | None) -> dict[str, Any]:
|
||||
|
||||
source = pick(item, "source") # 断言主体实体名（上游语义）
|
||||
title = pick(item, "title", "paper_title")
|
||||
# 缺陷 #27：title 为空时按固定优先级回填——
|
||||
# 顶层标题类字段 → metadata 标题类字段 → 顶层 source（断言主体实体名）。
|
||||
# 正文字段绝不参与（正文片段不是标题）。
|
||||
if not title:
|
||||
for _key in _TITLE_FALLBACK_TOPLEVEL:
|
||||
_candidate = pick(item, _key)
|
||||
if _candidate:
|
||||
title = _candidate
|
||||
break
|
||||
if not title and isinstance(pick(item, "metadata"), dict):
|
||||
for _key in _TITLE_FALLBACK_METADATA:
|
||||
_candidate = pick(item["metadata"], _key)
|
||||
if _candidate:
|
||||
title = _candidate
|
||||
break
|
||||
if not title:
|
||||
title = source
|
||||
|
||||
doi = pick(item, "doi")
|
||||
url = pick(item, "url", "doi_url")
|
||||
|
||||
@@ -127,6 +127,96 @@ def test_bold_italic_code_runs(tmp_path):
|
||||
assert code_found
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 缺陷 #47a（2026-09-11）：红头/标题样式从 render_html 产物解析（css 参数）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
HTML_OFFICIAL_STYLED = """<!DOCTYPE html><html><head><meta charset="utf-8">
|
||||
<title>红头公文</title></head><body>
|
||||
<div class="doc-cover"><div class="doc-letterhead">某某单位文件</div></div>
|
||||
<main><div class="doc-body">
|
||||
<h1 class="doc-title">关于加强安全生产的通知</h1>
|
||||
<h2>一、总体要求</h2>
|
||||
<h3>1.1 具体要求</h3>
|
||||
<p>正文内容。</p>
|
||||
</div></main>
|
||||
</body></html>"""
|
||||
|
||||
OFFICIAL_CSS = """
|
||||
.doc-letterhead { color: #c00000; font-size: 30px; font-weight: bold; }
|
||||
h1.doc-title { font-size: 22px; }
|
||||
h2 { font-size: 16px; }
|
||||
h3, h4, h5, h6 { font-size: 15px; }
|
||||
"""
|
||||
|
||||
|
||||
def test_letterhead_style_from_css_param(tmp_path):
|
||||
"""红头样式从模板 CSS 解析：#c00000 系颜色、30px→22.5pt 字号。"""
|
||||
out = docx_from_html_document(
|
||||
HTML_OFFICIAL_STYLED, title="红头公文",
|
||||
out_path=tmp_path / "a.docx", css=OFFICIAL_CSS,
|
||||
)
|
||||
doc = Document(str(out))
|
||||
head = doc.paragraphs[0]
|
||||
assert "某某单位文件" in head.text
|
||||
assert head.alignment.name == "CENTER"
|
||||
run = head.runs[0]
|
||||
assert str(_first_run_color(run)) == "C00000"
|
||||
assert run.font.size is not None and abs(run.font.size.pt - 22.5) < 0.01
|
||||
assert run.font.bold is True
|
||||
|
||||
|
||||
def test_heading_sizes_from_css_param(tmp_path):
|
||||
"""标题字号从模板 CSS 解析：h1 22px→16.5pt、h2 16px→12pt、h3 15px→11.25pt。"""
|
||||
out = docx_from_html_document(
|
||||
HTML_OFFICIAL_STYLED, title="红头公文",
|
||||
out_path=tmp_path / "b.docx", css=OFFICIAL_CSS,
|
||||
)
|
||||
doc = Document(str(out))
|
||||
sizes = {}
|
||||
for p in doc.paragraphs:
|
||||
if p.style.name.startswith("Heading") and p.runs:
|
||||
sizes[p.style.name] = p.runs[0].font.size.pt
|
||||
# DOCX 字号按半点（half-point）量化：11.25pt 落 22 半点 = 11.0pt
|
||||
assert abs(sizes.get("Heading 1") - 16.5) < 0.01
|
||||
assert abs(sizes.get("Heading 2") - 12.0) < 0.01
|
||||
assert abs(sizes.get("Heading 3") - 11.25) <= 0.26
|
||||
|
||||
|
||||
def test_no_style_falls_back_to_legacy_constants(tmp_path):
|
||||
"""无样式 HTML（不传 css）→ 回退 r3 常量（FF0000/22pt 红头，22/16/14pt 标题）。"""
|
||||
out = docx_from_html_document(HTML_SAMPLE, title="测试公文",
|
||||
out_path=tmp_path / "c.docx")
|
||||
doc = Document(str(out))
|
||||
head = doc.paragraphs[0]
|
||||
run = head.runs[0]
|
||||
assert str(_first_run_color(run)) == "FF0000"
|
||||
assert run.font.size.pt == 22.0
|
||||
sizes = {}
|
||||
for p in doc.paragraphs:
|
||||
if p.style.name.startswith("Heading") and p.runs:
|
||||
sizes[p.style.name] = p.runs[0].font.size.pt
|
||||
assert sizes.get("Heading 1") == 22.0
|
||||
assert sizes.get("Heading 2") == 16.0
|
||||
assert sizes.get("Heading 3") == 14.0
|
||||
|
||||
|
||||
def test_report_template_grey_letterhead_not_red(tmp_path):
|
||||
"""非红头模板（report：灰 #333、26px）不误渲染红头。"""
|
||||
report_css = """
|
||||
.doc-letterhead { color: #333333; font-size: 26px; font-weight: bold; }
|
||||
h1.doc-title { font-size: 22px; }
|
||||
"""
|
||||
out = docx_from_html_document(
|
||||
HTML_OFFICIAL_STYLED, title="报告",
|
||||
out_path=tmp_path / "d.docx", css=report_css,
|
||||
)
|
||||
doc = Document(str(out))
|
||||
run = doc.paragraphs[0].runs[0]
|
||||
assert str(_first_run_color(run)) == "333333"
|
||||
assert abs(run.font.size.pt - 19.5) < 0.01 # 26px * 0.75
|
||||
|
||||
|
||||
# r3-47-01 复现样例：真实 render_html 产物形态——
|
||||
# <main><div class="doc-body">…</div></main> 包裹层 + 嵌套 section
|
||||
HTML_WRAPPED = """<!DOCTYPE html><html><head><meta charset="utf-8">
|
||||
|
||||
@@ -47,6 +47,11 @@ CASES = [
|
||||
("帮我写个会议通知模板", False, True, "docgen"),
|
||||
("给这个通知加个结尾", False, True, "edit"),
|
||||
("增加一段关于风险的描述", False, True, "edit"),
|
||||
# r4 #52（2026-09-11）新增：总结/概括/归纳 三动词 + 动词自指保护
|
||||
("帮我总结论文", True, False, "docgen"),
|
||||
("总结一下植物光合作用的过程", False, False, "qa"),
|
||||
("概括一下这个通知", True, True, "docgen"),
|
||||
("归纳这篇文章", True, False, "docgen"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -84,6 +84,52 @@ def test_normalize_source_only_doi_derives_url():
|
||||
assert src["metadata"] == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# §8 title 回填（缺陷 #27，2026-09-11）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_normalize_source_title_fill_from_source_entity():
|
||||
"""title 为空 -> 回填 source（断言主体实体名），正文片段绝不参与。"""
|
||||
src = normalize_source({
|
||||
"source": "OsSPL14",
|
||||
"title": "",
|
||||
"evidence": "## 正文片段开头（不是标题）",
|
||||
"paper_id": "P1",
|
||||
})
|
||||
assert src["title"] == "OsSPL14"
|
||||
# title 即 source 回填：两者相等，source_entity 不落 metadata（不重复）
|
||||
assert src["metadata"].get("source_entity") is None
|
||||
assert src["paper_id"] == "P1"
|
||||
|
||||
|
||||
def test_normalize_source_title_fill_from_metadata():
|
||||
"""title/paper_title 均空 -> metadata 标题类字段回填。"""
|
||||
src = normalize_source({
|
||||
"source": "entity",
|
||||
"metadata": {"paper_title": "水稻抗病基因研究进展", "source_entity": "entity"},
|
||||
})
|
||||
assert src["title"] == "水稻抗病基因研究进展"
|
||||
|
||||
|
||||
def test_normalize_source_title_not_overridden():
|
||||
"""title 非空 -> 不回填（既有语义不变）。"""
|
||||
src = normalize_source({
|
||||
"title": "Real Title",
|
||||
"source": "Entity",
|
||||
"paper_title": "Other",
|
||||
"metadata": {"paper_title": "Meta"},
|
||||
})
|
||||
assert src["title"] == "Real Title"
|
||||
assert src["metadata"]["source_entity"] == "Entity"
|
||||
|
||||
|
||||
def test_normalize_source_title_all_empty_stays_none():
|
||||
"""全空 -> 保持 null（前端有 fallback）。"""
|
||||
src = normalize_source({"content": "snippet only", "paper_id": "P1"})
|
||||
assert src["title"] is None
|
||||
assert src["snippet"] == "snippet only"
|
||||
|
||||
|
||||
def test_normalize_source_alias_chains():
|
||||
src = normalize_source({
|
||||
"content": "snippet text",
|
||||
|
||||
Reference in New Issue
Block a user