jerryandClaude Fable 5
4749f77544
第一批：上传去限制+进度展示、模板HTML预览+按需导出、检索TAB实时更新
第二批：edit护栏移除、导出按钮纯化+file_name落库、编辑失焦还原、
二次预览白底根治（防竞态+渲染兜底）
模板体系：7套公文模板(templates/)+templating渲染器、websearch联网
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
142 lines
4.6 KiB
Python
142 lines
4.6 KiB
Python
"""services/docgen/websearch.py —— 文档模版联网参考（best-effort）。
|
||
|
||
库内无匹配模版时：cn.bing.com 搜索 → 首个结果页 → 提取 h1-h3 大纲。
|
||
严格超时（搜索 8s / 抓取 10s）、单次不重试、任何异常返回 None，
|
||
调用方静默降级到 LLM 大纲（绝不阻断生成）。
|
||
|
||
环境闸：DOCGEN_WEB_TEMPLATE_SEARCH（默认 "1" 启用；显式 "0" 关闭）。
|
||
Cython 约束：仅 dataclass/dict/stdlib+httpx。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import html as _html
|
||
import logging
|
||
import os
|
||
import re
|
||
import urllib.parse
|
||
from dataclasses import dataclass, field
|
||
from typing import Any
|
||
|
||
import httpx
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
TIMEOUT_SEARCH = 8.0
|
||
TIMEOUT_FETCH = 10.0
|
||
MAX_RESULT_BYTES = 512 * 1024
|
||
MIN_HEADINGS = 2
|
||
MAX_HEADINGS = 12
|
||
|
||
_BING_URL = "https://cn.bing.com/search"
|
||
_UA = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||
"(KHTML, like Gecko) Chrome/126.0 Safari/537.36"
|
||
)
|
||
# 2026-09-04 实测 cn.bing.com：<li class="b_algo" ...><h2 ...><a href="...">…
|
||
_ALGO_LINK_RE = re.compile(
|
||
r'<li class="b_algo"[\s\S]*?<h2[^>]*>\s*<a[^>]*href="([^"]+)"',
|
||
re.IGNORECASE,
|
||
)
|
||
_SKIP_DOMAINS = ("bing.com", "microsoft.com", "r.bing.com")
|
||
_H_RE = re.compile(r"<h([1-3])[^>]*>([\s\S]*?)</h\1>", re.IGNORECASE)
|
||
_TAG_RE = re.compile(r"<[^>]+>")
|
||
|
||
|
||
def web_search_enabled() -> bool:
|
||
"""DOCGEN_WEB_TEMPLATE_SEARCH（默认 1 启用；"0" 关闭）。"""
|
||
return os.environ.get("DOCGEN_WEB_TEMPLATE_SEARCH", "1") != "0"
|
||
|
||
|
||
def _make_client(timeout: float) -> httpx.AsyncClient:
|
||
"""HTTP 客户端工厂（测试可 monkeypatch 注入 MockTransport）。"""
|
||
return httpx.AsyncClient(timeout=timeout)
|
||
|
||
|
||
@dataclass
|
||
class WebOutline:
|
||
source_url: str = ""
|
||
title: str = ""
|
||
outline: list[str] = field(default_factory=list) # ["一、...", ...] h1-h3
|
||
|
||
|
||
def _decode(raw: bytes) -> str:
|
||
for enc in ("utf-8", "gb18030"):
|
||
try:
|
||
return raw.decode(enc)
|
||
except (UnicodeDecodeError, LookupError):
|
||
continue
|
||
return raw.decode("utf-8", errors="replace")
|
||
|
||
|
||
def _extract_headings(text: str) -> list[str]:
|
||
found: list[str] = []
|
||
for m in _H_RE.finditer(text):
|
||
label = _TAG_RE.sub("", m.group(2))
|
||
label = _html.unescape(label).strip()
|
||
label = re.sub(r"\s+", " ", label)
|
||
if label and label not in found:
|
||
found.append(label)
|
||
if len(found) >= MAX_HEADINGS:
|
||
break
|
||
return found if len(found) >= MIN_HEADINGS else []
|
||
|
||
|
||
async def search_template_outline(description: str) -> WebOutline | None:
|
||
"""bing 搜索 → 首个非广告结果 → 抓取 → h1-h3 大纲。任何异常返回 None。"""
|
||
if not web_search_enabled():
|
||
return None
|
||
query = f"{description} 模板 范文"
|
||
try:
|
||
async with _make_client(TIMEOUT_SEARCH) as client:
|
||
resp = await client.get(
|
||
_BING_URL,
|
||
params={
|
||
"q": query,
|
||
"setlang": "zh-CN",
|
||
"ensearch": "0",
|
||
},
|
||
headers={
|
||
"User-Agent": _UA,
|
||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||
},
|
||
follow_redirects=True,
|
||
)
|
||
resp.raise_for_status()
|
||
page = _decode(resp.content)
|
||
except Exception as exc:
|
||
logger.info("docgen: 模版联网搜索失败（%s）", exc)
|
||
return None
|
||
|
||
m = _ALGO_LINK_RE.search(page)
|
||
if not m:
|
||
return None
|
||
url = _html.unescape(m.group(1)).strip()
|
||
host = urllib.parse.urlparse(url).netloc.lower()
|
||
if not url.startswith(("http://", "https://")) or any(
|
||
d in host for d in _SKIP_DOMAINS
|
||
):
|
||
return None
|
||
|
||
try:
|
||
async with _make_client(TIMEOUT_FETCH) as client:
|
||
resp = await client.get(
|
||
url,
|
||
headers={"User-Agent": _UA, "Accept-Language": "zh-CN,zh;q=0.9"},
|
||
follow_redirects=True,
|
||
)
|
||
resp.raise_for_status()
|
||
content = resp.content
|
||
if len(content) > MAX_RESULT_BYTES:
|
||
content = content[:MAX_RESULT_BYTES]
|
||
text = _decode(content)
|
||
except Exception as exc:
|
||
logger.info("docgen: 模版参考页抓取失败（%s）", exc)
|
||
return None
|
||
|
||
outline = _extract_headings(text)
|
||
if not outline:
|
||
return None
|
||
title_m = re.search(r"<title[^>]*>([\s\S]*?)</title>", text, re.IGNORECASE)
|
||
page_title = _html.unescape(_TAG_RE.sub("", title_m.group(1))).strip() if title_m else ""
|
||
return WebOutline(source_url=url, title=page_title[:200], outline=outline)
|