jerryandClaude Fable 5
5888e1d5c1
- tests/docgen/:DC-UNIT/API/CONF/E2E/UI/REGRESSION/PERF 分组用例与可复跑脚本,报告逐条留痕(R1-R3 全记录) - 意图 30 条前后端同表双跑(pytest + node) - Playwright headless:docgen-ui/docgen-regression spec + b3 benchpanel 第 5 TAB 断言更新 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
129 lines
4.1 KiB
Python
129 lines
4.1 KiB
Python
"""test_llm.py —— LLM 调用（假 httpx，验证重试/JSON 修复，不连真 LLM）。"""
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
|
|
import httpx # noqa: E402
|
|
import pytest # noqa: E402
|
|
|
|
from app.services.docgen import llm # noqa: E402
|
|
|
|
|
|
class _FakeAsyncClient:
|
|
"""假 AsyncClient：按 scripts 列表依次返回响应/异常。"""
|
|
|
|
def __init__(self, scripts):
|
|
self.scripts = list(scripts)
|
|
self.calls = []
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *args):
|
|
pass
|
|
|
|
async def post(self, url, **kwargs):
|
|
self.calls.append(kwargs)
|
|
action = self.scripts.pop(0)
|
|
if isinstance(action, Exception):
|
|
raise action
|
|
status, payload = action
|
|
return httpx.Response(status, json=payload, request=httpx.Request("POST", url))
|
|
|
|
|
|
def _resp(content: str, status: int = 200) -> tuple[int, dict]:
|
|
return status, {"choices": [{"message": {"content": content}}]}
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _fast_backoff(monkeypatch):
|
|
monkeypatch.setattr(llm, "BACKOFF_SECONDS", (0, 0, 0))
|
|
|
|
|
|
def _run(coro):
|
|
import anyio
|
|
|
|
return anyio.run(lambda: coro)
|
|
|
|
|
|
def test_complete_text_ok(monkeypatch):
|
|
fake = _FakeAsyncClient([_resp("第一段内容")])
|
|
monkeypatch.setattr(llm.httpx, "AsyncClient", lambda: fake)
|
|
out = _run(llm.complete_text("sys", "user"))
|
|
assert out == "第一段内容"
|
|
body = fake.calls[0]["json"]
|
|
assert body["model"] == "deepseek-chat"
|
|
assert body["temperature"] == 0.3
|
|
assert "response_format" not in body
|
|
|
|
|
|
def test_complete_json_mode(monkeypatch):
|
|
fake = _FakeAsyncClient([_resp('{"title": "T", "sections": ["一"]}')])
|
|
monkeypatch.setattr(llm.httpx, "AsyncClient", lambda: fake)
|
|
out = _run(llm.complete_json("sys", "user"))
|
|
assert out == {"title": "T", "sections": ["一"]}
|
|
assert fake.calls[0]["json"]["response_format"] == {"type": "json_object"}
|
|
|
|
|
|
def test_retry_429_then_success(monkeypatch):
|
|
fake = _FakeAsyncClient([
|
|
(429, {"error": "rate"}),
|
|
(500, {"error": "boom"}),
|
|
_resp("最终内容"),
|
|
])
|
|
monkeypatch.setattr(llm.httpx, "AsyncClient", lambda: fake)
|
|
out = _run(llm.complete_text("s", "u"))
|
|
assert out == "最终内容"
|
|
assert len(fake.calls) == 3
|
|
|
|
|
|
def test_retries_exhausted_raises(monkeypatch):
|
|
fake = _FakeAsyncClient([
|
|
httpx.ConnectError("down", request=None),
|
|
httpx.ConnectError("down", request=None),
|
|
httpx.ConnectError("down", request=None),
|
|
httpx.ConnectError("down", request=None),
|
|
])
|
|
monkeypatch.setattr(llm.httpx, "AsyncClient", lambda: fake)
|
|
with pytest.raises(llm.LLMError):
|
|
_run(llm.complete_text("s", "u"))
|
|
assert len(fake.calls) == 4 # 首试 + 3 重试
|
|
|
|
|
|
def test_complete_json_repair_retry(monkeypatch):
|
|
"""第一次返回围栏 JSON（解析失败）→ 带提示重试一次 → 修复解析。"""
|
|
fake = _FakeAsyncClient([
|
|
_resp('```json\n{"title": "T"}\n```'),
|
|
_resp('{"title": "T2"}'),
|
|
])
|
|
monkeypatch.setattr(llm.httpx, "AsyncClient", lambda: fake)
|
|
out = _run(llm.complete_json("sys", "user"))
|
|
assert out == {"title": "T2"}
|
|
# 重试的 system 带 JSON 修复提示
|
|
assert "合法 JSON" in fake.calls[1]["json"]["messages"][0]["content"]
|
|
|
|
|
|
def test_complete_json_garbage_raises(monkeypatch):
|
|
fake = _FakeAsyncClient([_resp("这不是JSON"), _resp("也不是JSON")])
|
|
monkeypatch.setattr(llm.httpx, "AsyncClient", lambda: fake)
|
|
with pytest.raises(llm.LLMError):
|
|
_run(llm.complete_json("s", "u"))
|
|
|
|
|
|
def test_empty_content_raises(monkeypatch):
|
|
fake = _FakeAsyncClient([_resp("")])
|
|
monkeypatch.setattr(llm.httpx, "AsyncClient", lambda: fake)
|
|
with pytest.raises(llm.LLMError):
|
|
_run(llm.complete_text("s", "u"))
|
|
|
|
|
|
def test_auth_header_when_key(monkeypatch):
|
|
fake = _FakeAsyncClient([_resp("x")])
|
|
monkeypatch.setattr(llm.httpx, "AsyncClient", lambda: fake)
|
|
monkeypatch.setenv("DOCGEN_LLM_API_KEY", "sk-test")
|
|
_run(llm.complete_text("s", "u"))
|
|
assert fake.calls[0]["headers"]["Authorization"] == "Bearer sk-test"
|