jerryandClaude Fable 5
0875bb036c
- git mv 保留历史：ui/(Vue3+Vite)、sdk/(BFF 中间层)
- 路径语义修正：db_path/figures_cache 改相对 sdk/(BFF cwd)，
历史库 sdk/backend/data 迁移至 sdk/data/(会话/图片缓存无损)
- 引用修复 ~15 处：config.py、deploy/*.sh、vitest/playwright 配置、
单测导入路径、docs、README、gitignore、.env.example
- 骨架：agent/{qa,methodology,legacy-rag,vendor}、knowledge/manifest.yaml、
tooling/{compile,kb,crypto,assemble,audit,tools,scripts} + README
- 门禁验证：sdk pytest 125 通过、ui vitest 158 通过、
生产 BFF 脚本重启四上游全绿 + 静态托管 + 会话库无损
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
383 lines
13 KiB
Python
383 lines
13 KiB
Python
"""services/test_extraction.py —— 抽取代理单元测试（respx mock HTTP）。
|
||
|
||
覆盖：progress_pct 派生（四状态 + failed=null）、8000 down → 7900 回落命中、
|
||
alt 失败缓存（30s 内不再试 alt）、jobs 列表归一化、job 详情归一化、
|
||
plan 请求 409 → null 不报错、上传流式转发 + message 附加。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import httpx
|
||
import pytest
|
||
import respx
|
||
|
||
from app.services.extraction import (
|
||
ALT_FAILURE_TTL,
|
||
ExtractionService,
|
||
ExtractionUnavailable,
|
||
derive_progress_pct,
|
||
)
|
||
|
||
PRIMARY = "http://primary.test"
|
||
ALT = "http://alt.test"
|
||
PREFIX = "/api/v1/methodology"
|
||
|
||
|
||
def make_service(primary: str = PRIMARY, alt: str = ALT) -> ExtractionService:
|
||
return ExtractionService(primary_base_url=primary, alt_base_url=alt)
|
||
|
||
|
||
# ---------- progress_pct 派生（API.md §10） ----------
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"stage,expected",
|
||
[
|
||
("queued", 5),
|
||
("mineru", 35),
|
||
("llm", 75),
|
||
("completed", 100),
|
||
("failed", None),
|
||
(None, None),
|
||
("unknown_stage", None),
|
||
],
|
||
)
|
||
def test_derive_progress_pct(stage, expected):
|
||
assert derive_progress_pct(stage) == expected
|
||
|
||
|
||
# ---------- 回落逻辑 ----------
|
||
|
||
async def test_primary_down_falls_back_to_alt(respx_mock):
|
||
# 8000 down（连接失败），7900 命中
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock(
|
||
side_effect=httpx.ConnectError("primary down")
|
||
)
|
||
respx_mock.get(f"{ALT}{PREFIX}/extract-jobs?limit=20").mock(
|
||
return_value=httpx.Response(
|
||
200,
|
||
json={
|
||
"jobs": [
|
||
{
|
||
"job_id": "2943168e9c48",
|
||
"status": "completed",
|
||
"stage": "completed",
|
||
"substage": "done",
|
||
"message": "MinerU+LLM 方法学提取完成",
|
||
"paper_id": "demo_paper",
|
||
"family": "",
|
||
"source_filename": "wang2019.pdf",
|
||
"stage_count": 6,
|
||
"species_count": 1,
|
||
"figure_table_caption_count": 9,
|
||
"figure_table_stage_count": 12,
|
||
"updated_at": "2026-08-07T09:42:17",
|
||
"config_url": "/api/v1/methodology/extract-jobs/2943168e9c48/bbav5-config",
|
||
"methodology_plan_url": "/api/v1/methodology/extract-jobs/2943168e9c48/plan",
|
||
}
|
||
],
|
||
"count": 1,
|
||
},
|
||
)
|
||
)
|
||
|
||
service = make_service()
|
||
result = await service.list_jobs(limit=20)
|
||
assert result["count"] == 1
|
||
job = result["jobs"][0]
|
||
assert job["title"] == "wang2019.pdf" # source_filename 优先
|
||
assert job["progress_pct"] == 100
|
||
assert job["status"] == "completed"
|
||
await service.close()
|
||
|
||
async def test_primary_5xx_falls_back_to_alt(respx_mock):
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock(
|
||
return_value=httpx.Response(500)
|
||
)
|
||
respx_mock.get(f"{ALT}{PREFIX}/extract-jobs?limit=20").mock(
|
||
return_value=httpx.Response(200, json={"jobs": [], "count": 0})
|
||
)
|
||
service = make_service()
|
||
result = await service.list_jobs(limit=20)
|
||
assert result == {"jobs": [], "count": 0}
|
||
await service.close()
|
||
|
||
async def test_both_down_raises_unavailable(respx_mock):
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock(
|
||
side_effect=httpx.ConnectError("down")
|
||
)
|
||
respx_mock.get(f"{ALT}{PREFIX}/extract-jobs?limit=20").mock(
|
||
side_effect=httpx.ConnectError("down too")
|
||
)
|
||
service = make_service()
|
||
with pytest.raises(ExtractionUnavailable):
|
||
await service.list_jobs(limit=20)
|
||
await service.close()
|
||
|
||
async def test_alt_failure_not_cached_over_30s(respx_mock):
|
||
"""alt 失败后 30s 内不再试 alt；超过 30s 恢复尝试。"""
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock(
|
||
side_effect=httpx.ConnectError("down")
|
||
)
|
||
alt_route = respx_mock.get(f"{ALT}{PREFIX}/extract-jobs?limit=20").mock(
|
||
side_effect=httpx.ConnectError("down too")
|
||
)
|
||
|
||
service = make_service()
|
||
with pytest.raises(ExtractionUnavailable):
|
||
await service.list_jobs(limit=20)
|
||
assert alt_route.called is True # 第一次尝试了 alt
|
||
|
||
# 立刻再请求：alt 在失败窗口内，不应再被尝试
|
||
before = alt_route.call_count
|
||
with pytest.raises(ExtractionUnavailable):
|
||
await service.list_jobs(limit=20)
|
||
assert alt_route.call_count == before # 没有新增 alt 请求
|
||
|
||
# 模拟时间流逝超过 ALT_FAILURE_TTL
|
||
service._alt_dead_until -= ALT_FAILURE_TTL + 1
|
||
alt_route.side_effect = None # 必须先清 side_effect，respx 优先走 side_effect
|
||
alt_route.return_value = httpx.Response(200, json={"jobs": [], "count": 0})
|
||
result = await service.list_jobs(limit=20)
|
||
assert result == {"jobs": [], "count": 0}
|
||
assert alt_route.call_count == before + 1 # 恢复后重新尝试 alt
|
||
await service.close()
|
||
|
||
|
||
# ---------- jobs 归一化 ----------
|
||
|
||
|
||
def _raw_job(**overrides):
|
||
base = {
|
||
"job_id": "a1b2c3d4e5f6",
|
||
"status": "running",
|
||
"stage": "llm",
|
||
"substage": None,
|
||
"message": "LLM 关系抽取中…",
|
||
"paper_id": "pecan_spl",
|
||
"family": "SPL",
|
||
"source_filename": "Pecan_SPL_2021.pdf",
|
||
"stage_count": 4,
|
||
"species_count": 1,
|
||
"figure_table_caption_count": 3,
|
||
"figure_table_stage_count": 4,
|
||
"updated_at": "2026-08-25T09:52:00",
|
||
"config_url": None,
|
||
"methodology_plan_url": None,
|
||
}
|
||
base.update(overrides)
|
||
return base
|
||
|
||
async def test_jobs_normalization_stages(respx_mock):
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock(
|
||
return_value=httpx.Response(
|
||
200,
|
||
json={
|
||
"jobs": [
|
||
_raw_job(stage="queued", status="queued", substage=None,
|
||
message="排队等待处理"),
|
||
_raw_job(stage="mineru", status="running"),
|
||
_raw_job(stage="failed", status="failed",
|
||
source_filename=None),
|
||
],
|
||
"count": 3,
|
||
},
|
||
)
|
||
)
|
||
service = make_service()
|
||
result = await service.list_jobs(limit=20)
|
||
jobs = result["jobs"]
|
||
assert [j["progress_pct"] for j in jobs] == [5, 35, None]
|
||
# failed 且无 source_filename：title 回退 paper_id
|
||
assert jobs[2]["title"] == "pecan_spl"
|
||
assert jobs[0]["stage"] == "queued"
|
||
await service.close()
|
||
|
||
async def test_jobs_title_fallback_chain(respx_mock):
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock(
|
||
return_value=httpx.Response(
|
||
200,
|
||
json={
|
||
"jobs": [
|
||
_raw_job(job_id="onlyjobid", source_filename=None, paper_id=None),
|
||
],
|
||
"count": 1,
|
||
},
|
||
)
|
||
)
|
||
service = make_service()
|
||
result = await service.list_jobs(limit=20)
|
||
assert result["jobs"][0]["title"] == "onlyjobid" # 最后回退 job_id
|
||
await service.close()
|
||
|
||
|
||
# ---------- job 详情与 plan ----------
|
||
|
||
|
||
def _detail_job(**overrides):
|
||
base = {
|
||
"updated_at": "2026-08-07T09:42:17",
|
||
"job_id": "2943168e9c48",
|
||
"status": "completed",
|
||
"stage": "completed",
|
||
"substage": "done",
|
||
"message": "MinerU+LLM 方法学提取完成",
|
||
"source_path": "/home/xzq/5090/PlantAI-rag/data/evaluation/upload/wang2019.pdf",
|
||
"content_list": "/home/xzq/5090/PlantAI-rag/.../wang2019_content_list.json",
|
||
"output_dir": "/home/xzq/5090/PlantAI-rag/methodology_batch_demo_paper_2943168e9c48",
|
||
"paper_id": "demo_paper",
|
||
"family": "",
|
||
"stage_count": 6,
|
||
"species_count": 1,
|
||
"figure_table_caption_count": 9,
|
||
"figure_table_stage_count": 12,
|
||
"figure_table_warning": "",
|
||
"source_filename": "wang2019.pdf",
|
||
"config_url": "/api/v1/methodology/extract-jobs/2943168e9c48/bbav5-config",
|
||
"methodology_plan_url": "/api/v1/methodology/extract-jobs/2943168e9c48/plan",
|
||
}
|
||
base.update(overrides)
|
||
return base
|
||
|
||
async def test_get_job_completed_fetches_plan(respx_mock):
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/2943168e9c48").mock(
|
||
return_value=httpx.Response(200, json=_detail_job())
|
||
)
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/2943168e9c48/plan").mock(
|
||
return_value=httpx.Response(
|
||
200,
|
||
json={
|
||
"plan_id": "plan_2943168e9c48",
|
||
"paper_id": "demo_paper",
|
||
"gene_family": "",
|
||
"stages": [],
|
||
},
|
||
)
|
||
)
|
||
service = make_service()
|
||
detail = await service.get_job("2943168e9c48")
|
||
assert detail["job_id"] == "2943168e9c48"
|
||
assert detail["title"] == "wang2019.pdf"
|
||
assert detail["progress_pct"] == 100
|
||
assert detail["stats"] == {
|
||
"stage_count": 6,
|
||
"species_count": 1,
|
||
"figure_table_caption_count": 9,
|
||
"figure_table_stage_count": 12,
|
||
}
|
||
assert detail["detail"]["status"] == "completed" # 上游原始全量
|
||
assert detail["plan"] == {
|
||
"plan_id": "plan_2943168e9c48",
|
||
"paper_id": "demo_paper",
|
||
"gene_family": "",
|
||
"stages": [],
|
||
}
|
||
await service.close()
|
||
|
||
async def test_get_job_incomplete_plan_409_is_null(respx_mock):
|
||
"""未完成时上游 plan 返回 409：BFF 置 null 不报错（API.md §10）。"""
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/a1b2c3d4e5f6").mock(
|
||
return_value=httpx.Response(
|
||
200,
|
||
json=_detail_job(
|
||
job_id="a1b2c3d4e5f6",
|
||
status="running",
|
||
stage="llm",
|
||
substage=None,
|
||
message="LLM 关系抽取中…",
|
||
),
|
||
)
|
||
)
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/a1b2c3d4e5f6/plan").mock(
|
||
return_value=httpx.Response(
|
||
409, json={"detail": "Extraction job is not completed"}
|
||
)
|
||
)
|
||
service = make_service()
|
||
detail = await service.get_job("a1b2c3d4e5f6")
|
||
assert detail["status"] == "running"
|
||
assert detail["progress_pct"] == 75
|
||
assert detail["plan"] is None # 409 被吞掉，不报错
|
||
await service.close()
|
||
|
||
async def test_get_job_plan_upstream_down_yields_null_plan(respx_mock):
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/2943168e9c48").mock(
|
||
return_value=httpx.Response(200, json=_detail_job())
|
||
)
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/2943168e9c48/plan").mock(
|
||
side_effect=httpx.ConnectError("plan service down")
|
||
)
|
||
respx_mock.get(f"{ALT}{PREFIX}/extract-jobs/2943168e9c48/plan").mock(
|
||
side_effect=httpx.ConnectError("alt down")
|
||
)
|
||
service = make_service()
|
||
detail = await service.get_job("2943168e9c48")
|
||
assert detail["plan"] is None # plan 拉不到不影响 job 详情
|
||
await service.close()
|
||
|
||
|
||
# ---------- 透传 ----------
|
||
|
||
async def test_passthrough_run_log(respx_mock):
|
||
respx_mock.get(f"{PRIMARY}{PREFIX}/run-log").mock(
|
||
return_value=httpx.Response(
|
||
200,
|
||
json={
|
||
"text": "2026-08-25 10:00:01 INFO MinerU 解析完成\n"
|
||
"2026-08-25 10:12:09 INFO 关系抽取 751 relations\n"
|
||
},
|
||
)
|
||
)
|
||
service = make_service()
|
||
result = await service._passthrough("/run-log")
|
||
assert result["text"].startswith("2026-08-25 10:00:01")
|
||
await service.close()
|
||
|
||
|
||
# ---------- 上传 ----------
|
||
|
||
async def _file_stream(chunks: list[bytes]):
|
||
for c in chunks:
|
||
yield c
|
||
|
||
async def test_upload_streams_and_appends_message(respx_mock):
|
||
captured: dict = {}
|
||
|
||
def handler(request: httpx.Request) -> httpx.Response:
|
||
captured["url"] = str(request.url)
|
||
captured["body"] = request.content # 多部分请求体已完整接收
|
||
return httpx.Response(
|
||
200,
|
||
json={
|
||
"success": True,
|
||
"job_id": "d4e5f6a7b8c9",
|
||
"status_url": "/api/v1/methodology/extract-jobs/d4e5f6a7b8c9",
|
||
},
|
||
)
|
||
|
||
respx_mock.post(f"{PRIMARY}{PREFIX}/extract-upload").mock(side_effect=handler)
|
||
|
||
service = make_service()
|
||
result = await service.upload(
|
||
filename="wang2019.pdf",
|
||
stream=_file_stream([b"%PDF-1.4 mock bytes"]),
|
||
content_type="application/pdf",
|
||
paper_id="demo_paper",
|
||
family="SPL",
|
||
output_name="wang2019",
|
||
)
|
||
assert result == {
|
||
"success": True,
|
||
"job_id": "d4e5f6a7b8c9",
|
||
"status_url": "/api/v1/methodology/extract-jobs/d4e5f6a7b8c9",
|
||
"message": "已提交抽取任务",
|
||
}
|
||
# 请求被发到 primary（8000），multipart 含文件名、form 字段与文件字节
|
||
assert captured["url"] == f"{PRIMARY}{PREFIX}/extract-upload"
|
||
body = captured["body"]
|
||
assert b'name="file"' in body
|
||
assert b"wang2019.pdf" in body
|
||
assert b'name="paper_id"' in body and b"demo_paper" in body
|
||
assert b'name="family"' in body and b"SPL" in body
|
||
assert b"%PDF-1.4 mock bytes" in body
|
||
await service.close()
|