"""services/test_extraction.py —— 抽取代理单元测试（respx mock HTTP）。 覆盖：progress_pct 派生（四状态 + failed=null）、8000 down → 7900 回落命中、 alt 失败缓存（30s 内不再试 alt）、jobs 列表归一化、job 详情归一化、 plan 请求 409 → null 不报错、上传流式转发 + message 附加。 """ from __future__ import annotations import httpx import pytest import respx from app.services.extraction import ( ALT_FAILURE_TTL, ExtractionService, ExtractionUnavailable, derive_progress_pct, ) PRIMARY = "http://primary.test" ALT = "http://alt.test" PREFIX = "/api/v1/methodology" def make_service(primary: str = PRIMARY, alt: str = ALT) -> ExtractionService: return ExtractionService(primary_base_url=primary, alt_base_url=alt) # ---------- progress_pct 派生（API.md §10） ---------- @pytest.mark.parametrize( "stage,expected", [ ("queued", 5), ("mineru", 35), ("llm", 75), ("completed", 100), ("failed", None), (None, None), ("unknown_stage", None), ], ) def test_derive_progress_pct(stage, expected): assert derive_progress_pct(stage) == expected # ---------- 回落逻辑 ---------- async def test_primary_down_falls_back_to_alt(respx_mock): # 8000 down（连接失败），7900 命中 respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock( side_effect=httpx.ConnectError("primary down") ) respx_mock.get(f"{ALT}{PREFIX}/extract-jobs?limit=20").mock( return_value=httpx.Response( 200, json={ "jobs": [ { "job_id": "2943168e9c48", "status": "completed", "stage": "completed", "substage": "done", "message": "MinerU+LLM 方法学提取完成", "paper_id": "demo_paper", "family": "", "source_filename": "wang2019.pdf", "stage_count": 6, "species_count": 1, "figure_table_caption_count": 9, "figure_table_stage_count": 12, "updated_at": "2026-08-07T09:42:17", "config_url": "/api/v1/methodology/extract-jobs/2943168e9c48/bbav5-config", "methodology_plan_url": "/api/v1/methodology/extract-jobs/2943168e9c48/plan", } ], "count": 1, }, ) ) service = make_service() result = await service.list_jobs(limit=20) assert result["count"] == 1 job = result["jobs"][0] assert job["title"] == "wang2019.pdf" # source_filename 优先 assert job["progress_pct"] == 100 assert job["status"] == "completed" await service.close() async def test_primary_5xx_falls_back_to_alt(respx_mock): respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock( return_value=httpx.Response(500) ) respx_mock.get(f"{ALT}{PREFIX}/extract-jobs?limit=20").mock( return_value=httpx.Response(200, json={"jobs": [], "count": 0}) ) service = make_service() result = await service.list_jobs(limit=20) assert result == {"jobs": [], "count": 0} await service.close() async def test_both_down_raises_unavailable(respx_mock): respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock( side_effect=httpx.ConnectError("down") ) respx_mock.get(f"{ALT}{PREFIX}/extract-jobs?limit=20").mock( side_effect=httpx.ConnectError("down too") ) service = make_service() with pytest.raises(ExtractionUnavailable): await service.list_jobs(limit=20) await service.close() async def test_alt_failure_not_cached_over_30s(respx_mock): """alt 失败后 30s 内不再试 alt；超过 30s 恢复尝试。""" respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock( side_effect=httpx.ConnectError("down") ) alt_route = respx_mock.get(f"{ALT}{PREFIX}/extract-jobs?limit=20").mock( side_effect=httpx.ConnectError("down too") ) service = make_service() with pytest.raises(ExtractionUnavailable): await service.list_jobs(limit=20) assert alt_route.called is True # 第一次尝试了 alt # 立刻再请求：alt 在失败窗口内，不应再被尝试 before = alt_route.call_count with pytest.raises(ExtractionUnavailable): await service.list_jobs(limit=20) assert alt_route.call_count == before # 没有新增 alt 请求 # 模拟时间流逝超过 ALT_FAILURE_TTL service._alt_dead_until -= ALT_FAILURE_TTL + 1 alt_route.side_effect = None # 必须先清 side_effect，respx 优先走 side_effect alt_route.return_value = httpx.Response(200, json={"jobs": [], "count": 0}) result = await service.list_jobs(limit=20) assert result == {"jobs": [], "count": 0} assert alt_route.call_count == before + 1 # 恢复后重新尝试 alt await service.close() # ---------- jobs 归一化 ---------- def _raw_job(**overrides): base = { "job_id": "a1b2c3d4e5f6", "status": "running", "stage": "llm", "substage": None, "message": "LLM 关系抽取中…", "paper_id": "pecan_spl", "family": "SPL", "source_filename": "Pecan_SPL_2021.pdf", "stage_count": 4, "species_count": 1, "figure_table_caption_count": 3, "figure_table_stage_count": 4, "updated_at": "2026-08-25T09:52:00", "config_url": None, "methodology_plan_url": None, } base.update(overrides) return base async def test_jobs_normalization_stages(respx_mock): respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock( return_value=httpx.Response( 200, json={ "jobs": [ _raw_job(stage="queued", status="queued", substage=None, message="排队等待处理"), _raw_job(stage="mineru", status="running"), _raw_job(stage="failed", status="failed", source_filename=None), ], "count": 3, }, ) ) service = make_service() result = await service.list_jobs(limit=20) jobs = result["jobs"] assert [j["progress_pct"] for j in jobs] == [5, 35, None] # failed 且无 source_filename：title 回退 paper_id assert jobs[2]["title"] == "pecan_spl" assert jobs[0]["stage"] == "queued" await service.close() async def test_jobs_title_fallback_chain(respx_mock): respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs?limit=20").mock( return_value=httpx.Response( 200, json={ "jobs": [ _raw_job(job_id="onlyjobid", source_filename=None, paper_id=None), ], "count": 1, }, ) ) service = make_service() result = await service.list_jobs(limit=20) assert result["jobs"][0]["title"] == "onlyjobid" # 最后回退 job_id await service.close() # ---------- job 详情与 plan ---------- def _detail_job(**overrides): base = { "updated_at": "2026-08-07T09:42:17", "job_id": "2943168e9c48", "status": "completed", "stage": "completed", "substage": "done", "message": "MinerU+LLM 方法学提取完成", "source_path": "/home/xzq/5090/PlantAI-rag/data/evaluation/upload/wang2019.pdf", "content_list": "/home/xzq/5090/PlantAI-rag/.../wang2019_content_list.json", "output_dir": "/home/xzq/5090/PlantAI-rag/methodology_batch_demo_paper_2943168e9c48", "paper_id": "demo_paper", "family": "", "stage_count": 6, "species_count": 1, "figure_table_caption_count": 9, "figure_table_stage_count": 12, "figure_table_warning": "", "source_filename": "wang2019.pdf", "config_url": "/api/v1/methodology/extract-jobs/2943168e9c48/bbav5-config", "methodology_plan_url": "/api/v1/methodology/extract-jobs/2943168e9c48/plan", } base.update(overrides) return base async def test_get_job_completed_fetches_plan(respx_mock): respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/2943168e9c48").mock( return_value=httpx.Response(200, json=_detail_job()) ) respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/2943168e9c48/plan").mock( return_value=httpx.Response( 200, json={ "plan_id": "plan_2943168e9c48", "paper_id": "demo_paper", "gene_family": "", "stages": [], }, ) ) service = make_service() detail = await service.get_job("2943168e9c48") assert detail["job_id"] == "2943168e9c48" assert detail["title"] == "wang2019.pdf" assert detail["progress_pct"] == 100 assert detail["stats"] == { "stage_count": 6, "species_count": 1, "figure_table_caption_count": 9, "figure_table_stage_count": 12, } assert detail["detail"]["status"] == "completed" # 上游原始全量 assert detail["plan"] == { "plan_id": "plan_2943168e9c48", "paper_id": "demo_paper", "gene_family": "", "stages": [], } await service.close() async def test_get_job_incomplete_plan_409_is_null(respx_mock): """未完成时上游 plan 返回 409：BFF 置 null 不报错（API.md §10）。""" respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/a1b2c3d4e5f6").mock( return_value=httpx.Response( 200, json=_detail_job( job_id="a1b2c3d4e5f6", status="running", stage="llm", substage=None, message="LLM 关系抽取中…", ), ) ) respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/a1b2c3d4e5f6/plan").mock( return_value=httpx.Response( 409, json={"detail": "Extraction job is not completed"} ) ) service = make_service() detail = await service.get_job("a1b2c3d4e5f6") assert detail["status"] == "running" assert detail["progress_pct"] == 75 assert detail["plan"] is None # 409 被吞掉，不报错 await service.close() async def test_get_job_plan_upstream_down_yields_null_plan(respx_mock): respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/2943168e9c48").mock( return_value=httpx.Response(200, json=_detail_job()) ) respx_mock.get(f"{PRIMARY}{PREFIX}/extract-jobs/2943168e9c48/plan").mock( side_effect=httpx.ConnectError("plan service down") ) respx_mock.get(f"{ALT}{PREFIX}/extract-jobs/2943168e9c48/plan").mock( side_effect=httpx.ConnectError("alt down") ) service = make_service() detail = await service.get_job("2943168e9c48") assert detail["plan"] is None # plan 拉不到不影响 job 详情 await service.close() # ---------- 透传 ---------- async def test_passthrough_run_log(respx_mock): respx_mock.get(f"{PRIMARY}{PREFIX}/run-log").mock( return_value=httpx.Response( 200, json={ "text": "2026-08-25 10:00:01 INFO MinerU 解析完成\n" "2026-08-25 10:12:09 INFO 关系抽取 751 relations\n" }, ) ) service = make_service() result = await service._passthrough("/run-log") assert result["text"].startswith("2026-08-25 10:00:01") await service.close() # ---------- 上传 ---------- async def _file_stream(chunks: list[bytes]): for c in chunks: yield c async def test_upload_streams_and_appends_message(respx_mock): captured: dict = {} def handler(request: httpx.Request) -> httpx.Response: captured["url"] = str(request.url) captured["body"] = request.content # 多部分请求体已完整接收 return httpx.Response( 200, json={ "success": True, "job_id": "d4e5f6a7b8c9", "status_url": "/api/v1/methodology/extract-jobs/d4e5f6a7b8c9", }, ) respx_mock.post(f"{PRIMARY}{PREFIX}/extract-upload").mock(side_effect=handler) service = make_service() result = await service.upload( filename="wang2019.pdf", stream=_file_stream([b"%PDF-1.4 mock bytes"]), content_type="application/pdf", paper_id="demo_paper", family="SPL", output_name="wang2019", ) assert result == { "success": True, "job_id": "d4e5f6a7b8c9", "status_url": "/api/v1/methodology/extract-jobs/d4e5f6a7b8c9", "message": "已提交抽取任务", } # 请求被发到 primary（8000），multipart 含文件名、form 字段与文件字节 assert captured["url"] == f"{PRIMARY}{PREFIX}/extract-upload" body = captured["body"] assert b'name="file"' in body assert b"wang2019.pdf" in body assert b'name="paper_id"' in body and b"demo_paper" in body assert b'name="family"' in body and b"SPL" in body assert b"%PDF-1.4 mock bytes" in body await service.close()