351 lines
14 KiB
Python
351 lines
14 KiB
Python
"""tests/conftest.py —— 生信服务测试共享夹具。
|
||
|
||
分层标记（TEST_PLAN §9-12 / TEST_CASES §6）：
|
||
- bioflow_unit：服务层单元（runner 状态机/kernel 拼装/manifest/DB 迁移/意图分流）
|
||
- bioflow_api：端点用例（TestClient + mock docker/LLM/vstore）
|
||
|
||
全部 mock docker/LLM/vstore，不依赖生产服务（8090/8091/8094/8098 绝不探测）。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
|
||
# 测试运行 cwd = bioflow_service/（任务书自测命令）；repo 根在其上一级
|
||
REPO_ROOT = Path(__file__).resolve().parent.parent.parent
|
||
if str(REPO_ROOT) not in sys.path:
|
||
sys.path.insert(0, str(REPO_ROOT))
|
||
|
||
from app.config import Settings # noqa: E402
|
||
from app.context import Context, build_context # noqa: E402
|
||
from app.db import Database # noqa: E402
|
||
from app.services.bioflow.catalog import StageCatalog, build_catalog # noqa: E402
|
||
from app.services.bioflow.events import EventBus # noqa: E402
|
||
from app.services.bioflow.kernel import ImageMap, Kernel, KernelConfig, UnknownStageError # noqa: E402
|
||
from app.services.bioflow.runner import Runner # noqa: E402
|
||
|
||
|
||
def _run(coro):
|
||
"""同步跑一个协程（新建 event loop；docgen 测试同款可靠模式）。"""
|
||
return asyncio.new_event_loop().run_until_complete(coro)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 静态映射表 fixture（不依赖真实 stage_image_map.json；测试确定性）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
STAGE_MAP_JSON = {
|
||
"version": "2.0.0",
|
||
"images": {
|
||
"bioflow-gf": {"python": "3.11.14", "source_env": "test4py311", "docker_tag": "bioflow-gf"},
|
||
"bioflow-rnaseq": {"python": "3.11.15", "source_env": "rnaseq", "docker_tag": "bioflow-rnaseq"},
|
||
"bioflow-rnaR": {"python": "3.14.3", "source_env": "rnaR", "docker_tag": "bioflow-rnar",
|
||
"auxiliary": True},
|
||
},
|
||
"stages": {
|
||
"identify": {"image": "bioflow-gf", "module": "cli.run_identify", "executable": True},
|
||
"genome_prepare": {"image": "bioflow-gf", "module": "cli.run_genome_prepare", "executable": True},
|
||
"phylogeny": {"image": "bioflow-gf", "module": "cli.run_phylogeny", "executable": True},
|
||
"motif": {"image": "bioflow-gf", "module": "cli.run_motif", "executable": True},
|
||
"expression": {
|
||
"image": "bioflow-rnaR", "module": "cli.run_expression", "executable": True,
|
||
"note": "跨环境: 主 rnaR; rnaseq_quant 矩阵子步骤拆 bioflow-rnaseq",
|
||
},
|
||
"rnaseq_quant": {"image": "bioflow-rnaseq", "module": "cli.run_rnaseq_quant", "executable": True},
|
||
"synteny_plot_revise": {"image": "bioflow-gf", "module": None, "executable": True,
|
||
"note": "入口脚本缺失"},
|
||
},
|
||
"family_chain": ["genome_prepare", "identify", "phylogeny", "motif", "expression"],
|
||
}
|
||
|
||
|
||
@pytest.fixture()
|
||
def image_map_path(tmp_path):
|
||
p = tmp_path / "stage_image_map.json"
|
||
p.write_text(json.dumps(STAGE_MAP_JSON), encoding="utf-8")
|
||
return p
|
||
|
||
|
||
@pytest.fixture()
|
||
def image_map(image_map_path) -> ImageMap:
|
||
return Kernel.load(image_map_path, KernelConfig()).map
|
||
|
||
|
||
@pytest.fixture()
|
||
def catalog(image_map) -> StageCatalog:
|
||
return build_catalog(image_map)
|
||
|
||
|
||
@pytest.fixture()
|
||
def kcfg(tmp_path) -> KernelConfig:
|
||
return KernelConfig(
|
||
host_uid=1234,
|
||
docker_tag_suffix="dev",
|
||
data_root=str(tmp_path / "data"),
|
||
bba_env={"BBA_PFAM_DIR": "/bioflow-data/hmm"},
|
||
)
|
||
|
||
|
||
@pytest.fixture()
|
||
def kernel(image_map, kcfg) -> Kernel:
|
||
return Kernel(image_map, kcfg)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# DB（临时 SQLite；三表幂等迁移）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.fixture()
|
||
def db(tmp_path):
|
||
database = Database(tmp_path / "bioflow" / "bioflow.db")
|
||
_run(database.connect())
|
||
yield database
|
||
_run(database.close())
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# runner（mock docker 执行器）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class FakeExecutor:
|
||
"""mock docker：可编程退出码/超时/延迟/日志；记录全部调用。"""
|
||
|
||
def __init__(self, exit_code: int = 0, log: str = "", delay: float = 0.0,
|
||
manifest_status: str | None = None) -> None:
|
||
self.exit_code = exit_code
|
||
self.log = log or "stage running log line\n"
|
||
self.delay = delay
|
||
self.manifest_status = manifest_status # None=用参数决定；"success"/"failed"/"missing"
|
||
self.calls: list[dict[str, Any]] = []
|
||
self.running = 0
|
||
self.max_running = 0
|
||
|
||
def _write_manifest(self, workspace, stage: str, status: str, prefix: str = "Ft.SPL") -> None:
|
||
final = Path(workspace) / "final"
|
||
final.mkdir(parents=True, exist_ok=True)
|
||
manifest = {
|
||
"manifest_version": "1", "job_id": "t", "stage": stage,
|
||
"status": status, "prefix": prefix,
|
||
"artifacts": {"final_members": {"artifact": "final_members",
|
||
"full_path": str(final / f"{prefix}.identified_members.id"),
|
||
"type": "table", "size": 10}},
|
||
"parameters": {}, "metrics": {"final_members": 24},
|
||
}
|
||
(final / f"{prefix}.{stage}.manifest.json").write_text(
|
||
json.dumps(manifest), encoding="utf-8")
|
||
|
||
async def __call__(self, stage, input_dir, workspace, params=None,
|
||
log_path=None, **kwargs):
|
||
params = params or {}
|
||
if self.delay:
|
||
await asyncio.sleep(self.delay)
|
||
self.running += 1
|
||
self.max_running = max(self.max_running, self.running)
|
||
try:
|
||
log = str(self.log) if isinstance(self.log, str) else str(self.log)
|
||
status = self.manifest_status
|
||
if status is None:
|
||
status = "success" if self.exit_code == 0 else "failed"
|
||
if status != "missing" and self.exit_code == 0:
|
||
self._write_manifest(workspace, stage, status)
|
||
if log_path:
|
||
Path(log_path).parent.mkdir(parents=True, exist_ok=True)
|
||
Path(log_path).write_text(log, encoding="utf-8")
|
||
cmd_hint = {"stage": stage, "cpu": params.get("cpu"), "memory": params.get("memory")}
|
||
self.calls.append(cmd_hint)
|
||
from app.services.bioflow.kernel import tail_lines
|
||
return {"exit_code": self.exit_code, "timed_out": False,
|
||
"log_tail": tail_lines(log, 80), "cmd": []}
|
||
finally:
|
||
self.running -= 1
|
||
|
||
|
||
class FakeTimedOutExecutor(FakeExecutor):
|
||
async def __call__(self, stage, input_dir, workspace, params=None,
|
||
log_path=None, **kwargs):
|
||
self.calls.append({"stage": stage, "timed_out": True})
|
||
return {"exit_code": 137, "timed_out": True, "log_tail": "timeout killed\n", "cmd": []}
|
||
|
||
|
||
@pytest.fixture()
|
||
def fake_executor() -> FakeExecutor:
|
||
return FakeExecutor()
|
||
|
||
|
||
def make_runner(db, events, executor, catalog, *, concurrency: int = 3,
|
||
check_inputs: bool = False, **kwargs) -> Runner:
|
||
"""Runner 工厂：默认 force_input_check=False（状态机用例不卡输入检查），
|
||
需要输入检查的用例传 check_inputs=True（见 test_await_files_when_missing）。"""
|
||
return Runner(
|
||
db, events, executor, catalog, asyncio.Semaphore(concurrency),
|
||
force_input_check=check_inputs, **kwargs,
|
||
)
|
||
|
||
|
||
@pytest.fixture()
|
||
def events(tmp_path) -> EventBus:
|
||
return EventBus(tmp_path / "events")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# context（端点用例用）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
class FakeFlowStore:
|
||
"""mock vstore FlowStore（P6 接口形态）。"""
|
||
|
||
def __init__(self, items=None) -> None:
|
||
self.items = items or []
|
||
|
||
def search(self, query, top_k=6, **kwargs):
|
||
return {"mode": "hybrid", "items": self.items[:top_k]}
|
||
|
||
def get_chunk(self, chunk_key):
|
||
for it in self.items:
|
||
if str(it.get("chunk_key") or it.get("id")) == chunk_key:
|
||
return {"meta": it, "text": it.get("text", "")}
|
||
return None
|
||
|
||
|
||
class FakeLLM:
|
||
"""mock 规划 LLM：可编程候选/异常。"""
|
||
|
||
def __init__(self, candidates=None, error: str | None = None) -> None:
|
||
self.candidates = candidates or [
|
||
{"purpose": "SPL 家族标准鉴定流程", "stages": ["identify", "phylogeny", "motif"]},
|
||
]
|
||
self.error = error
|
||
|
||
async def __call__(self, system, user, **kwargs):
|
||
if self.error == "timeout":
|
||
from app.services.bioflow.llm import LLMTimeout
|
||
raise LLMTimeout("timeout")
|
||
if self.error == "unavailable":
|
||
from app.services.bioflow.llm import LLMUnavailable
|
||
raise LLMUnavailable("unavailable")
|
||
if self.error == "bad":
|
||
from app.services.bioflow.llm import LLMError
|
||
raise LLMError("bad")
|
||
return {"candidates": self.candidates}
|
||
|
||
|
||
@pytest.fixture()
|
||
def test_settings(tmp_path, monkeypatch):
|
||
"""测试 Settings（全部路径指向 tmp；零宿主路径）。"""
|
||
s = Settings(
|
||
_env_file=None,
|
||
data_root=str(tmp_path / "data"),
|
||
jobs_dir=str(tmp_path / "jobs"),
|
||
db_path=str(tmp_path / "bioflow.db"),
|
||
archives_dir=str(tmp_path / "archives"),
|
||
image_map_path=str(tmp_path / "stage_image_map.json"),
|
||
flow_docs_dir=str(tmp_path / "flow_docs"),
|
||
concurrency=3,
|
||
host_uid_env="BIOFLOW_TEST_UID",
|
||
service_url="http://127.0.0.1:9999",
|
||
)
|
||
Path(s.image_map_abs).parent.mkdir(parents=True, exist_ok=True)
|
||
Path(s.image_map_abs).write_text(json.dumps(STAGE_MAP_JSON), encoding="utf-8")
|
||
for d in (s.data_root_abs, s.jobs_dir_abs, s.archives_dir_abs, s.flow_docs_dir_abs):
|
||
Path(d).mkdir(parents=True, exist_ok=True)
|
||
monkeypatch.setenv("BIOFLOW_TEST_UID", "1234")
|
||
|
||
import app.config as config_mod
|
||
monkeypatch.setattr(config_mod, "settings", s)
|
||
monkeypatch.setattr(config_mod, "get_settings", lambda: s)
|
||
|
||
import app.context as context_mod
|
||
monkeypatch.setattr(context_mod, "get_settings", lambda: s)
|
||
monkeypatch.setattr(context_mod, "context", Context())
|
||
return s
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _isolated_env(tmp_path, monkeypatch):
|
||
"""每个用例隔离：关键环境键清空（不污染生产键名）且 Settings 默认全部指向 tmp。"""
|
||
for key in ("BIOFLOW_DATA_ROOT", "BIOFLOW_JOBS_DIR", "BIOFLOW_DB_PATH",
|
||
"BIOFLOW_ARCHIVES_DIR", "BIOFLOW_IMAGE_MAP", "BIOFLOW_FLOW_DOCS_DIR"):
|
||
monkeypatch.delenv(key, raising=False)
|
||
s = Settings(
|
||
_env_file=None,
|
||
data_root=str(tmp_path / "data"),
|
||
jobs_dir=str(tmp_path / "jobs"),
|
||
db_path=str(tmp_path / "bioflow.db"),
|
||
archives_dir=str(tmp_path / "archives"),
|
||
image_map_path=str(tmp_path / "stage_image_map.json"),
|
||
flow_docs_dir=str(tmp_path / "flow_docs"),
|
||
)
|
||
import app.config as config_mod
|
||
monkeypatch.setattr(config_mod, "settings", s)
|
||
monkeypatch.setattr(config_mod, "get_settings", lambda: s)
|
||
import app.context as context_mod
|
||
monkeypatch.setattr(context_mod, "get_settings", lambda: s)
|
||
# 各模块经 from ..config import get_settings 持有原始函数引用，须逐模块替换。
|
||
# 仅替换模块顶层确有此名字的（kernel/runner 用 _default_kernel_config 惰性读 config）。
|
||
import app.db as db_mod
|
||
import app.routers.bioflow as r_flow
|
||
import app.routers.biojobs as r_jobs
|
||
import app.routers.bioarchives as r_arch
|
||
for mod in (db_mod, r_flow, r_jobs, r_arch):
|
||
if hasattr(mod, "get_settings"):
|
||
monkeypatch.setattr(mod, "get_settings", lambda: s)
|
||
return s
|
||
|
||
|
||
@pytest.fixture()
|
||
def api_context(db, tmp_path, monkeypatch, image_map, fake_executor, fake_llm):
|
||
"""端点用例 context：mock docker/LLM/vstore，注入到全部路由模块。
|
||
|
||
fake_executor/fake_llm 为模块级 fixture（可被用例覆盖行为后 re-patch）。
|
||
"""
|
||
events = EventBus(tmp_path / "jobs" / "events")
|
||
catalog = build_catalog(image_map)
|
||
runner = make_runner(db, events, fake_executor, catalog)
|
||
from app.services.bioflow.planner import Planner
|
||
|
||
flow_store = FakeFlowStore([
|
||
{"chunk_key": "c1", "id": "c1", "source": "stage_contract",
|
||
"doc_kind": "contract", "stage_name": "identify",
|
||
"title": "Identify Stage Contract", "title_path": "Identify Stage Contract",
|
||
"path": "gfa_core/schemas/stages/identify_contract_v1.md",
|
||
"cite": {"anchor": "## 1", "line": 1}, "line_start": 1, "line_end": 10,
|
||
"text": "Identify 需要全蛋白 FASTA 输入。"},
|
||
])
|
||
planner = Planner(catalog, flow_store, archive_finder=None)
|
||
|
||
async def find_archive(name):
|
||
return None
|
||
planner.archive_finder = find_archive
|
||
|
||
ctx = Context(db=db, events=events, kernel=None, catalog=catalog,
|
||
runner=runner, semaphore=runner.semaphore, planner=planner,
|
||
flow_store=flow_store, executor=fake_executor)
|
||
|
||
# 路由模块以 from ..context import context 绑定，须逐模块替换
|
||
import app.routers.bioflow as r_flow
|
||
import app.routers.biojobs as r_jobs
|
||
import app.routers.bioarchives as r_arch
|
||
monkeypatch.setattr(r_flow, "context", ctx)
|
||
monkeypatch.setattr(r_jobs, "context", ctx)
|
||
monkeypatch.setattr(r_arch, "context", ctx)
|
||
import app.context as context_mod
|
||
monkeypatch.setattr(context_mod, "context", ctx)
|
||
|
||
# 规划 LLM mock（complete_json → fake_llm）
|
||
import app.services.bioflow.planner as planner_mod
|
||
monkeypatch.setattr(planner_mod, "complete_json", fake_llm)
|
||
return ctx
|
||
|
||
|
||
@pytest.fixture()
|
||
def fake_llm():
|
||
return FakeLLM()
|