diff --git a/bioflow_service/app/__init__.py b/bioflow_service/app/__init__.py new file mode 100644 index 0000000..e99f6e2 --- /dev/null +++ b/bioflow_service/app/__init__.py @@ -0,0 +1 @@ +"""app/__init__.py —— bioflow-service 应用包。""" diff --git a/bioflow_service/app/config.py b/bioflow_service/app/config.py new file mode 100644 index 0000000..8fd2f31 --- /dev/null +++ b/bioflow_service/app/config.py @@ -0,0 +1,133 @@ +"""config.py —— 生信服务（bioflow-service 8098）环境变量驱动配置（pydantic-settings）。 + +全部键可用环境变量覆盖（前缀 BIOFLOW_；LLM 沿用 docgen 的 DOCGEN_LLM_* 直连模式）。 +零硬编码纪律：不出现任何写死宿主路径；仓库内相对路径按 __file__ 推导 repo 根。 +""" +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +from pydantic_settings import BaseSettings, SettingsConfigDict + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent # bioflow_service/.. = 仓库根 + + +def _resolve_root(v: str) -> str: + """仓库内相对路径按 __file__ 推导 repo 根解析为绝对路径。""" + p = Path(v).expanduser() + if not p.is_absolute(): + p = _REPO_ROOT / p + return str(p) + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + env_prefix="BIOFLOW_", + ) + + # ---- 应用 ---- + app_name: str = "plantai-bioflow-service" + version: str = "0.1.0" + + # ---- 阶段→镜像映射（A1 交付物，仓库内 bioflow_kernel/stage_image_map.json）---- + image_map_path: str = "bioflow_kernel/stage_image_map.json" + + # ---- 数据与任务目录 ---- + data_root: str = "bioflow_data" # 生信数据根（hmm/rag_db 等，只读挂载进容器） + jobs_dir: str = "bioflow_data/jobs" # 任务目录根（input/workspace/events） + db_path: str = "bioflow_data/bioflow.db" # 独立 SQLite（与 BFF 完全隔离，FR-BIO-028） + archives_dir: str = "bioflow_data/archives" + + # ---- 执行 ---- + concurrency: int = 3 # 全局并发信号量上限（FR-BIO-031，可配） + stage_timeout_sec: float = 36000.0 # 单阶段 docker 硬超时（秒，10h） + docker_image_tag_suffix: str = "dev" # 镜像 tag 后缀（P0 交付物为 :dev） + host_uid_env: str = "BIOFLOW_HOST_UID" # 宿主机 uid 环境变量名；未设置时回退服务进程 uid + container_data_dir: str = "/bioflow-data" + container_input_dir: str = "/job/input" + container_workspace_dir: str = "/job/workspace" + default_cpus: str = "4" + default_memory: str = "32g" + log_tail_lines: int = 80 # manifest 接口日志尾行数（≤80） + max_upload_gb: float = 100.0 # 上传大小上限（GB；超限 413） + upload_ext_allow: str = ".fasta,.fa,.faa,.fna,.pep,.cds,.gz,.gff,.gff3,.gtf,.txt,.tsv,.csv,.xlsx,.xls,.bed,.vcf,.vcf.gz,.bam,.cram,.sam,.hmm,.nwk,.newick,.treefile,.json,.yaml,.yml,.pdf" + + # ---- 规划（planner）---- + llm_plan_timeout: float = 60.0 # 规划 LLM 单次调用超时（秒） + plan_candidates: int = 3 # 候选路径条数（2~3，FR-BIO-002） + plan_max_rounds: int = 2 # LLM 输出硬校验失败的最多重试轮数 + + # ---- 知识库（vstore/flow.py 本地实例；无 vstore 时自动降级 llm_only）---- + flow_docs_dir: str = "vstore/data/flow_docs" + + # ---- LLM（沿用 docgen deepseek 直连模式，DOCGEN_LLM_* 键）---- + docgen_llm_base_url: str = "https://api.deepseek.com/v1" + docgen_llm_api_key: str = "" + docgen_llm_model: str = "deepseek-chat" + docgen_llm_timeout: float = 300.0 + + # ---- 报告登记（BFF 反代端点；空=未配置，登记降级为仅落表）---- + service_url: str = "http://127.0.0.1:8098" + + # ---- 容器环境变量（BBA_*；零硬编码：路径全部从本配置推导）---- + @property + def bba_env(self) -> dict[str, str]: + """docker run 注入的 BBA_* 环境（值全部来自配置推导，无写死宿主路径）。""" + return { + "BBA_PFAM_DIR": f"{self.container_data_dir}/hmm", + "BBA_RAG_MAPPING_FILE": f"{self.container_data_dir}/rag_db/pfam_mapping.json", + "BBA_DISABLE_RAG_PFAM_LOOKUP": "0", + } + + @property + def host_uid(self) -> int: + """--user 注入 uid：BIOFLOW_HOST_UID 环境变量优先，否则服务进程 uid。""" + import os + + raw = os.environ.get(self.host_uid_env, "").strip() + if raw: + try: + return int(raw) + except ValueError: + pass + return os.getuid() + + @property + def image_map_abs(self) -> str: + return _resolve_root(self.image_map_path) + + @property + def data_root_abs(self) -> str: + return _resolve_root(self.data_root) + + @property + def jobs_dir_abs(self) -> str: + return _resolve_root(self.jobs_dir) + + @property + def db_path_abs(self) -> str: + return _resolve_root(self.db_path) + + @property + def archives_dir_abs(self) -> str: + return _resolve_root(self.archives_dir) + + @property + def flow_docs_dir_abs(self) -> str: + return _resolve_root(self.flow_docs_dir) + + @property + def upload_ext_set(self) -> set[str]: + return {ext.strip().lower() for ext in self.upload_ext_allow.split(",") if ext.strip()} + + +@lru_cache(maxsize=1) +def get_settings() -> Settings: + return Settings() + + +settings = get_settings() diff --git a/bioflow_service/app/context.py b/bioflow_service/app/context.py new file mode 100644 index 0000000..c82ae75 --- /dev/null +++ b/bioflow_service/app/context.py @@ -0,0 +1,213 @@ +"""app/context.py —— 生信服务共享实例容器（db/events/kernel/catalog/runner/semaphore/planner）。 + +- 生产：app 启动时 init_context() 加载真实依赖（SQLite、stage_image_map.json、 + vstore FlowStore 本地实例（存在时）、真 docker 执行器）。 +- 测试：build_context() 可注入 fake db/executor/flow_store（mock docker/LLM/vstore）。 +- vstore 不存在/不可用：FlowStore 为 None，planner 自动降级 llm_only（FR-BIO-010 降级）。 +- 并发信号量（BIOFLOW_CONCURRENCY，FR-BIO-031）每 context 一个。 +""" +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass, field +from typing import Any + +from .config import get_settings +from .db import Database, db as default_db +from .services.bioflow.catalog import StageCatalog, build_catalog +from .services.bioflow.events import EventBus +from .services.bioflow.kernel import Kernel, KernelConfig, load_kernel +from .services.bioflow.planner import Planner +from .services.bioflow.runner import Runner + +logger = logging.getLogger(__name__) + + +@dataclass +class Context: + db: Any = None + events: EventBus | None = None + kernel: Kernel | None = None + catalog: StageCatalog | None = None + runner: Runner | None = None + semaphore: asyncio.Semaphore | None = None + planner: Planner | None = None + flow_store: Any = None + executor: Any = None + records: dict[str, Any] = field(default_factory=dict) # job_id → JobRecord（内存） + _tasks: list[asyncio.Task] = field(default_factory=list) + + def spawn(self, coro: Any) -> asyncio.Task: + task = asyncio.create_task(coro) + self._tasks.append(task) + task.add_done_callback(lambda t: self._tasks.remove(t) if t in self._tasks else None) + return task + + async def close(self) -> None: + for task in self._tasks: + task.cancel() + if self._tasks: + await asyncio.gather(*self._tasks, return_exceptions=True) + self._tasks.clear() + + +# 全局 context（测试可整体替换） +context: Context = Context() + + +def init_events(jobs_dir: str) -> EventBus: + from pathlib import Path + + return EventBus(Path(jobs_dir) / "events") + + +def init_flow_store(flow_docs_dir: str) -> Any | None: + """vstore FlowStore 本地实例（存在则加载；失败/缺失返回 None → planner 降级）。""" + try: + import sys + from pathlib import Path + + repo_root = Path(__file__).resolve().parent.parent.parent + if str(repo_root) not in sys.path: + sys.path.insert(0, str(repo_root)) + from vstore.flow import FlowStore # P6 A4 交付；缺失则降级 + + return FlowStore(flow_docs_dir) + except Exception as exc: # noqa: BLE001 - 缺失/损坏降级 llm_only，不阻断服务 + logger.warning("context: vstore FlowStore 不可用（planner 降级 llm_only）: %s", exc) + return None + + +def build_context( + *, + db: Database | None = None, + events: EventBus | None = None, + kernel: Kernel | None = None, + catalog: StageCatalog | None = None, + executor: Any = None, + semaphore: asyncio.Semaphore | None = None, + flow_store: Any = None, + runner_kwargs: dict[str, Any] | None = None, +) -> Context: + """构造 context（生产与测试共用；各依赖可注入）。 + + executor=None 时用 kernel.run_stage（真 docker）。 + """ + settings = get_settings() + database = db or default_db + bus = events or init_events(settings.jobs_dir_abs) + kern = kernel or load_kernel() + cat = catalog or build_catalog(kern.map) + sem = semaphore or asyncio.Semaphore(settings.concurrency) + exec_fn = executor or kern.run_stage + runner = Runner( + database, + bus, + exec_fn, + cat, + sem, + **(runner_kwargs or {}), + ) + planner = Planner( + cat, + flow_store, + llm_plan_timeout=settings.llm_plan_timeout, + plan_candidates=settings.plan_candidates, + plan_max_rounds=settings.plan_max_rounds, + archive_finder=_make_archive_finder(database), + ) + return Context( + db=database, + events=bus, + kernel=kern, + catalog=cat, + runner=runner, + semaphore=sem, + planner=planner, + flow_store=flow_store, + executor=exec_fn, + ) + + +def _make_archive_finder(database: Database): + """planner 复用归档查找回调（async；store 查询）。""" + from .services.bioflow import store + + async def find(name: str) -> dict[str, Any] | None: + try: + return await store.find_archive_by_name(database, name) + except Exception: # noqa: BLE001 + return None + + return find + + +async def init_context(db_path: str | None = None) -> Context: + """应用启动：初始化全局 context（幂等；db_path 供测试替换）。""" + global context + from .db import init_db + + database = context.db + if db_path is not None: + database = await init_db(db_path) + elif context.db is None: + database = await init_db() + context = build_context(db=database) + return context + + +async def shutdown_context() -> None: + await context.close() + + +# ---- JobRecord 注册与恢复（路由层操作任务的内存记录）---- + + +def register_record(record: Any) -> None: + context.records[record.id] = record + + +def _record_from_job(job: dict[str, Any], stages: list[dict[str, Any]]) -> Any: + """DB 行 → JobRecord（无运行中内存记录时恢复，跨重启安全）。""" + from .services.bioflow.runner import JobRecord, params_dict + + record = JobRecord( + id=job["id"], + session_id=job["session_id"], + title=job["title"], + status=job["status"], + stage_chain=list(job["stage_chain"] or []), + params=params_dict(job.get("params")), + auto_continue=bool(job.get("auto_continue")), + archive_id=job.get("archive_id"), + ) + for s in stages: + record._stage_states[s["stage"]] = { + "status": s["status"], + "params": params_dict(s.get("params")), + "attempt": int(s.get("attempt") or 0), + "manifest_path": s.get("manifest_path"), + "log_tail": s.get("log_tail") or "", + "error": s.get("error"), + "started_at": s.get("started_at"), + "finished_at": s.get("finished_at"), + } + return record + + +async def load_record(job_id: str) -> Any | None: + """取任务 JobRecord：内存优先，否则 DB 恢复（并注册回内存）。""" + if job_id in context.records: + return context.records[job_id] + if context.db is None: + return None + from .services.bioflow import store + + job = await store.get_job(context.db, job_id) + if job is None: + return None + stages = await store.list_stages(context.db, job_id) + record = _record_from_job(job, stages) + context.records[job_id] = record + return record diff --git a/bioflow_service/app/db.py b/bioflow_service/app/db.py new file mode 100644 index 0000000..5e4d2a7 --- /dev/null +++ b/bioflow_service/app/db.py @@ -0,0 +1,200 @@ +"""db.py —— 生信服务独立 SQLite（aiosqlite + WAL，无 ORM），三张表 + 幂等迁移。 + +FR-BIO-028：与 BFF 完全隔离（独立进程 + 独立 bioflow.db）。 +表（实施计划 §六三表清单）： +- bio_jobs：id / session_id / title / status / stage_chain(JSON) / params(JSON) / + archive_id / auto_continue / progress / error / created_at / updated_at +- bio_stages：id / job_id(FK CASCADE) / stage / status / params(JSON) / manifest_path / + log_tail / attempt / started_at / finished_at / error +- bio_archives：id / job_id / name / description / stage_chain(JSON) / params(JSON) / + report_artifact / disk_path / size_bytes / reused_count / created_at + +时间一律 ISO 8601 UTC（由服务侧生成，同 docgen 模式）。 +模式沿用 docgen 同款：CREATE TABLE IF NOT EXISTS + PRAGMA table_info 幂等加列。 +""" +from __future__ import annotations + +import asyncio +import json +import logging +import time +from collections.abc import AsyncIterator, Iterable +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import aiosqlite + +from .config import get_settings + +logger = logging.getLogger(__name__) + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS bio_jobs ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '生信任务', + status TEXT NOT NULL DEFAULT 'created', + stage_chain TEXT NOT NULL DEFAULT '[]', + params TEXT NOT NULL DEFAULT '{}', + archive_id TEXT, + auto_continue INTEGER NOT NULL DEFAULT 0, + progress INTEGER NOT NULL DEFAULT 0, + error TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_bio_jobs_session ON bio_jobs(session_id); +CREATE INDEX IF NOT EXISTS idx_bio_jobs_status ON bio_jobs(status); +CREATE INDEX IF NOT EXISTS idx_bio_jobs_updated ON bio_jobs(updated_at); + +CREATE TABLE IF NOT EXISTS bio_stages ( + id TEXT PRIMARY KEY, + job_id TEXT NOT NULL REFERENCES bio_jobs(id) ON DELETE CASCADE, + stage TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'waiting', + params TEXT NOT NULL DEFAULT '{}', + manifest_path TEXT, + log_tail TEXT NOT NULL DEFAULT '', + attempt INTEGER NOT NULL DEFAULT 0, + started_at TEXT, + finished_at TEXT, + error TEXT, + UNIQUE(job_id, stage) +); + +CREATE INDEX IF NOT EXISTS idx_bio_stages_job ON bio_stages(job_id); + +CREATE TABLE IF NOT EXISTS bio_archives ( + id TEXT PRIMARY KEY, + job_id TEXT NOT NULL, + name TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + stage_chain TEXT NOT NULL DEFAULT '[]', + params TEXT NOT NULL DEFAULT '{}', + report_artifact TEXT, + disk_path TEXT, + size_bytes INTEGER NOT NULL DEFAULT 0, + reused_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_bio_archives_name ON bio_archives(name); +CREATE INDEX IF NOT EXISTS idx_bio_archives_created ON bio_archives(created_at); +""" + +# 增量列幂等迁移（老库已建表时 CREATE IF NOT EXISTS 不会加列） +_BIO_JOBS_MIGRATIONS = ( + ("archive_id", "ALTER TABLE bio_jobs ADD COLUMN archive_id TEXT"), + ("error", "ALTER TABLE bio_jobs ADD COLUMN error TEXT"), +) +_BIO_ARCHIVES_MIGRATIONS = ( + ("report_artifact", "ALTER TABLE bio_archives ADD COLUMN report_artifact TEXT"), +) + + +def now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def dumps_json(value: Any, default: Any = None) -> str: + try: + return json.dumps(value, ensure_ascii=False) + except (TypeError, ValueError): + return json.dumps(default, ensure_ascii=False) + + +def loads_json(text: str | None, default: Any = None) -> Any: + if text is None: + return default + try: + return json.loads(text) + except (TypeError, ValueError): + return default + + +class Database: + """aiosqlite 单连接封装（WAL；写串行化，同 docgen 模式）。""" + + def __init__(self, path: str | Path | None = None) -> None: + self._path: Path | None = Path(path) if path is not None else None + self._conn: aiosqlite.Connection | None = None + self._lock = asyncio.Lock() + + @property + def path(self) -> Path: + if self._path is None: + self._path = Path(get_settings().db_path_abs) + return self._path + + async def connect(self) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + self._conn = await aiosqlite.connect(self.path) + self._conn.row_factory = aiosqlite.Row + await self._conn.execute("PRAGMA journal_mode=WAL") + await self._conn.execute("PRAGMA foreign_keys=ON") + await self._conn.execute("PRAGMA busy_timeout=5000") + await self._conn.executescript(_SCHEMA) + await self._migrate_columns() + await self._conn.commit() + + async def close(self) -> None: + if self._conn is not None: + await self._conn.close() + self._conn = None + + async def _migrate_columns(self) -> None: + """三表增量列幂等迁移（同 docgen _migrate_docgen_columns 模式）。""" + for table, migrations in ( + ("bio_jobs", _BIO_JOBS_MIGRATIONS), + ("bio_archives", _BIO_ARCHIVES_MIGRATIONS), + ): + cur = await self._conn.execute(f"PRAGMA table_info({table})") + cols = {row[1] for row in await cur.fetchall()} + for name, ddl in migrations: + if name not in cols: + await self._conn.execute(ddl) + + @property + def conn(self) -> aiosqlite.Connection: + if self._conn is None: + raise RuntimeError("Database 未初始化：先 await init_db()") + return self._conn + + async def fetchall(self, sql: str, params: Iterable[Any] = ()) -> list[aiosqlite.Row]: + async with self._conn.execute(sql, tuple(params)) as cursor: + return list(await cursor.fetchall()) + + async def fetchone(self, sql: str, params: Iterable[Any] = ()) -> aiosqlite.Row | None: + async with self._conn.execute(sql, tuple(params)) as cursor: + return await cursor.fetchone() + + async def execute(self, sql: str, params: Iterable[Any] = ()) -> int: + async with self._lock: + cur = await self._conn.execute(sql, tuple(params)) + await self._conn.commit() + return cur.lastrowid + + +db = Database() + + +async def init_db(path: str | Path | None = None) -> Database: + global db + if path is not None: + db = Database(path) + await db.connect() + return db + + +async def get_db() -> AsyncIterator[Database]: + yield db + + +async def close_db() -> None: + await db.close() + + +def monotonic() -> float: + return time.monotonic() diff --git a/bioflow_service/app/main.py b/bioflow_service/app/main.py new file mode 100644 index 0000000..21a8f3c --- /dev/null +++ b/bioflow_service/app/main.py @@ -0,0 +1,80 @@ +"""main.py —— 生信服务应用入口（uvicorn 入口，明文白名单文件）。 + +- 独立部署树 bioflow_service/（FR-BIO-028：独立进程 8098 + 独立 SQLite）。 +- lifespan：init_db（幂等迁移）→ init_context（kernel/catalog/planner/runner） + → recover（服务重启恢复 in-flight 续跑，FR-BIO-031）→ shutdown 清理。 +- 端点：health（/api/bioflow/health，前端探活 FR-BIO-025）。 +- 不依赖 BFF/生产服务（8090/8091 等绝不探测/连接）。 +""" +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware + +from .config import get_settings +from .context import context, init_context, shutdown_context +from .db import close_db, init_db +from .routers import bioarchives, bioflow, biojobs + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def create_app() -> FastAPI: + settings = get_settings() + app = FastAPI(title=settings.app_name, version=settings.version) + + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], # 生信服务直连形态（FR-BIO-027）；BFF 反代为同源 + allow_credentials=False, + allow_methods=["*"], + allow_headers=["*"], + ) + + @asynccontextmanager + async def _lifespan(_app: FastAPI): + # 测试注入：context.db 已由测试预置（build_context 注入的临时库）， + # 跳过启动初始化（不替换全局 db、不加载真实映射表、不 recover）。 + # 注意经模块属性动态读取（monkeypatch 替换的是 context 模块的全局）。 + from . import context as context_mod + + injected = context_mod.context.db is not None + if not injected: + try: + await init_db() + except Exception: + # SQLite 初始化失败不阻断服务启动（health 报 db 降级；端点返回 INTERNAL） + logger.exception("main: 数据库初始化失败（health 将报 db 降级）") + try: + await init_context() + except Exception: + logger.exception("main: context 初始化失败（kernel 映射表缺失时 docker 执行不可用）") + try: + if context_mod.context.runner is not None: + recovered = await context_mod.context.runner.recover() + if recovered: + logger.info("main: 重启恢复 %d 个 in-flight 任务续跑", len(recovered)) + except Exception: + logger.exception("main: 重启恢复扫描失败（任务不重跑，等待后续操作）") + try: + yield + finally: + await shutdown_context() + try: + await close_db() + except Exception: + pass + + app.router.lifespan_context = _lifespan + + app.include_router(bioflow.router) + app.include_router(biojobs.router) + app.include_router(bioarchives.router) + return app + + +app = create_app() diff --git a/bioflow_service/app/routers/__init__.py b/bioflow_service/app/routers/__init__.py new file mode 100644 index 0000000..c76a78a --- /dev/null +++ b/bioflow_service/app/routers/__init__.py @@ -0,0 +1 @@ +"""app/routers/__init__.py —— bioflow-service routers 包。""" diff --git a/bioflow_service/app/routers/bioarchives.py b/bioflow_service/app/routers/bioarchives.py new file mode 100644 index 0000000..c58741d --- /dev/null +++ b/bioflow_service/app/routers/bioarchives.py @@ -0,0 +1,240 @@ +"""app/routers/bioarchives.py —— 存档端点（archives 全套 + 报告登记接口）。 + +端点（实施计划 §六清单）： +- POST /api/bioflow/archives 存档（FR-BIO-020/021） +- GET /api/bioflow/archives 列表（FR-BIO-022） +- GET /api/bioflow/archives/{id} 详情 +- DELETE /api/bioflow/archives/{id} 删除（磁盘目录 + 行） +- POST /api/bioflow/archives/{id}/run 复用发起（FR-BIO-023） +- POST /api/bioflow/archives/run 按名称匹配复用（精确优先/模糊候选） +- GET /api/bioflow/archives/{id}/download 打包下载（zip） +- GET /api/bioflow/archives/{id}/report 报告单独取（HTML） +- POST /api/bioflow/reports/register 报告登记接口（BFF 反代调此端点 + → 落 doc_artifacts 协调字段） +""" +from __future__ import annotations + +import logging +import shutil +import uuid +import zipfile +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, Query +from fastapi.responses import FileResponse, JSONResponse + +from ..config import get_settings +from ..context import Context, context, load_record +from ..schemas import ArchiveCreateRequest, ArchiveRunRequest +from ..services.bioflow import store +from ..services.bioflow.archive import ( + archive_from_job, + normalize_archive_name, + remove_archive_dir, +) +from ..services.bioflow.reports import extract_figure_refs, locate_report, report_payload +from .bioflow import job_or_404 + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/bioflow", tags=["bioflow-archives"]) + + +def _err(code: str, message: str, status: int) -> JSONResponse: + return JSONResponse(status_code=status, content={ + "ok": False, + "error": {"code": code, "message": message, "degraded": False}, + }) + + +# --------------------------------------------------------------------------- +# 报告登记（BFF 反代调此端点；FR-BIO-016 与 A3 report 模板协调） +# --------------------------------------------------------------------------- + + +@router.post("/reports/register") +async def reports_register(payload: dict[str, Any]) -> dict[str, Any]: + """报告登记接口：BFF 把生信报告登记进 doc_artifacts（字段约定 + job_id/session_id/标题/md_path/图清单——写入报告交 TL 转达 A3）。 + + 生信服务侧只做 payload 归一化与图清单回填；doc_artifacts 落库由 BFF + 反代路由 bioflow_proxy.py 完成（BFF 拥有 docgen 表）。 + """ + job_id = str(payload.get("job_id") or "") + record = await load_record(job_id) if job_id else None + report_path = payload.get("md_path") or payload.get("report_path") or "" + figures: list[str] = list(payload.get("figures") or []) + if record is not None and not figures: + located = locate_report(record.workspace_dir) + if located: + figures = extract_figure_refs(located) + normalized = { + "job_id": job_id, + "session_id": payload.get("session_id") or "", + "title": payload.get("title") or (record.title if record else "生信分析报告"), + "md_path": str(report_path), + "figures": figures, + "registered_at": _now_iso(), + } + return {"ok": True, "data": normalized} + + +# --------------------------------------------------------------------------- +# archives +# --------------------------------------------------------------------------- + + +@router.post("/archives", status_code=201) +async def create_archive(body: ArchiveCreateRequest) -> dict[str, Any]: + """存档（FR-BIO-020/021）：打包任务目录 + 落 bio_archives 行。""" + record = await job_or_404(body.job_id) + ctx: Context = context + name = normalize_archive_name(body.name) + # 重名 409（TC-BIO-API-098 两态断言取拒绝态） + existing = await store.find_archive_by_name(ctx.db, name) + if existing is not None: + return _err("ARCHIVE_EXISTS", f"归档名已存在: {name}", 409) + report_path = locate_report(record.workspace_dir) + result = await archive_from_job( + { + "id": record.id, "session_id": record.session_id, + "title": record.title, "stage_chain": record.stage_chain, + "params": record.params, + }, + record.job_dir, + get_settings().archives_dir_abs, + name=name, + description=body.description, + include_intermediate=body.include_intermediate, + report_artifact=str(report_path) if report_path else None, + ) + await store.insert_archive(ctx.db, { + "id": result.id, + "job_id": record.id, + "name": name, + "description": body.description, + "stage_chain": record.stage_chain, + "params": record.params, + "report_artifact": result.report_artifact, + "disk_path": result.disk_path, + "size_bytes": result.size_bytes, + }) + ctx.events.publish("job_archived", {"job_id": record.id, "archive_id": result.id}) + return {"ok": True, "data": { + "id": result.id, + "job_id": record.id, + "name": name, + "description": body.description, + "disk_path": result.disk_path, + "size_bytes": result.size_bytes, + "report_artifact": result.report_artifact, + }} + + +@router.get("/archives") +async def list_archives( + page: int = Query(default=1, ge=1), + page_size: int = Query(default=20, ge=1, le=100), +) -> dict[str, Any]: + ctx: Context = context + items, total = await store.list_archives(ctx.db, page=page, page_size=page_size) + return {"ok": True, "data": {"items": items, "total": total, "page": page, "page_size": page_size}} + + +@router.get("/archives/{archive_id}") +async def get_archive(archive_id: str) -> dict[str, Any]: + ctx: Context = context + archive = await store.get_archive(ctx.db, archive_id) + if archive is None: + return _err("NOT_FOUND", "存档不存在", 404) + return {"ok": True, "data": archive} + + +@router.delete("/archives/{archive_id}", status_code=204) +async def delete_archive(archive_id: str) -> JSONResponse: + ctx: Context = context + archive = await store.get_archive(ctx.db, archive_id) + if archive is None: + return _err("NOT_FOUND", "存档不存在", 404) + if archive.get("disk_path"): + remove_archive_dir(get_settings().archives_dir_abs, archive_id) + await store.delete_archive(ctx.db, archive_id) + return JSONResponse(status_code=204, content=None) + + +@router.post("/archives/{archive_id}/run", status_code=201) +async def archive_run(archive_id: str, body: ArchiveRunRequest) -> dict[str, Any]: + """复用发起（FR-BIO-023）：装载阶段链+参数 → 新建任务（表单预填存档值）。""" + ctx: Context = context + archive = await store.get_archive(ctx.db, archive_id) + if archive is None: + return _err("ARCHIVE_NOT_FOUND", f"存档不存在: {archive_id}", 404) + from ..services.bioflow.runner import RunnerError + + params = {**dict(archive.get("params") or {}), **dict(body.params or {})} + title = f"{archive.get('name') or '复用任务'}（{body.target or '复用'}）" + try: + record = await ctx.runner.create_job( + body.session_id, title, list(archive.get("stage_chain") or []), + params, archive_id=archive_id, + ) + except RunnerError as exc: + return _err(exc.code, exc.message, 422) + await store.bump_archive_reuse(ctx.db, archive_id) + ctx.records[record.id] = record + from .biojobs import _job_detail + + return {"ok": True, "data": await _job_detail(record)} + + +@router.post("/archives/run", status_code=201) +async def archive_run_by_name(body: ArchiveRunRequest) -> dict[str, Any]: + """按名称匹配复用（FR-BIO-023）：精确优先；模糊给候选；无匹配 404。""" + ctx: Context = context + name = (body.name or "").strip() + if not name: + return _err("VALIDATION", "缺少归档名 name", 422) + archive = await store.find_archive_by_name(ctx.db, name) + if archive is None: + return _err("ARCHIVE_NOT_FOUND", f"未找到匹配存档: {name}", 404) + return await archive_run(archive["id"], ArchiveRunRequest( + session_id=body.session_id, target=body.target, params=body.params or {}, + )) + + +@router.get("/archives/{archive_id}/download") +async def archive_download(archive_id: str) -> FileResponse: + """归档打包下载（zip；TC-BIO-API-141）。""" + ctx: Context = context + archive = await store.get_archive(ctx.db, archive_id) + if archive is None: + return _err("NOT_FOUND", "存档不存在", 404) + disk = Path(archive.get("disk_path") or "") + if not disk.is_dir(): + return _err("NOT_FOUND", "档案磁盘目录缺失", 404) + zip_path = disk.parent / f"{archive_id}.zip" + if not zip_path.exists(): + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + for p in disk.rglob("*"): + if p.is_file(): + zf.write(p, p.relative_to(disk)) + return FileResponse(zip_path, filename=f"{archive.get('name') or archive_id}.zip") + + +@router.get("/archives/{archive_id}/report") +async def archive_report(archive_id: str) -> FileResponse: + """归档报告单独取（HTML；FR-BIO-022 详情回看）。""" + ctx: Context = context + archive = await store.get_archive(ctx.db, archive_id) + if archive is None: + return _err("NOT_FOUND", "存档不存在", 404) + report = archive.get("report_artifact") + if not report or not Path(report).exists(): + return _err("NOT_FOUND", "该存档无报告", 404) + return FileResponse(report, media_type="text/html") + + +def _now_iso() -> str: + from datetime import datetime, timezone + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/bioflow_service/app/routers/bioflow.py b/bioflow_service/app/routers/bioflow.py new file mode 100644 index 0000000..fd41f04 --- /dev/null +++ b/bioflow_service/app/routers/bioflow.py @@ -0,0 +1,276 @@ +"""app/routers/bioflow.py —— 生信服务核心端点（health/chat 流/catalog/知识库咨询）。 + +端点（实施计划 §六清单）： +- GET /api/bioflow/health 探活（前端显隐 FR-BIO-025） +- POST /api/bioflow/chat/stream SSE 入口：意图分流→规划候选→事件流 +- GET /api/bioflow/catalog 阶段目录（?chain=gene_family / ?stage=identify） +- POST /api/bioflow/ask 生信流内流程咨询（知识库引用 FR-BIO-026） + +事件类型全集（12 类）与 seq 严格递增由 app/context 的 EventBus 保证。 +LLM/vstore/docker 全部 mock 时端点行为确定性可断言（L2 层用例）。 +""" +from __future__ import annotations + +import asyncio +import json +import logging +from typing import Any + +from fastapi import APIRouter, Depends, Query, Request +from fastapi.responses import JSONResponse, StreamingResponse + +from ..config import get_settings +from ..context import Context, context, load_record +from ..db import Database, get_db +from ..schemas import ChatStreamRequest, FlowAskRequest +from ..services.bioflow import store +from ..services.bioflow.intent import route as route_intent +from ..services.bioflow.planner import PlannerError, PlanResult + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/bioflow", tags=["bioflow"]) + +KEEPALIVE_INTERVAL = 15.0 +KEEPALIVE_COMMENT = b": keepalive\n\n" + + +def _err(code: str, message: str, status: int) -> JSONResponse: + from fastapi.responses import JSONResponse + + return JSONResponse(status_code=status, content={ + "ok": False, + "error": {"code": code, "message": message, "degraded": False}, + }) + + +def _ctx() -> Context: + return context + + +# --------------------------------------------------------------------------- +# 健康探活（FR-BIO-025；db 连通状态字段 TC-BIO-API-099） +# --------------------------------------------------------------------------- + + +@router.get("/health") +async def health() -> dict[str, Any]: + ctx = _ctx() + db_ok = True + db_error: str | None = None + if ctx.db is not None: + try: + await ctx.db.fetchone("SELECT 1 AS ok") + except Exception as exc: # noqa: BLE001 + db_ok = False + db_error = str(exc) + return { + "ok": True, + "status": "ok" if db_ok else "degraded", + "db": {"connected": db_ok, "error": db_error}, + "version": get_settings().version, + } + + +# --------------------------------------------------------------------------- +# 知识库流程咨询（FR-BIO-026；引用块携带 cite 供前端跳转——A3 渲染） +# --------------------------------------------------------------------------- + + +@router.post("/ask") +async def flow_ask(body: FlowAskRequest) -> dict[str, Any]: + """生信流内流程咨询：FlowStore 检索 + 引用块；无 vstore 时降级空引用。""" + ctx = _ctx() + hits: list[dict[str, Any]] = [] + if ctx.flow_store is not None: + try: + result = ctx.flow_store.search(body.question, top_k=8) + hits = result.get("items") or [] + except Exception: # noqa: BLE001 + hits = [] + citations = [] + for hit in hits: + cite = hit.get("cite") or {} + citations.append({ + "source": hit.get("source") or hit.get("doc_kind") or "", + "title_path": hit.get("title_path") or "", + "path": hit.get("path") or "", + "cite": cite, + "line_start": hit.get("line_start"), + "line_end": hit.get("line_end"), + "text": _hit_text(hit), + }) + return { + "ok": True, + "answer": { + "question": body.question, + "citations": citations, + "degraded": not citations, + }, + } + + +def _hit_text(hit: dict[str, Any]) -> str: + """引用块原文（chunks 明文；检索 meta 不含正文时尝试 get_chunk）。""" + text = str(hit.get("text") or "") + if text: + return text[:1200] + ctx = _ctx() + if ctx.flow_store is not None: + doc = ctx.flow_store.get_chunk(str(hit.get("chunk_key") or hit.get("id") or "")) + if doc: + return str(doc.get("text") or "")[:1200] + return "" + + +# --------------------------------------------------------------------------- +# chat/stream —— SSE 入口（FR-BIO-001/002/003/005/023/026 全链路） +# --------------------------------------------------------------------------- + + +def _sse(event: dict[str, Any]) -> bytes: + """SSE 帧编码：event: 类型 + data: 载荷（不含信封壳，前端 parser 契约）。""" + data = json.dumps(event.get("data") or {}, ensure_ascii=False) + return f"event: {event['type']}\ndata: {data}\n\n".encode("utf-8") + + +def _sse_error(code: str, message: str, seq: int = -1) -> bytes: + payload = {"type": "error", "seq": seq, "data": {"code": code, "message": message}} + return _sse(payload) + + +@router.post("/chat/stream") +async def chat_stream(body: ChatStreamRequest) -> StreamingResponse: + """对话 SSE：意图分流 → 规划（plan_ready）→ 事件订阅流。 + + 非生信消息：message_done（不建任务，TC-BIO-API-005）。 + 规划降级：PLAN_LLM_TIMEOUT / PLAN_UNAVAILABLE / PLAN_FALLBACK（内置兜底链）。 + 客户端断开：生成器取消 → 订阅清理（TC-BIO-API-014）。 + """ + ctx = _ctx() + last_seq = body.last_seq if body.last_seq is not None else -1 + + async def gen(): + # 0. 生信流内流程咨询（has_bioflow_context + FLOW_QA_WORD）→ 知识库引用 + intent = route_intent( + body.message, + has_attachments=body.has_attachments, + has_active_artifact=body.has_active_artifact, + has_bioflow_context=body.has_bioflow_context, + ) + if intent == "bioflow_qa": + qa_payload = await flow_ask(FlowAskRequest(session_id=body.session_id, question=body.message)) + yield _sse({"type": "message_done", "seq": -2, "data": { + "kind": "flow_qa", + "citations": qa_payload["answer"]["citations"], + "degraded": qa_payload["answer"]["degraded"], + }}) + return + if intent != "bioflow": + yield _sse({"type": "message_done", "seq": -2, "data": { + "kind": "qa", "message": "已转交知识问答处理", "routed": intent, + }}) + return + + # 1. 规划（候选路径；复用/LLM/降级） + try: + plan: PlanResult = await ctx.planner.plan(body.message) + except PlannerError as exc: + # 降级为内置兜底链（不中断 SSE；TC-BIO-API-009/020） + from ..services.bioflow.planner import PlanCandidate + + fallback = [s for s in ("identify",) if ctx.catalog.has(s)] + plan = PlanResult( + candidates=[PlanCandidate( + purpose="基因家族鉴定标准流程（内置默认）", + stages=fallback, + )], + source="llm_only", + message=exc.message, + ) + plan_payload = { + "candidates": [{"purpose": c.purpose, "stages": c.stages} for c in plan.candidates], + "source": plan.source, + "auto_filled": plan.auto_filled, + "reuse_archive": plan.reuse_archive_id, + "reuse_name": plan.reuse_name, + "fallback": plan.message or None, + } + # 若该会话已有历史事件流（断线重连走 events 接口），此处 job_id 待前端确认创建 + yield _sse({"type": "plan_ready", "seq": -2, "data": plan_payload}) + + # 2. 后续事件：规划完成后会话级任务可能在别处创建（本接口为规划入口）， + # 保持连接发送心跳；对 job_created 后的任务终态事件做会话级短订阅。 + # P1 语义：前端在此流上收到 plan_ready 后经 /jobs 创建任务， + # 任务事件经 /jobs/{id}/events 订阅；本流保持心跳即可。 + try: + while True: + await asyncio.sleep(KEEPALIVE_INTERVAL) + yield KEEPALIVE_COMMENT + except asyncio.CancelledError: + logger.info("chat_stream: 客户端断开 %s", body.session_id) + + headers = { + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "Connection": "keep-alive", + } + return StreamingResponse(gen(), media_type="text/event-stream", headers=headers) + + +# --------------------------------------------------------------------------- +# catalog（FR-BIO-002/003；catalog 返回 73 阶段 TC-BIO-API-021） +# --------------------------------------------------------------------------- + + +@router.get("/catalog") +async def catalog( + chain: str | None = Query(default=None), + stage: str | None = Query(default=None), +) -> dict[str, Any]: + ctx = _ctx() + cat = ctx.catalog + + if stage: + entry = cat.get(stage) + if entry is None: + return _err("NOT_FOUND", f"阶段不存在: {stage}", 404) + return {"ok": True, "data": {"stage": entry.to_dict()}} + + if chain: + resolved = cat.resolve_chain(chain) + if resolved is None: + return _err("NOT_FOUND", f"链不存在: {chain}", 404) + items = [] + for name in resolved: + entry = cat.get(name) + if entry: + items.append(entry.to_dict()) + return {"ok": True, "data": {"chain": chain, "stages": items, "total": len(items)}} + + items = [] + for name in cat.stage_names(): + entry = cat.get(name) + if entry: + items.append(entry.to_dict()) + return {"ok": True, "data": {"stages": items, "total": len(items)}} + + +# --------------------------------------------------------------------------- +# jobs 通用取记录依赖（供 biojobs/bioarchives 共用） +# --------------------------------------------------------------------------- + + +async def get_record(job_id: str) -> Any | None: + return await load_record(job_id) + + +async def job_or_404(job_id: str) -> Any: + record = await get_record(job_id) + if record is None: + from fastapi import HTTPException + + raise HTTPException(status_code=404, detail={ + "ok": False, "error": {"code": "NOT_FOUND", "message": "任务不存在", "degraded": False}, + }) + return record diff --git a/bioflow_service/app/routers/biojobs.py b/bioflow_service/app/routers/biojobs.py new file mode 100644 index 0000000..f7a4d88 --- /dev/null +++ b/bioflow_service/app/routers/biojobs.py @@ -0,0 +1,485 @@ +"""app/routers/biojobs.py —— 任务端点（jobs 全套 + files 上传 + stages 控制 + events SSE）。 + +端点（实施计划 §六清单）： +- POST /api/bioflow/jobs 创建任务（201；FR-BIO-004） +- GET /api/bioflow/jobs 列表（分页/过滤；FR-BIO-024） +- GET /api/bioflow/jobs/{id} 详情（含 stages） +- DELETE /api/bioflow/jobs/{id} 删除（running 409） +- POST /api/bioflow/jobs/{id}/files 上传（multipart 流式，大文件超时单独调大） + 或服务器路径登记（JSON body） +- GET /api/bioflow/jobs/{id}/files 文件列表 +- GET /api/bioflow/jobs/{id}/files/{fid}/content 文件内容 +- POST /api/bioflow/jobs/{id}/stages/{stage}/confirm 参数确认执行 +- POST /api/bioflow/jobs/{id}/stages/{stage}/rerun 失败重跑（带参数→waiting_confirm） +- POST /api/bioflow/jobs/{id}/stages/{stage}/skip 失败跳过 +- POST /api/bioflow/jobs/{id}/pause / resume 暂停/恢复 +- GET /api/bioflow/jobs/{id}/stages/{stage}/manifest manifest + 日志尾 80 行 +- GET /api/bioflow/jobs/{id}/events SSE 事件流（快照首帧 + 断线回放） +- POST /api/bioflow/report/{job_id}/revise 报告图表修订（FR-BIO-019，最小接口） +""" +from __future__ import annotations + +import asyncio +import json +import logging +import re +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile +from fastapi.responses import FileResponse, JSONResponse, StreamingResponse + +from ..config import get_settings +from ..context import Context, context, load_record +from ..db import Database, get_db +from ..schemas import ( + ArchiveRunRequest, + JobCreateRequest, + PathRegisterRequest, + StageConfirmRequest, + StageRerunRequest, + ReportReviseRequest, +) +from ..services.bioflow import store +from ..services.bioflow.kernel import tail_lines +from ..services.bioflow.manifests import ManifestError, locate_manifest, parse_manifest +from ..services.bioflow.runner import ( + JOB_RUNNING, + STAGE_FAILED, + STAGE_RUNNING, + RunnerError, +) +from .bioflow import job_or_404 + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/bioflow", tags=["bioflow-jobs"]) + +KEEPALIVE_INTERVAL = 15.0 +KEEPALIVE_COMMENT = b": keepalive\n\n" +SNAPSHOT_TYPE = "snapshot" + + +def _err(code: str, message: str, status: int) -> JSONResponse: + return JSONResponse(status_code=status, content={ + "ok": False, + "error": {"code": code, "message": message, "degraded": False}, + }) + + +# --------------------------------------------------------------------------- +# jobs +# --------------------------------------------------------------------------- + + +@router.post("/jobs", status_code=201) +async def create_job(body: JobCreateRequest) -> dict[str, Any]: + """创建任务（FR-BIO-004；未知阶段 422；复用归档 TC-BIO-API-029）。""" + ctx: Context = context + params: dict[str, dict[str, Any]] = dict(body.params or {}) + archive_id = body.archive_id + if body.archive_id: + archive = await store.get_archive(ctx.db, body.archive_id) + if archive is None: + return _err("ARCHIVE_NOT_FOUND", f"存档不存在: {body.archive_id}", 404) + params = {**dict(archive.get("params") or {}), **params} + await store.bump_archive_reuse(ctx.db, body.archive_id) + try: + record = await ctx.runner.create_job( + body.session_id, body.title, body.stages, params, + auto_continue=body.auto_continue, archive_id=archive_id, + ) + except RunnerError as exc: + return _err(exc.code, exc.message, 422) + ctx.records[record.id] = record + return {"ok": True, "data": await _job_detail(record)} + + +async def _job_detail(record: Any) -> dict[str, Any]: + stages = [] + for stage in record.stage_chain: + st = record._stage_states.get(stage, {}) + stages.append({ + "stage": stage, + "status": st.get("status", "waiting"), + "params": st.get("params") or {}, + "attempt": int(st.get("attempt") or 0), + "manifest_path": st.get("manifest_path"), + "log_tail": st.get("log_tail") or "", + "error": st.get("error"), + "started_at": st.get("started_at"), + "finished_at": st.get("finished_at"), + }) + return { + "id": record.id, + "session_id": record.session_id, + "title": record.title, + "status": record.status, + "stages": stages, + "params": record.params, + "auto_continue": record.auto_continue, + "archive_id": record.archive_id, + "progress": record.progress(), + } + + +@router.get("/jobs") +async def list_jobs( + session_id: str | None = Query(default=None), + status: str | None = Query(default=None), + page: int = Query(default=1, ge=1), + page_size: int = Query(default=20, ge=1, le=100), +) -> dict[str, Any]: + ctx: Context = context + items, total = await store.list_jobs( + ctx.db, session_id=session_id, status=status, page=page, page_size=page_size, + ) + for item in items: + stages = await store.list_stages(ctx.db, item["id"]) + done = sum(1 for s in stages if s["status"] in ("done", "skipped")) + total_stages = len(stages) + item["progress"] = int(done * 100 / total_stages) if total_stages else 100 + item["stages_count"] = total_stages + return {"ok": True, "data": {"items": items, "total": total, "page": page, "page_size": page_size}} + + +@router.get("/jobs/{job_id}") +async def get_job(job_id: str) -> dict[str, Any]: + record = await load_record(job_id) + if record is None: + return _err("NOT_FOUND", "任务不存在", 404) + return {"ok": True, "data": await _job_detail(record)} + + +@router.delete("/jobs/{job_id}", status_code=204) +async def delete_job(job_id: str) -> JSONResponse: + record = await load_record(job_id) + if record is None: + return _err("NOT_FOUND", "任务不存在", 404) + if record.status in (JOB_RUNNING,): + return _err("JOB_RUNNING", "任务执行中不可删除", 409) + ctx: Context = context + await store.delete_job(ctx.db, job_id) # bio_stages 级联删除 + ctx.records.pop(job_id, None) + return JSONResponse(status_code=204, content=None) + + +# --------------------------------------------------------------------------- +# files（FR-BIO-006 上传 / FR-BIO-007 路径登记） +# --------------------------------------------------------------------------- + + +def _safe_name(name: str) -> str: + """文件名净化（保留中文；去路径分隔符）。""" + return Path(name or "upload").name or "upload" + + +@router.post("/jobs/{job_id}/files") +async def upload_file( + job_id: str, + request: Request, + file: UploadFile | None = File(default=None), +) -> dict[str, Any]: + """上传（multipart 流式）或服务器路径登记（JSON body）。 + + 大文件超时单独调大：本端点自身逐块流式写盘（不整读内存，TC-BIO-API-041）。 + """ + record = await job_or_404(job_id) + ctx: Context = context + settings = get_settings() + input_dir = record.input_dir + input_dir.mkdir(parents=True, exist_ok=True) + + content_type = request.headers.get("content-type", "") + if "multipart/form-data" not in content_type: + # JSON 路径登记（FR-BIO-007） + try: + body = await request.json() + except Exception: # noqa: BLE001 + return _err("VALIDATION", "请求体须为 multipart 或 JSON 路径登记", 400) + try: + payload = PathRegisterRequest.model_validate(body) + except Exception as exc: # noqa: BLE001 + return _err("VALIDATION", f"路径登记参数不合法: {exc}", 422) + return await _register_path(record, payload.server_path, settings) + + if file is None or not file.filename: + return _err("VALIDATION", "缺少上传文件", 422) + ext = Path(file.filename).suffix.lower() + if ext and ext not in settings.upload_ext_set: + return _err("VALIDATION", f"扩展名不允许: {ext}", 422) + dest = input_dir / _safe_name(file.filename) + written = 0 + max_bytes = int(settings.max_upload_gb * 1024 ** 3) + try: + with dest.open("wb") as fh: + while True: + chunk = await file.read(1024 * 1024) + if not chunk: + break + written += len(chunk) + if written > max_bytes: + fh.close() + dest.unlink(missing_ok=True) + return _err("UPLOAD_TOO_LARGE", + f"超过上传上限 {settings.max_upload_gb}GB（已收 {written} 字节）", 413) + fh.write(chunk) + except Exception as exc: # noqa: BLE001 - 写盘失败不登记（TC-BIO-API-050） + logger.exception("files: 上传写盘失败 %s", job_id) + dest.unlink(missing_ok=True) + return _err("INTERNAL", f"上传写盘失败: {exc}", 500) + file_id = f"{job_id}:{dest.name}" + return {"ok": True, "data": { + "file_id": file_id, "name": dest.name, "size": written, + "path": str(dest), "source": "upload", + }} + + +async def _register_path(record: Any, server_path: str, settings: Any) -> dict[str, Any]: + """服务器路径登记（FR-BIO-007）：限 BIOFLOW_DATA_ROOT/任务目录内。""" + path = Path(server_path).expanduser().resolve() + if not path.exists(): + return _err("PATH_NOT_EXIST", f"路径不存在: {server_path}", 400) + if not path.is_file(): + return _err("NOT_A_FILE", f"非文件: {server_path}", 400) + allowed_roots = [Path(settings.data_root_abs).resolve(), record.job_dir.resolve()] + if not any(path.is_relative_to(root) for root in allowed_roots): + return _err("PATH_NOT_ALLOWED", f"路径不在白名单根内: {server_path}", 403) + return {"ok": True, "data": { + "file_id": f"{record.id}:{path.name}", "name": path.name, + "size": path.stat().st_size, "path": str(path), "source": "server_path", + }} + + +@router.get("/jobs/{job_id}/files") +async def list_files(job_id: str) -> dict[str, Any]: + record = await job_or_404(job_id) + items: list[dict[str, Any]] = [] + input_dir = record.input_dir + if input_dir.is_dir(): + for p in sorted(input_dir.iterdir()): + if p.is_file(): + items.append({ + "file_id": f"{job_id}:{p.name}", "name": p.name, + "size": p.stat().st_size, "path": str(p), + "source": "upload", "registered_at": None, + }) + return {"ok": True, "data": {"items": items, "total": len(items)}} + + +@router.get("/jobs/{job_id}/files/{file_id}/content") +async def file_content(job_id: str, file_id: str) -> FileResponse: + record = await job_or_404(job_id) + name = file_id.split(":", 1)[-1] if ":" in file_id else file_id + path = record.input_dir / _safe_name(name) + if not path.is_file(): + return _err("NOT_FOUND", "文件不存在", 404) + return FileResponse(path) + + +# --------------------------------------------------------------------------- +# stages 控制（confirm/rerun/skip/pause/resume/manifest） +# --------------------------------------------------------------------------- + + +def _stage_exists(record: Any, stage: str) -> None: + if stage not in record.stage_chain: + raise HTTPException(status_code=404, detail={ + "ok": False, "error": {"code": "NOT_FOUND", "message": f"阶段不存在: {stage}", "degraded": False}, + }) + + +@router.post("/jobs/{job_id}/stages/{stage}/confirm") +async def confirm_stage(job_id: str, stage: str, body: StageConfirmRequest) -> dict[str, Any]: + record = await job_or_404(job_id) + _stage_exists(record, stage) + ctx: Context = context + try: + await ctx.runner.confirm_stage(record, stage, dict(body.params or {})) + except RunnerError as exc: + return _err(exc.code, exc.message, 409) + # 确认已受理：执行任务已派生（异步推进中）；当前快照可能仍为 waiting_confirm + return {"ok": True, "data": { + "job_id": job_id, "stage": stage, + "status": "running", + "params": record._stage_states.get(stage, {}).get("params") or {}, + }} + + +@router.post("/jobs/{job_id}/stages/{stage}/rerun") +async def rerun_stage(job_id: str, stage: str, body: StageRerunRequest) -> dict[str, Any]: + record = await job_or_404(job_id) + _stage_exists(record, stage) + ctx: Context = context + try: + await ctx.runner.rerun_stage(record, stage, dict(body.params) if body.params else None) + except RunnerError as exc: + return _err(exc.code, exc.message, 409) + return {"ok": True, "data": { + "job_id": job_id, "stage": stage, + "status": ctx.runner.stage_status(record, stage), + "attempt": record._stage_states.get(stage, {}).get("attempt"), + }} + + +@router.post("/jobs/{job_id}/stages/{stage}/skip") +async def skip_stage(job_id: str, stage: str) -> dict[str, Any]: + record = await job_or_404(job_id) + _stage_exists(record, stage) + ctx: Context = context + try: + await ctx.runner.skip_stage(record, stage) + except RunnerError as exc: + return _err(exc.code, exc.message, 409) + return {"ok": True, "data": { + "job_id": job_id, "stage": stage, + "status": ctx.runner.stage_status(record, stage), + }} + + +@router.post("/jobs/{job_id}/pause") +async def pause_job(job_id: str) -> dict[str, Any]: + record = await job_or_404(job_id) + ctx: Context = context + try: + await ctx.runner.pause_job(record) + except RunnerError as exc: + return _err(exc.code, exc.message, 409) + return {"ok": True, "data": {"job_id": job_id, "status": record.status}} + + +@router.post("/jobs/{job_id}/resume") +async def resume_job(job_id: str) -> dict[str, Any]: + record = await job_or_404(job_id) + ctx: Context = context + try: + await ctx.runner.resume_job(record) + except RunnerError as exc: + return _err(exc.code, exc.message, 409) + return {"ok": True, "data": {"job_id": job_id, "status": record.status}} + + +@router.get("/jobs/{job_id}/stages/{stage}/manifest") +async def stage_manifest(job_id: str, stage: str) -> dict[str, Any]: + """manifest + 日志尾 80 行（FR-BIO-011；TC-BIO-API-071~076）。""" + record = await job_or_404(job_id) + _stage_exists(record, stage) + manifest: dict[str, Any] | None = None + manifest_error: str | None = None + try: + parsed = parse_manifest(record.workspace_dir, stage, None) + manifest = parsed.raw + except ManifestError as exc: + manifest_error = str(exc) + log_tail = "" + log_path = record.logs_dir / f"{stage}.log" + if log_path.exists(): + try: + log_tail = tail_lines(log_path.read_text(encoding="utf-8", errors="replace"), + get_settings().log_tail_lines) + except OSError: + log_tail = "" + if manifest is None and not manifest_error: + manifest_error = "manifest 未产出" + return {"ok": True, "data": { + "job_id": job_id, "stage": stage, + "manifest": manifest, "manifest_error": manifest_error, + "log_tail": log_tail, + "status": record._stage_states.get(stage, {}).get("status"), + }} + + +# --------------------------------------------------------------------------- +# events SSE（FR-BIO-014 断线重连回放） +# --------------------------------------------------------------------------- + + +@router.get("/jobs/{job_id}/events") +async def job_events( + job_id: str, + stage: str | None = Query(default=None), + last_seq: int = Query(default=-1, ge=-1), +) -> StreamingResponse: + """任务事件 SSE：快照首帧 + 历史回放（>last_seq）+ 实时订阅 + 心跳。 + + TC-BIO-API-081：任务终态（done/failed/archived）且回放完成后服务端正常关闭连接。 + """ + record = await job_or_404(job_id) + ctx: Context = context + + async def gen(): + # 1. 快照首帧（TC-BIO-SSE-022） + snap = ctx.events.snapshot(job_id) + if snap is not None: + yield f"event: snapshot\ndata: {json.dumps(snap, ensure_ascii=False)}\n\n".encode() + # 2. 历史回放（>last_seq；TC-BIO-SSE-024） + for ev in ctx.events.replay(job_id, after_seq=last_seq): + if stage and ev.get("data", {}).get("stage") != stage: + continue + yield _sse_bytes(ev) + # 3. 任务终态 → 回放后正常关闭（TC-BIO-API-081） + if record.status in ("done", "failed", "archived"): + return + # 4. 实时订阅 + sub = await ctx.events.subscribe(job_filter=job_id, last_seq=last_seq) + try: + while True: + try: + ev = await asyncio.wait_for(sub.queue.get(), timeout=KEEPALIVE_INTERVAL) + except asyncio.TimeoutError: + yield KEEPALIVE_COMMENT + continue + if int(ev.get("seq") or 0) <= last_seq: + continue + if stage and ev.get("data", {}).get("stage") != stage: + continue + yield _sse_bytes(ev) + finally: + await ctx.events.unsubscribe(sub) + + headers = { + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "Connection": "keep-alive", + } + return StreamingResponse(gen(), media_type="text/event-stream", headers=headers) + + +def _sse_bytes(event: dict[str, Any]) -> bytes: + """SSE 帧编码（data 行只携带载荷，与 bioflow.py 契约一致）。""" + data = json.dumps(event.get("data") or {}, ensure_ascii=False) + return f"event: {event['type']}\ndata: {data}\n\n".encode("utf-8") + + +# --------------------------------------------------------------------------- +# report revise（FR-BIO-019，最小接口：绘图阶段重跑预留；纯文字走 docgen 由 A3 协调） +# --------------------------------------------------------------------------- + + +@router.post("/report/{job_id}/revise") +async def report_revise(job_id: str, body: ReportReviseRequest) -> dict[str, Any]: + """图表修订（FR-BIO-019）：解析图表目标 → 重跑绘图阶段（预留）。 + + P1 最小实现：无绘图目标 → 422（纯文字排版指令由前端走 docgen 编辑链路）。 + 有目标但无对应阶段 → NOT_FOUND。绘图阶段重跑链路（rerun）复用现有端点。 + """ + record = await job_or_404(job_id) + instruction = (body.instruction or "").strip() + target = (body.target or "").strip() + if not target or not re.search(r"图|表|颜色|色", instruction + target): + return _err("VALIDATION", "无明确图表修订目标（纯文字排版指令请走文档编辑）", 422) + # 候选绘图阶段（基因家族链含绘图能力的阶段） + plot_stages = [s for s in ("synteny", "karyotype", "motif", "combo", "gene_family_report") + if s in record.stage_chain] + rerun_stage = None + for s in plot_stages: + if s in target or target in s: + rerun_stage = s + break + if rerun_stage is None: + return _err("NOT_FOUND", "未找到可重跑的绘图阶段（P2 编排层细化）", 404) + return {"ok": True, "data": { + "job_id": job_id, "rerun_stage": rerun_stage, + "target": target, "instruction": instruction, + "params": body.params or {}, + }} diff --git a/bioflow_service/app/schemas.py b/bioflow_service/app/schemas.py new file mode 100644 index 0000000..4bb94ac --- /dev/null +++ b/bioflow_service/app/schemas.py @@ -0,0 +1,112 @@ +"""schemas.py —— 生信服务请求/响应 pydantic 模型（明文白名单文件，同 rag-new 规则）。 + +服务层（app/services/bioflow/）禁止 pydantic（Cython 编译纪律）， +全部 pydantic 模型只放本文件。路由层把模型 .model_dump() 成 dict 再交给服务层。 +""" +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, Field, field_validator + +# ---- chat ---- + +MAX_MESSAGE_CHARS = 100_000 # 超长消息拒绝（TC-BIO-API-015） + + +class ChatStreamRequest(BaseModel): + session_id: str = Field(min_length=1, max_length=128) + message: str = Field(min_length=1, max_length=MAX_MESSAGE_CHARS) + has_attachments: bool = False + has_active_artifact: bool = False + has_bioflow_context: bool = False + last_seq: int | None = Field(default=None, ge=0) + + +class ChatRequest(BaseModel): + session_id: str = Field(min_length=1, max_length=128) + message: str = Field(min_length=1, max_length=MAX_MESSAGE_CHARS) + + +class FlowAskRequest(BaseModel): + """生信流内流程咨询（知识库问答）。""" + + session_id: str = Field(min_length=1, max_length=128) + question: str = Field(min_length=1, max_length=MAX_MESSAGE_CHARS) + + +# ---- jobs ---- + +STAGE_STATUSES = ( + "waiting", "await_files", "waiting_confirm", "queued", "running", + "done", "failed", "skipped", "paused", +) +StageStatus = Literal[ + "waiting", "await_files", "waiting_confirm", "queued", "running", + "done", "failed", "skipped", "paused", +] + +JOB_STATUSES = ( + "created", "await_inputs", "running", "paused", "done", "failed", "archiving", "archived", +) +JobStatus = Literal[ + "created", "await_inputs", "running", "paused", "done", "failed", "archiving", "archived", +] + + +class JobCreateRequest(BaseModel): + session_id: str = Field(min_length=1, max_length=128) + title: str = Field(default="生信任务", max_length=500) + stages: list[str] = Field(min_length=1) + params: dict[str, dict[str, Any]] = Field(default_factory=dict) + auto_continue: bool = False + archive_id: str | None = None + + @field_validator("stages") + @classmethod + def _stages_nonempty_names(cls, v: list[str]) -> list[str]: + if any(not s or not s.strip() for s in v): + raise ValueError("stages 含空阶段名") + return [s.strip() for s in v] + + +class ArchiveRunRequest(BaseModel): + session_id: str = Field(min_length=1, max_length=128) + name: str | None = Field(default=None, max_length=200) + target: str | None = Field(default=None, max_length=2000) + params: dict[str, dict[str, Any]] = Field(default_factory=dict) + + +class StageConfirmRequest(BaseModel): + params: dict[str, Any] = Field(default_factory=dict) + + +class StageRerunRequest(BaseModel): + params: dict[str, Any] | None = None + + +class PauseResumeRequest(BaseModel): + reason: str | None = Field(default=None, max_length=500) + + +class PathRegisterRequest(BaseModel): + server_path: str = Field(min_length=1, max_length=4096) + + +# ---- archives ---- + + +class ArchiveCreateRequest(BaseModel): + job_id: str = Field(min_length=1) + name: str = Field(min_length=1, max_length=200) + description: str = Field(default="", max_length=2000) + include_intermediate: bool = False + + +# ---- report revise ---- + + +class ReportReviseRequest(BaseModel): + target: str | None = Field(default=None, max_length=500) + instruction: str = Field(min_length=1, max_length=2000) + params: dict[str, Any] = Field(default_factory=dict) diff --git a/bioflow_service/app/services/__init__.py b/bioflow_service/app/services/__init__.py new file mode 100644 index 0000000..d7e7239 --- /dev/null +++ b/bioflow_service/app/services/__init__.py @@ -0,0 +1 @@ +"""app/services/__init__.py —— 生信服务 services 包。""" diff --git a/bioflow_service/app/services/bioflow/__init__.py b/bioflow_service/app/services/bioflow/__init__.py new file mode 100644 index 0000000..4c58fd7 --- /dev/null +++ b/bioflow_service/app/services/bioflow/__init__.py @@ -0,0 +1 @@ +"""app/services/bioflow/__init__.py —— 生信服务层包（dataclass/dict/stdlib 纪律）。""" diff --git a/bioflow_service/app/services/bioflow/archive.py b/bioflow_service/app/services/bioflow/archive.py new file mode 100644 index 0000000..706768f --- /dev/null +++ b/bioflow_service/app/services/bioflow/archive.py @@ -0,0 +1,125 @@ +"""app/services/bioflow/archive.py —— 存档/复用（过程记录/中间文件/结果文件/报告打包落盘）。 + +FR-BIO-020/021/022/023： +- 任务完成后可存档：过程记录（events.jsonl）、中间文件（workspace//analysis， + 可选 include_intermediate）、结果文件（final/）、报告（报告链产物 HTML）打包到 + BIOFLOW_ARCHIVES_DIR/{archive_id}/。 +- bio_archives 行登记（名称/磁盘路径/大小/阶段链+参数/报告制品 id）。 +- 名称匹配复用：精确优先，模糊给候选（find_archive_by_name/find_archive_candidates + 在 store；本模块提供 archive_from_job 打包逻辑与名称规范化）。 + +服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。 +""" +from __future__ import annotations + +import logging +import shutil +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +INTERMEDIATE_DIR_NAMES = ("analysis",) + + +@dataclass +class ArchiveResult: + id: str + name: str + disk_path: str + size_bytes: int + report_artifact: str | None + + +def archive_dir_for(archives_root: str | Path, archive_id: str) -> Path: + return Path(archives_root) / archive_id + + +async def archive_from_job( + job: dict[str, Any], + job_dir: str | Path, + archives_root: str | Path, + *, + name: str, + description: str = "", + include_intermediate: bool = False, + report_artifact: str | None = None, +) -> ArchiveResult: + """任务目录 → 档案目录打包（纯 shutil，不依赖 docker/网络）。""" + job_dir = Path(job_dir) + archives_root = Path(archives_root) + archive_id = uuid.uuid4().hex[:16] + dest = archive_dir_for(archives_root, archive_id) + dest.mkdir(parents=True, exist_ok=True) + + # 结果文件（workspace/final/）与过程记录（events.jsonl、logs/） + final_dir = job_dir / "workspace" / "final" + if final_dir.is_dir(): + _copytree_into(final_dir, dest / "final") + logs_dir = job_dir / "logs" + if logs_dir.is_dir(): + _copytree_into(logs_dir, dest / "logs") + events_file = job_dir / "events.jsonl" + if events_file.exists(): + shutil.copy2(events_file, dest / "events.jsonl") + else: + # 事件持久日志在 events_dir/{job_id}.jsonl（与任务目录分离）；归档一并打包 + job_id = job.get("id") or "" + if job_id: + ext_events = job_dir.parent / "events" / f"{job_id}.jsonl" + if ext_events.exists(): + shutil.copy2(ext_events, dest / "events.jsonl") + input_dir = job_dir / "input" + if input_dir.is_dir(): + _copytree_into(input_dir, dest / "input") + + # 中间文件（可选，FR-BIO-020 include_intermediate 开关） + if include_intermediate: + workspace = job_dir / "workspace" + if workspace.is_dir(): + for stage_dir in workspace.iterdir(): + if not stage_dir.is_dir() or stage_dir.name in ("final", "logs"): + continue + analysis = stage_dir / "analysis" + if analysis.is_dir(): + _copytree_into(analysis, dest / "intermediate" / stage_dir.name / "analysis") + + # 报告：任务目录内报告 HTML（final 内 *.html / report.html）随 final 已复制； + # report_artifact 为 doc_artifacts 登记 id（由 reports 模块/BFF 登记后回传）。 + size_bytes = _dir_size(dest) + return ArchiveResult( + id=archive_id, + name=name, + disk_path=str(dest), + size_bytes=size_bytes, + report_artifact=report_artifact, + ) + + +def _copytree_into(src: Path, dest: Path) -> None: + """复制目录内容到 dest（dest 可能已存在，逐项合并）。""" + dest.mkdir(parents=True, exist_ok=True) + for item in src.iterdir(): + target = dest / item.name + if item.is_dir(): + if target.exists(): + _copytree_into(item, target) + else: + shutil.copytree(item, target) + else: + shutil.copy2(item, target) + + +def _dir_size(path: Path) -> int: + return sum(p.stat().st_size for p in path.rglob("*") if p.is_file()) + + +def remove_archive_dir(archives_root: str | Path, archive_id: str) -> None: + shutil.rmtree(archive_dir_for(archives_root, archive_id), ignore_errors=True) + + +def normalize_archive_name(name: str) -> str: + """归档名称规范化（trim；空→占位）。""" + return (name or "").strip() or "未命名存档" diff --git a/bioflow_service/app/services/bioflow/catalog.py b/bioflow_service/app/services/bioflow/catalog.py new file mode 100644 index 0000000..db87890 --- /dev/null +++ b/bioflow_service/app/services/bioflow/catalog.py @@ -0,0 +1,588 @@ +"""app/services/bioflow/catalog.py —— 阶段目录（stage_image_map.json + STAGE_INFO 移植）。 + +- 阶段列表/参数 schema/镜像映射的权威来源：A1 的 stage_image_map.json（不运行时反射）。 +- 中文名/预计耗时：从内核 gfa_core/mas/stage_catalog.py STAGE_INFO 移植的静态快照 + （同一仓库内文件，加载失败降级为空，不阻断）。 +- 前置依赖：从内核 STAGE_DEPENDENCIES 移植（供 planner 前置补全）。 + +服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。 +""" +from __future__ import annotations + +import json +import logging +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +# ---- 从内核 stage_catalog.py 移植的静态快照（2026-09-06，A1 交付树）---- + +# 阶段中文名 + 预计耗时（分钟） +STAGE_INFO_SNAPSHOT: dict[str, dict[str, Any]] = { + "wgs_qc": {"name": "WGS质控", "estimated_time": 30}, + "wgs_index": {"name": "WGS参考索引", "estimated_time": 15}, + "wgs_align": {"name": "WGS序列比对", "estimated_time": 120}, + "wgs_markdup": {"name": "WGS去重复", "estimated_time": 90}, + "wgs_bam_qc": {"name": "WGS BAM质控", "estimated_time": 30}, + "wgs_haplotype_calling": {"name": "WGS单样本变异检测", "estimated_time": 120}, + "wgs_gvcf_combine": {"name": "WGS GVCF合并", "estimated_time": 60}, + "wgs_joint_genotyping": {"name": "WGS联合分型", "estimated_time": 90}, + "wgs_variant_select": {"name": "WGS变异类型选择", "estimated_time": 30}, + "wgs_variant_filter": {"name": "WGS变异硬过滤", "estimated_time": 30}, + "wgs_impute": {"name": "WGS基因型填充", "estimated_time": 60}, + "wgs_maf_missing_filter": {"name": "WGS MAF与缺失率过滤", "estimated_time": 30}, + "pop_pca": {"name": "群体PCA分析", "estimated_time": 30}, + "pop_phylogeny": {"name": "群体系统树分析", "estimated_time": 45}, + "pop_structure": {"name": "群体结构分析", "estimated_time": 90}, + "pop_ld_decay": {"name": "群体LD衰减分析", "estimated_time": 60}, + "pop_snp_density": {"name": "SNP密度分析", "estimated_time": 20}, + "gwas_fmt": {"name": "GWAS VCF排序与HapMap转换", "estimated_time": 20}, + "gwas_kinship": {"name": "GWAS TASSEL亲缘关系矩阵", "estimated_time": 20}, + "gwas_trait": {"name": "GWAS表型数据准备", "estimated_time": 5}, + "gwas_cov": {"name": "GWAS群体结构协变量准备", "estimated_time": 5}, + "gwas_mlm": {"name": "GWAS TASSEL MLM/CMLM", "estimated_time": 60}, + "gwas_plot": {"name": "GWAS MLM结果绘图", "estimated_time": 10}, + "gwas_emmax": {"name": "GWAS EMMAX关联分析", "estimated_time": 60}, + "gwas_emplot": {"name": "GWAS EMMAX结果绘图", "estimated_time": 10}, + "gwas_glm": {"name": "GWAS TASSEL GLM", "estimated_time": 60}, + "identify": {"name": "基因家族鉴定", "estimated_time": 5}, + "identify_refine": {"name": "修订鉴定成员", "estimated_time": 5}, + "subfamily_confirm": {"name": "确认亚家族分类", "estimated_time": 3}, + "phylogeny": {"name": "系统发育分析", "estimated_time": 30}, + "reference_classification": {"name": "参考亚家族分类", "estimated_time": 5}, + "motif": {"name": "Motif分析", "estimated_time": 20}, + "domain": {"name": "结构域分析", "estimated_time": 15}, + "structure": {"name": "基因结构分析", "estimated_time": 10}, + "promoter": {"name": "启动子分析", "estimated_time": 20}, + "tfbs": {"name": "TF结合位点预测", "estimated_time": 30}, + "synteny": {"name": "种内共线性分析", "estimated_time": 30}, + "karyotype": {"name": "核型分析", "estimated_time": 10}, + "interspecies_synteny": {"name": "物种间共线性", "estimated_time": 40}, + "expression": {"name": "表达分析", "estimated_time": 15}, + "kaks": {"name": "Ka/Ks分析", "estimated_time": 20}, + "ppi": {"name": "PPI分析", "estimated_time": 10}, + "pep": {"name": "蛋白特征分析", "estimated_time": 10}, + "ortholog": {"name": "直系同源基因鉴定", "estimated_time": 15}, + "msa": {"name": "多序列比对", "estimated_time": 10}, + "enrichment": {"name": "富集分析", "estimated_time": 15}, + "combo": {"name": "组合图", "estimated_time": 10}, + "gene_family_report": {"name": "基因家族结果报告", "estimated_time": 2}, + "genome_prepare": {"name": "基因组准备", "estimated_time": 10}, + "ncbi_fetch_sequences": {"name": "NCBI序列下载", "estimated_time": 5}, + "sequence_extract_by_id": {"name": "按ID提取序列", "estimated_time": 5}, + "msa_region_plot": {"name": "MSA区间绘图", "estimated_time": 3}, + "synteny_plot_revise": {"name": "种内共线性Circos重绘", "estimated_time": 3}, + "promoter_plot_revise": {"name": "启动子自定义绘图", "estimated_time": 3}, + "prepare_organize": {"name": "数据文件整理", "estimated_time": 5}, + "prepare_normalize_gff": {"name": "GFF ID规范化", "estimated_time": 5}, + "prepare_longest_transcript": {"name": "提取最长转录本", "estimated_time": 10}, + "prepare_extract_pep": {"name": "从GFF提取PEP", "estimated_time": 10}, + "prepare_extract_cds": {"name": "从GFF提取CDS", "estimated_time": 10}, + "prepare_filter_fasta": {"name": "按ID提取FASTA", "estimated_time": 5}, + "prepare_simplify_ncbi": {"name": "NCBI ID简化", "estimated_time": 5}, + "rnaseq_qc": {"name": "RNA-seq质控", "estimated_time": 30}, + "rnaseq_align": {"name": "RNA-seq比对", "estimated_time": 60}, + "rnaseq_quant": {"name": "RNA-seq定量", "estimated_time": 30}, + "rnaseq_sample_qc": {"name": "RNA-seq样本质量评估", "estimated_time": 10}, + "rnaseq_de": {"name": "差异表达分析（自动生成FPKM表达图）", "estimated_time": 20}, + "rnaseq_wgcna": {"name": "RNA-seq WGCNA分析", "estimated_time": 30}, + "annotation": {"name": "背景注释", "estimated_time": 180}, + "rnaseq_go_enrich": {"name": "RNA-seq GO富集分析", "estimated_time": 15}, + "rnaseq_kegg_enrich": {"name": "RNA-seq KEGG富集分析", "estimated_time": 15}, + "rnaseq_enrich": {"name": "RNA-seq富集分析", "estimated_time": 15}, + "rnaseq_fgsea": {"name": "RNA-seq fgsea分析", "estimated_time": 15}, + "rnaseq_candidate_family": {"name": "RNA-seq候选家族筛选", "estimated_time": 20}, + "rnaseq_report": {"name": "RNA-seq结果报告", "estimated_time": 2}, +} + +# 前置依赖（STAGE_DEPENDENCIES 快照；planner 前置补全依据） +STAGE_DEPENDENCIES_SNAPSHOT: dict[str, list[str]] = { + "wgs_qc": [], "wgs_index": [], "wgs_align": ["wgs_qc", "wgs_index"], + "wgs_markdup": ["wgs_align"], "wgs_bam_qc": ["wgs_markdup", "wgs_index"], + "wgs_haplotype_calling": ["wgs_markdup", "wgs_bam_qc", "wgs_index"], + "wgs_gvcf_combine": ["wgs_haplotype_calling", "wgs_index"], + "wgs_joint_genotyping": ["wgs_gvcf_combine", "wgs_index"], + "wgs_variant_select": ["wgs_joint_genotyping"], + "wgs_variant_filter": ["wgs_variant_select"], "wgs_impute": ["wgs_variant_filter"], + "wgs_maf_missing_filter": ["wgs_impute"], + "pop_pca": [], "pop_phylogeny": [], "pop_structure": [], "pop_ld_decay": [], + "pop_snp_density": [], "gwas_fmt": [], "gwas_kinship": ["gwas_fmt"], + "gwas_trait": [], "gwas_cov": [], "gwas_mlm": ["gwas_fmt", "gwas_kinship", "gwas_trait", "gwas_cov"], + "gwas_plot": ["gwas_mlm"], "gwas_emmax": ["gwas_fmt", "gwas_trait"], + "gwas_emplot": ["gwas_emmax"], "gwas_glm": ["gwas_fmt", "gwas_trait", "gwas_cov"], + "identify": [], "identify_refine": [], "subfamily_confirm": ["reference_classification"], + "genome_prepare": [], "ncbi_fetch_sequences": [], "sequence_extract_by_id": [], + "msa_region_plot": [], "tree_plot_revise": [], "synteny_plot_revise": [], + "promoter_plot_revise": [], "prepare_organize": [], "prepare_normalize_gff": [], + "prepare_longest_transcript": [], "prepare_extract_pep": [], "prepare_extract_cds": [], + "prepare_filter_fasta": [], "prepare_simplify_ncbi": [], "rnaseq_qc": [], + "phylogeny": ["identify"], + "reference_classification": ["identify", "phylogeny", "motif", "structure", "domain", "combo"], + "synteny": ["identify"], "pep": ["identify"], "msa": ["identify"], + "motif": ["identify", "phylogeny"], "domain": ["identify", "phylogeny"], + "structure": ["identify", "phylogeny"], "promoter": ["identify", "phylogeny"], + "tfbs": ["genome_prepare", "identify"], "expression": ["identify", "phylogeny"], + "enrichment": ["identify", "phylogeny"], "ortholog": ["identify"], + "interspecies_synteny": ["identify"], "karyotype": ["identify", "synteny"], + "ppi": ["identify"], "kaks": ["identify"], + "combo": ["identify", "phylogeny", "motif", "structure", "domain"], + "rnaseq_align": ["rnaseq_qc"], "rnaseq_quant": ["rnaseq_align"], + "rnaseq_sample_qc": ["rnaseq_quant"], "rnaseq_de": ["rnaseq_sample_qc"], + "rnaseq_wgcna": ["rnaseq_sample_qc"], "annotation": [], + "rnaseq_enrich": ["rnaseq_de"], "rnaseq_fgsea": ["rnaseq_de"], + "rnaseq_go_enrich": ["rnaseq_de"], "rnaseq_kegg_enrich": ["rnaseq_de"], + "rnaseq_candidate_family": ["rnaseq_de", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea"], + "rnaseq_report": [], +} + +GENE_FAMILY_CHAIN = [ + "genome_prepare", "identify", "identify_refine", "phylogeny", + "reference_classification", "subfamily_confirm", "motif", "domain", "structure", + "combo", "promoter", "tfbs", "synteny", "interspecies_synteny", "karyotype", + "pep", "ortholog", "kaks", "ppi", "gene_family_report", "expression", +] + +CHAIN_ALIASES = {"gene_family": GENE_FAMILY_CHAIN, "gf": GENE_FAMILY_CHAIN, + "wgs": None, "rnaseq": None} + + +@dataclass +class StageDef: + """阶段目录条目（catalog 端点返回形态）。""" + + name: str + description: str + image: str + module: str + executable: bool + estimated_time: int + params: list[dict[str, Any]] = field(default_factory=list) + inputs: list[dict[str, Any]] = field(default_factory=list) + outputs: list[dict[str, Any]] = field(default_factory=list) + note: str = "" + dependencies: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "description": self.description, + "image": self.image, + "module": self.module, + "executable": self.executable, + "estimated_time": self.estimated_time, + "params": self.params, + "inputs": self.inputs, + "outputs": self.outputs, + "note": self.note, + "dependencies": self.dependencies, + } + + +# ---- 基因家族链参数 schema（手写；参数 help 文案自入口 argparse 提取整理）---- + +_GENOME_PREPARE_PARAMS = [ + {"name": "species", "type": "string", "default": "", "source": "user", + "help": "目标物种代码（如 Ft），基因组数据文件名中的物种标识"}, + {"name": "suffix", "type": "string", "default": "", "source": "user", + "help": "文件后缀过滤（如 .fa.gz），留空自动匹配"}, +] +_IDENTIFY_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀，例如 Ft.SPL（输出文件命名空间）"}, + {"name": "required_pfams", "type": "string", "default": "", "source": "contract", + "help": "逗号分隔的 Pfam accession，如 PF03110（家族识别条件）"}, + {"name": "dom_i_evalue_max", "type": "float", "default": 1e-3, "source": "contract", + "help": "domain i-Evalue threshold（结构域独立 E 值阈值）"}, + {"name": "hmm_cov_min", "type": "float", "default": 0.3, "source": "contract", + "help": "HMM coverage threshold（HMM 覆盖度阈值）"}, + {"name": "blast_evalue", "type": "float", "default": 1e-7, "source": "contract", + "help": "Blast E-value threshold（BLAST E 值阈值）"}, + {"name": "blast_identity_min", "type": "float", "default": 30.0, "source": "contract", + "help": "minimum BLAST percent identity（参考蛋白命中最低一致度）"}, + {"name": "hmm_combine_mode", "type": "choice", "choices": ["intersection", "union"], + "default": "union", "source": "contract", + "help": "多 HMM 文件整合方式（交集/并集），默认并集"}, + {"name": "hmm_blast_merge_mode", "type": "choice", "choices": ["intersection", "union"], + "default": "", "source": "contract", "help": "HMM 和 BLAST 结果整合方式"}, + {"name": "cut_ga", "type": "boolean", "default": False, "source": "contract", + "help": "use Pfam GA thresholds（使用 Pfam GA 阈值）"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数（--cpus 注入）"}, + {"name": "memory", "type": "choice", "choices": ["16g", "32g", "64g"], "default": "32g", + "source": "runtime", "help": "容器内存上限（--memory 注入）"}, +] +_PHYLOGENY_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀（同 identify）"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, + {"name": "memory", "type": "choice", "choices": ["16g", "32g", "64g"], "default": "32g", + "source": "runtime", "help": "容器内存上限"}, +] +_MOTIF_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, + {"name": "memory", "type": "choice", "choices": ["16g", "32g", "64g"], "default": "32g", + "source": "runtime", "help": "容器内存上限"}, +] +_DOMAIN_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_STRUCTURE_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_COMBO_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_PROMOTER_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "upstream", "type": "integer", "default": 2000, "source": "contract", + "help": "启动子上游区间长度（bp）"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_TFBS_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_SYNTENY_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_INTERSPECIES_SYNTENY_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "ref_species", "type": "string", "default": "", "source": "user", + "help": "参考物种代码（跨物种共线性）"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_KARYOTYPE_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_PEP_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_ORTHOLOG_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "ref_species", "type": "string", "default": "", "source": "user", + "help": "参考物种代码（直系同源鉴定）"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_KAKS_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_PPI_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "species_code", "type": "string", "default": "", "source": "user", + "help": "物种代码（STRING 蛋白互作）"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_REFERENCE_CLASSIFICATION_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_SUBFAMILY_CONFIRM_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_EXPRESSION_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_REPORT_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, +] +_RNASEQ_QC_PARAMS = [ + {"name": "samples", "type": "string", "default": "", "source": "user", + "help": "样本列表（逗号分隔）"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "8", + "source": "runtime", "help": "容器 CPU 数"}, +] +_RNASEQ_ALIGN_PARAMS = [ + {"name": "samples", "type": "string", "default": "", "source": "user", + "help": "样本列表（逗号分隔）"}, + {"name": "ref_genome", "type": "string", "default": "", "source": "user", + "help": "参考基因组路径/索引名"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "8", + "source": "runtime", "help": "容器 CPU 数"}, +] +_RNASEQ_QUANT_PARAMS = [ + {"name": "samples", "type": "string", "default": "", "source": "user", + "help": "样本列表（逗号分隔）"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "6", + "source": "runtime", "help": "容器 CPU 数"}, +] +_RNASEQ_SAMPLE_QC_PARAMS = [ + {"name": "samples", "type": "string", "default": "", "source": "user", + "help": "样本列表（逗号分隔）"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_RNASEQ_DE_PARAMS = [ + {"name": "design", "type": "string", "default": "", "source": "user", + "help": "实验设计矩阵文件（对比组声明）"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_RNASEQ_WGCNA_PARAMS = [ + {"name": "design", "type": "string", "default": "", "source": "user", + "help": "实验设计矩阵文件"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_RNASEQ_ENRICH_PARAMS = [ + {"name": "gene_list", "type": "string", "default": "", "source": "contract", + "help": "差异基因列表（上游 DE 结果）"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_RNASEQ_CANDIDATE_FAMILY_PARAMS = [ + {"name": "family_pfams", "type": "string", "default": "", "source": "user", + "help": "候选家族 Pfam accession"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", + "source": "runtime", "help": "容器 CPU 数"}, +] +_RNASEQ_REPORT_PARAMS = [ + {"name": "prefix", "type": "string", "default": "", "source": "contract", + "help": "分析前缀"}, +] + +# 通用输入/输出描述（契约文档摘要；契约文件为权威，这里给目录层最小描述） +_IDENTIFY_INPUTS = [ + {"name": "protein_fasta", "type": "fasta", "required": True, + "description": "物种全蛋白质 FASTA（input_dir 内查找）"}, + {"name": "hmm_file", "type": "hmm", "required": False, + "description": "用户上传 HMM 优先；否则按 required_pfams 从 Pfam-A.hmm 获取（BBA_PFAM_DIR）"}, +] +_IDENTIFY_OUTPUTS = [ + {"name": "final_members", "type": "table", "description": "final/{prefix}.identified_members.id"}, + {"name": "final_proteins", "type": "fasta", "description": "final/{prefix}.identified_members.pep.fa"}, + {"name": "identify_report", "type": "markdown", "description": "final/{prefix}.identify_report.md"}, + {"name": "manifest", "type": "manifest", "description": "final/{prefix}.identify.manifest.json"}, +] +_GENOME_PREPARE_INPUTS = [ + {"name": "genome_fasta", "type": "fasta", "required": False, + "description": "{species}.genome.fasta.gz"}, + {"name": "pep_fasta", "type": "fasta", "required": False, + "description": "{species}.pep.fasta.gz"}, + {"name": "cds_fasta", "type": "fasta", "required": False, + "description": "{species}.cds.fasta.gz"}, + {"name": "gff3", "type": "gff3", "required": False, "description": "{species}.gff3.gz"}, +] +_GENOME_PREPARE_OUTPUTS = [ + {"name": "normalized_pep", "type": "fasta", "description": "{species}.pep.fasta"}, + {"name": "normalized_cds", "type": "fasta", "description": "{species}.cds.fasta"}, + {"name": "normalized_gff", "type": "gff3", "description": "{species}_final.gff3"}, + {"name": "manifest", "type": "manifest", "description": "{prefix}.genome_prepare.manifest.json"}, +] + +# 下游阶段通用描述（目录层最小；契约文档权威） +_DOWNSTREAM_INPUTS = [ + {"name": "final_members", "type": "table", "required": True, + "description": "上游 identify 的 final_members.id（跨阶段输入）"}, +] +_DOWNSTREAM_OUTPUTS = [ + {"name": "manifest", "type": "manifest", "description": "final/{prefix}.{stage}.manifest.json"}, + {"name": "report", "type": "markdown", "description": "final/{prefix}.{stage}_report.md"}, +] + +# stage → (params, inputs, outputs) 手写表 +_STAGE_PARAM_SCHEMAS: dict[str, tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]] = { + "genome_prepare": (_GENOME_PREPARE_PARAMS, _GENOME_PREPARE_INPUTS, _GENOME_PREPARE_OUTPUTS), + "identify": (_IDENTIFY_PARAMS, _IDENTIFY_INPUTS, _IDENTIFY_OUTPUTS), + "identify_refine": ([ + {"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", "source": "runtime", "help": "容器 CPU 数"}, + ], _IDENTIFY_INPUTS, _DOWNSTREAM_OUTPUTS), + "phylogeny": (_PHYLOGENY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "reference_classification": (_REFERENCE_CLASSIFICATION_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "subfamily_confirm": (_SUBFAMILY_CONFIRM_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "motif": (_MOTIF_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "domain": (_DOMAIN_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "structure": (_STRUCTURE_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "combo": (_COMBO_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "promoter": (_PROMOTER_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "tfbs": (_TFBS_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "synteny": (_SYNTENY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "interspecies_synteny": (_INTERSPECIES_SYNTENY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "karyotype": (_KARYOTYPE_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "pep": (_PEP_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "ortholog": (_ORTHOLOG_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "kaks": (_KAKS_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "ppi": (_PPI_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "expression": (_EXPRESSION_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "gene_family_report": (_REPORT_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "msa": ([ + {"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", "source": "runtime", "help": "容器 CPU 数"}, + ], _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "enrichment": ([ + {"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"}, + ], _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_qc": (_RNASEQ_QC_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_align": (_RNASEQ_ALIGN_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_quant": (_RNASEQ_QUANT_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_sample_qc": (_RNASEQ_SAMPLE_QC_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_de": (_RNASEQ_DE_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_wgcna": (_RNASEQ_WGCNA_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_enrich": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_go_enrich": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_kegg_enrich": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_fgsea": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_candidate_family": (_RNASEQ_CANDIDATE_FAMILY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "rnaseq_report": (_RNASEQ_REPORT_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), + "annotation": ([ + {"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"}, + {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "8", "source": "runtime", "help": "容器 CPU 数"}, + ], _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS), +} + + +class StageCatalog: + """阶段目录（数据来自 stage_image_map.json + 手写快照，不运行时反射）。""" + + def __init__(self, image_map: Any) -> None: + self._map = image_map # ImageMap 实例（kernel 模块） + + # ---- 查询 ---- + + def stage_names(self) -> list[str]: + return self._map.all_stage_names() + + def family_chain(self) -> list[str]: + return list(self._map.family_chain) or GENE_FAMILY_CHAIN + + def has(self, stage: str) -> bool: + return stage in self._map.stages + + def validate_chain(self, stages: list[str]) -> list[str]: + """链硬校验：返回未知阶段名列表（空=全合法）。""" + return [s for s in stages if not self.has(s)] + + def get(self, stage: str) -> StageDef | None: + """返回阶段目录条目（未知阶段 None）。""" + if not self.has(stage): + return None + entry = self._map.image_for(stage) or {} + params, inputs, outputs = _STAGE_PARAM_SCHEMAS.get( + stage, ([], _DOWNSTREAM_INPUTS if stage not in ("identify", "genome_prepare") else [], []) + ) + if stage not in _STAGE_PARAM_SCHEMAS: + params, inputs, outputs = ( + [{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], + "default": "4", "source": "runtime", "help": "容器 CPU 数"}], + _DOWNSTREAM_INPUTS if stage not in ("identify", "genome_prepare") else [], + _DOWNSTREAM_OUTPUTS, + ) + info = STAGE_INFO_SNAPSHOT.get(stage, {}) + return StageDef( + name=stage, + description=str(info.get("name") or stage), + image=str(entry.get("image") or ""), + module=str(entry.get("module") or ""), + executable=bool(entry.get("executable")), + estimated_time=int(info.get("estimated_time") or 0), + params=params, + inputs=inputs, + outputs=outputs, + note=str(entry.get("note") or ""), + dependencies=STAGE_DEPENDENCIES_SNAPSHOT.get(stage, []), + ) + + def resolve_chain(self, chain: str | None) -> list[str] | None: + """chain 别名解析（gene_family 链 21 阶段）；None 表示无该链。""" + if not chain: + return None + alias = CHAIN_ALIASES.get(chain) + if alias is None: + return None + return list(alias) + + def auto_fill_prerequisites(self, stages: list[str]) -> tuple[list[str], list[str]]: + """前置补全（FR-BIO-003）：缺失的前置阶段自动补全。 + + 返回 (补全后的链, 被补全的阶段名列表)。 + 链内阶段按依赖拓扑顺序排列（依赖在前）。 + """ + missing: list[str] = [] + added: list[str] = [] + chain_set = set(stages) + # 反复补全直到闭合（依赖的依赖也补） + changed = True + while changed: + changed = False + for s in list(stages): + for dep in STAGE_DEPENDENCIES_SNAPSHOT.get(s, []): + if dep not in chain_set and self.has(dep): + chain_set.add(dep) + missing.append(dep) + added.append(dep) + changed = True + ordered = topo_order([s for s in GENE_FAMILY_CHAIN if s in chain_set] + or list(chain_set)) + return ordered, added + + def dependencies(self, stage: str) -> list[str]: + return STAGE_DEPENDENCIES_SNAPSHOT.get(stage, []) + + +def topo_order(stages: list[str]) -> list[str]: + """按依赖拓扑排序（依赖在前，保持输入相对顺序稳定）。""" + result: list[str] = [] + seen: set[str] = set() + + def visit(s: str) -> None: + if s in seen: + return + seen.add(s) + for dep in STAGE_DEPENDENCIES_SNAPSHOT.get(s, []): + visit(dep) + result.append(s) + + for s in stages: + visit(s) + return result + + +def build_catalog(image_map: Any) -> StageCatalog: + return StageCatalog(image_map) diff --git a/bioflow_service/app/services/bioflow/events.py b/bioflow_service/app/services/bioflow/events.py new file mode 100644 index 0000000..578962f --- /dev/null +++ b/bioflow_service/app/services/bioflow/events.py @@ -0,0 +1,202 @@ +"""app/services/bioflow/events.py —— 进程内事件总线（pub/sub 供 SSE）+ 持久事件日志。 + +- 发布：runner/路由在任务状态变化时 publish(type, data)（事件类型清单 = 实施计划 §六 + 12 类 + 内部 job_recovered）。 +- 订阅：async queue（供 SSE 端点）；内存环形缓冲（断线重连回放）。 +- 持久：events/{job_id}.jsonl 追加落盘（服务重启恢复回放；FR-BIO-031）。 +- 快照：snapshot(job_id) 构造断线重连首帧（TC-BIO-SSE-022/024）。 + +服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。 +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import threading +from collections import deque +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +EVENT_TYPES = ( + "job_created", "plan_ready", "stage_started", "stage_log", "stage_done", + "stage_failed", "stage_skipped", "stage_await_confirm", "stage_await_files", + "job_done", "job_paused", "job_archived", "job_recovered", +) + +# 断线重连时需要回放的关键状态事件（含历史事件全量） +REPLAYABLE_TYPES = set(EVENT_TYPES) + +MEMORY_BUFFER_LIMIT = 500 +SNAPSHOT_TYPE = "snapshot" + + +@dataclass(eq=False) +class Subscriber: + """订阅者（eq=False：身份哈希，可入 set）。""" + + queue: asyncio.Queue = field(default_factory=asyncio.Queue) + last_seq: int = -1 + job_filter: str | None = None + + +class EventBus: + """进程内事件总线：publish → 订阅队列广播 + 内存环形缓冲 + jsonl 持久日志。""" + + def __init__(self, events_dir: str | Path) -> None: + self.events_dir = Path(events_dir) + self.events_dir.mkdir(parents=True, exist_ok=True) + self._subscribers: set[Subscriber] = set() + self._sub_lock = asyncio.Lock() + self._buffer: deque[dict[str, Any]] = deque(maxlen=MEMORY_BUFFER_LIMIT) + self._seq = 0 + self._file_lock = threading.Lock() + + # ---- 内部 ---- + + def _events_file(self, job_id: str) -> Path: + return self.events_dir / f"{job_id}.jsonl" + + def _append_file(self, event: dict[str, Any]) -> None: + try: + job_id = str((event.get("data") or {}).get("job_id") or "") + with self._file_lock: + with self._events_file(job_id).open("a", encoding="utf-8") as fh: + fh.write(json.dumps(event, ensure_ascii=False) + "\n") + except OSError: + logger.warning("bioflow events: 持久化失败 %s", event.get("type")) + + # ---- 发布 ---- + + def publish(self, event_type: str, data: dict[str, Any]) -> dict[str, Any]: + """发布事件（同步）：返回完整事件（含 seq），并唤醒所有订阅者。""" + if event_type not in EVENT_TYPES: + raise ValueError(f"未知事件类型: {event_type}") + self._seq += 1 + event: dict[str, Any] = { + "type": event_type, + "seq": self._seq, + "data": dict(data), + } + self._buffer.append(event) + self._append_file(event) + # 广播给订阅者（跨线程安全：queue.put_nowait 本身线程安全） + for sub in list(self._subscribers): + if sub.job_filter is not None and sub.job_filter != str(data.get("job_id") or ""): + continue + try: + sub.queue.put_nowait(event) + except asyncio.QueueFull: # pragma: no cover - 防御 + logger.warning("bioflow events: 订阅队列满，丢弃事件 %s", event_type) + return event + + def publish_async(self, event_type: str, data: dict[str, Any]) -> None: + """异步发布（供路由协程调用，publish 本身为同步安全实现）。""" + self.publish(event_type, data) + + # ---- 订阅（SSE 端点）---- + + async def subscribe(self, job_filter: str | None = None, last_seq: int = -1) -> Subscriber: + sub = Subscriber(last_seq=last_seq, job_filter=job_filter) + async with self._sub_lock: + self._subscribers.add(sub) + return sub + + async def unsubscribe(self, sub: Subscriber) -> None: + async with self._sub_lock: + self._subscribers.discard(sub) + + # ---- 断线重连回放 ---- + + def snapshot(self, job_id: str) -> dict[str, Any] | None: + """构造任务快照（首帧 data.type=snapshot；TC-BIO-SSE-022）。 + + 从持久日志读该任务全部事件，拼出 job + stages 当前状态。 + 事件不存在（未知任务）返回 None。 + """ + path = self._events_file(job_id) + if not path.exists(): + return None + events: list[dict[str, Any]] = [] + try: + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + ev = json.loads(line) + if ev.get("type") in REPLAYABLE_TYPES: + events.append(ev) + except ValueError: + continue + except OSError: + return None + if not events: + return None + # 由事件序列推导 job/stages 状态 + job_status = "created" + stage_states: dict[str, dict[str, Any]] = {} + for ev in events: + t, d = ev["type"], ev.get("data") or {} + if t == "job_created": + job_status = "created" + for s in d.get("stages") or []: + stage_states.setdefault(str(s), {"stage": str(s), "status": "waiting"}) + elif t == "stage_started": + st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))}) + st["status"] = "running" + elif t == "stage_done": + st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))}) + st["status"] = "done" + elif t == "stage_failed": + st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))}) + st["status"] = "failed" + elif t == "stage_skipped": + st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))}) + st["status"] = "skipped" + elif t == "stage_await_confirm": + st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))}) + st["status"] = "waiting_confirm" + elif t == "stage_await_files": + st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))}) + st["status"] = "await_files" + elif t == "job_done": + job_status = "done" + elif t == "job_paused": + job_status = "paused" + elif t == "job_archived": + job_status = "archived" + return { + "type": SNAPSHOT_TYPE, + "job_id": job_id, + "job": {"status": job_status}, + "stages": [stage_states[k] for k in stage_states], + "last_seq": events[-1]["seq"] if events else 0, + } + + def replay(self, job_id: str, after_seq: int = -1) -> list[dict[str, Any]]: + """回放任务历史事件（seq > after_seq；断线重连补齐；TC-BIO-SSE-023/024）。""" + path = self._events_file(job_id) + if not path.exists(): + return [] + out: list[dict[str, Any]] = [] + try: + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + ev = json.loads(line) + if ev.get("type") in REPLAYABLE_TYPES and int(ev.get("seq") or 0) > after_seq: + out.append(ev) + except (ValueError, TypeError): + continue + except OSError: + return [] + return out + + def memory_replay(self, after_seq: int = -1) -> list[dict[str, Any]]: + """内存环形缓冲回放（进程重启后内存为空，此时走持久回放）。""" + return [ev for ev in self._buffer if int(ev.get("seq") or 0) > after_seq] diff --git a/bioflow_service/app/services/bioflow/intent.py b/bioflow_service/app/services/bioflow/intent.py new file mode 100644 index 0000000..5b45604 --- /dev/null +++ b/bioflow_service/app/services/bioflow/intent.py @@ -0,0 +1,218 @@ +"""app/services/bioflow/intent.py —— 生信意图分流（词典唯一事实源 docs/team/cases_bioflow_intent.md v1.0）。 + +服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。 +route(text, has_attachments, has_active_artifact, has_bioflow_context) + -> "qa" | "docgen" | "edit" | "export" | "bioflow" | "bioflow_qa" +判定顺序与词典一致（优先级从上到下，命中即返回；漏判安全）。 +""" +from __future__ import annotations + +# ---- 词典（与 cases_bioflow_intent.md §3 逐条一致，小写不敏感比较）---- + +EXPORT_WORD = ["导出", "转成", "转为", "转换成", "下载成", "输出为", "存为", "转pdf", "转word"] +FORMAT_WORD = ["pdf", "docx", "doc", "word"] + +EDIT_WORD = [ + "改", "修改", "调整", "删", "删除", "加一段", "补充", "润色", + "缩写", "扩写", "重写", "改写", "优化", "加个", "增加一段", +] + +CHART_WORD = [ + "图表", "图颜色", "颜色换成", "图改成", "柱状图", "热图", "火山图", "曼哈顿图", + "韦恩图", "气泡图", "散点图", "折线图", "改成红色", "换成蓝色", "颜色", +] + +# 图表对象词（FR-BIO-019 触发：活动文档 + 图表对象词 → 生信图表调整） +CHART_OBJECT_WORD = [ + "图表", "图颜色", "颜色换成", "图改成", "柱状图", "热图", "火山图", "曼哈顿图", + "韦恩图", "气泡图", "散点图", "折线图", +] + +# 家族名语境词（词典用例表：家族名后须跟语境词或位于句尾才判定为家族命中， +# 如「SPL 会议」不命中、「玉米 bZIP」句尾命中） +FAMILY_CONTEXT = ("家族", "因子", "基因家族", "转录因子", "family", "成员", "基因") + +# docgen 写文档意图词典（沿用 docgen intent.py 全表，cases_intent.md 冻结） +WRITE_VERB = ["写", "撰写", "拟", "起草", "编制", "生成", "编写", "拟定", "草拟", "制定", "整理"] + +DOC_TYPE = [ + "论文", "报告", "通知", "通告", "公告", "请示", "批复", "函", "申请书", + "申请", "申报书", "标书", "合同", "协议", "方案", "计划", "总结", "汇报", + "讲话稿", "致辞", "证明", "纪要", "简报", "材料", "说明书", "指南", "文档", + "申报材料", "工作总结", "调研报告", "可行性报告", "项目书", "策划书", + "演讲稿", "新闻稿", "文案", "简历", "制度", "规定", "办法", "细则", "章程", + "声明", "邀请函", "感谢信", "慰问信", "介绍信", "推荐信", "倡议书", + "述职报告", "开题报告", "结题报告", "实验报告", "分析报告", "会议纪要", +] + +DOC_TYPE_EXCLUDE = ["介绍", "内容", "代码", "诗", "作业", "问题", "建议", "想法"] + +# 生信词典补充「文档类型」表达（基因家族分析报告等，与 DOC_TYPE 并集） +_BIO_DOC_TYPE_EXTRA = ["分析报告", "研究论文"] + +FLOW_QA_WORD = [ + "需要什么输入", "有什么输入", "输入是什么", "需要什么参数", "参数是什么", + "什么参数", "怎么设置", "如何设置", "什么软件", "用什么软件", "前置条件", + "介绍", "是什么", "怎么跑", "怎么做", "需要准备什么输入", "需要准备什么参数", +] + +ANALYZE_VERB = [ + "跑", "跑一下", "分析", "鉴定", "识别", "研究", "做个", "做一个", "做一下", "做", + "执行", "运行", "run", "identify", +] + +# 分析对象词（按词典长词优先；「做」动词 + 对象组合） +ANALYSIS_OBJECT = [ + "rna-seq", "rnaseq", "转录组", "差异表达", "wgs", "全基因组测序", "全基因组", + "变异检测", "gwas", "全基因组关联", "关联分析", "群体遗传", "遗传多样性", + "进化", "系统发育", "选择性清除", "群体结构", "群体", "拷贝数变异", "基因预测", + "基因注释", "结构注释", "功能注释", "宏基因组", +] + +# 英文家族名 → 中文语境等价词（英文句「run rice SPL family identification」 +# 需 family/identification 等语境） +_FAMILY_EN_CONTEXT = ("family", "gene", "factor", "identification", "analysis", "member") + +FAMILY_NAME = [ + "spl", "sbp", "mads", "mads-box", "madsbox", "wrky", "nac", "myb", "bzip", "bhlh", + "ap2", "erf", "gras", "tcp", "hd-zip", "hdzip", "arf", "aux", "iaa", "dof", "gata", + "hsf", "lbd", "lob", "trihelix", "e2f", "cpp", "nf-y", "nfy", "c2h2", "zf-hd", + "zfhd", "far1", "frs", "hb", "wox", "pin", "mate", "abc", "pp2c", "snrk", "cdpk", + "mapk", "rlck", "nbs", "lrr", "nlr", "tps", "cyp", "gst", "ugt", "转录因子家族", + "基因家族", +] + +ANALYSIS_OBJECT = [ + "rna-seq", "rnaseq", "rna-seq", "转录组", "差异表达", "wgs", "全基因组", "变异检测", + "全基因组测序", "gwas", "关联分析", "全基因组关联", "群体", "群体遗传", "遗传多样性", + "进化", "系统发育", "选择性清除", "群体结构", "拷贝数变异", "基因预测", "基因注释", + "结构注释", "功能注释", "宏基因组", +] + +REUSE_PHRASE = ["存档任务的方法", "存档的方法"] + + +def _contains_any_longest_first(text_lower: str, words: list[str]) -> bool: + """词表按长度降序匹配（长词优先，避免「改」命中「改成红色」类图表词）。""" + for w in sorted(words, key=len, reverse=True): + if w in text_lower: + return True + return False + + +def _contains_any(text_lower: str, words: list[str]) -> bool: + return _contains_any_longest_first(text_lower, words) + + +def _has_doc_type(text_lower: str) -> bool: + """含 DOC_TYPE 任一，且不含 DOC_TYPE_EXCLUDE 任一（排除优先，同 docgen）。""" + if _contains_any(text_lower, DOC_TYPE_EXCLUDE): + return False + return _contains_any(text_lower, DOC_TYPE) or _contains_any(text_lower, _BIO_DOC_TYPE_EXTRA) + + +def extract_reuse_name(text: str) -> str | None: + """提取存档复用句式的「方法名」（REUSE_PHRASE 之前最近的 3-20 字符片段）。 + + FR-BIO-023：「用 XXX 存档任务的方法帮我跑 xxxx」→ 方法名 = XXX。 + 返回 None 表示未命中复用句式。 + """ + text_lower = text.lower() + pos = -1 + phrase = "" + for p in REUSE_PHRASE: + idx = text_lower.rfind(p) + if idx > pos: + pos, phrase = idx, p + if pos < 0: + return None + before = text[:pos].strip() + # 取短语前最近的 3-20 字符片段（剔除「用/根据/参考/按」等引导词） + for lead in ("根据", "参考", "用", "按", "按照", "《", "》", "的"): + before = before.replace(lead, " ") + parts = [s for s in before.replace("《", " ").replace("》", " ").split() if s.strip()] + if not parts: + return None + name = parts[-1] + if 3 <= len(name) <= 20: + return name + return None + + +def route( + text: str, + has_attachments: bool = False, + has_active_artifact: bool = False, + has_bioflow_context: bool = False, +) -> str: + """生信意图分流主入口（判定顺序同词典 §2，命中即返回；漏判安全）。""" + t = text.strip() + if not t: + return "qa" + tl = t[:2000].lower() # 边界约定：>2000 字符按截断前 2000 判定 + + # 1. export（需 has_active_artifact） + if has_active_artifact and _contains_any(tl, EXPORT_WORD) and _contains_any(tl, FORMAT_WORD): + return "export" + + # 2. edit（需 has_active_artifact；含图表对象词跳过 → 生信 FR-BIO-019） + if ( + has_active_artifact + and _contains_any(tl, EDIT_WORD) + and not _contains_any(tl, CHART_OBJECT_WORD) + ): + return "edit" + + # 2b. 图表调整走生信（FR-BIO-019）：活动文档 + 图表对象词 → bioflow + if has_active_artifact and _contains_any(tl, CHART_OBJECT_WORD): + return "bioflow" + + # 3. docgen（写文档意图；DOC_TYPE_EXCLUDE 排除优先） + if _contains_any(tl, WRITE_VERB) and _has_doc_type(tl): + return "docgen" + + # 4. docgen（附件信号） + if has_attachments and _contains_any(tl, WRITE_VERB): + return "docgen" + + # 5. bioflow_qa（生信流内流程咨询；FLOW_QA 命中即拦，不建任务） + if has_bioflow_context and _contains_any(tl, FLOW_QA_WORD): + return "bioflow_qa" + + # 5b. 流程咨询句式（无上下文）：词典用例表 #18「跑 SPL 家族鉴定需要准备什么输入？」 + # 期望 qa——FLOW_QA 词命中即拦（漏判安全），不进 bioflow + if _contains_any(tl, FLOW_QA_WORD): + return "qa" + + has_analyze = _contains_any(tl, ANALYZE_VERB) + has_object = _family_hit(tl) or _contains_any(tl, ANALYSIS_OBJECT) + + # 6. bioflow（A 分析动词 且 B 家族名/分析对象；C 复用句式同判） + if has_analyze and has_object: + return "bioflow" + + # 7. 漏判安全：不命中绝不进生信流 + return "qa" + + +def _family_hit(text_lower: str) -> bool: + """家族名命中：词典词 + 语境词（中文）或英文语境（family/gene 等）。 + + 家族名后须跟语境词或位于句尾（词典用例表：SPL 会议不命中、玉米 bZIP 句尾命中）。 + """ + for name in sorted(FAMILY_NAME, key=len, reverse=True): + idx = text_lower.find(name) + if idx < 0: + continue + tail = text_lower[idx + len(name):] + tail_stripped = tail.lstrip() + # 中文语境：家族名后紧跟语境词 + if tail_stripped.startswith(FAMILY_CONTEXT): + return True + # 英文语境（family/identification 等在句内任意位置即算） + if _contains_any(text_lower, _FAMILY_EN_CONTEXT) and "family" in text_lower: + return True + # 句尾（家族名后仅空白/标点） + if not tail_stripped.strip("，。！？；：,.!?;:（）() "): + return True + return False diff --git a/bioflow_service/app/services/bioflow/kernel.py b/bioflow_service/app/services/bioflow/kernel.py new file mode 100644 index 0000000..ee70239 --- /dev/null +++ b/bioflow_service/app/services/bioflow/kernel.py @@ -0,0 +1,288 @@ +"""app/services/bioflow/kernel.py —— Docker 执行器（阶段→镜像映射表驱动）。 + +职责： +- 读 A1 交付的 stage_image_map.json（路径来自配置 BIOFLOW_IMAGE_MAP_PATH， + config 按 __file__ 推导 repo 根；零硬编码）。 +- 按阶段→镜像查表拼装 docker run 命令（list 形式，无 shell；命令拼装可 mock 断言）。 +- 容器内统一 `python -B -m cli.run_{stage}`（模块名来自映射表 module 字段）。 +- --user 宿主 uid（BIOFLOW_HOST_UID 环境变量，默认服务进程 os.getuid()）。 +- -v 任务目录 + BIOFLOW_DATA 只读挂载 + -e BBA_* + --cpus/--memory + --rm。 +- 超时 kill；日志全量落任务 logs/，返回尾部截断文本。 + +服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。执行经 async subprocess +（真跑 docker 用默认实现；测试注入 fake 执行器 mock docker）。 +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import shlex +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +CONTAINER_CMD_PREFIX = ("python", "-B") # 容器内命令形态（TC-BIO-RUN-037） + + +class ImageMapError(Exception): + """镜像映射表缺失/非法（服务可用但所有 docker 执行前置拦截）。""" + + +class UnknownStageError(Exception): + """阶段不在映射表（catalog 校验后仍未知 → 执行前置拦截）。""" + + +@dataclass +class ImageMap: + """stage_image_map.json 内存形态。""" + + images: dict[str, dict[str, Any]] = field(default_factory=dict) + stages: dict[str, dict[str, Any]] = field(default_factory=dict) + family_chain: list[str] = field(default_factory=list) + + def image_for(self, stage: str) -> dict[str, Any] | None: + return self.stages.get(stage) + + def docker_tag(self, image: str) -> str: + return str(self.images.get(image, {}).get("docker_tag") or image) + + def all_stage_names(self) -> list[str]: + return sorted(self.stages.keys()) + + +@dataclass +class KernelConfig: + """docker 执行配置（由 app.config 注入；测试可构造最小实例）。""" + + host_uid: int = 0 + docker_tag_suffix: str = "dev" + data_root: str = "" + container_data_dir: str = "/bioflow-data" + container_input_dir: str = "/job/input" + container_workspace_dir: str = "/job/workspace" + default_cpus: str = "4" + default_memory: str = "32g" + stage_timeout_sec: float = 36000.0 + log_tail_lines: int = 80 + bba_env: dict[str, str] = field(default_factory=dict) + cpu_allow: str = "4,8,16,32" + memory_allow: str = "16g,32g,64g" + + +class Kernel: + """Docker 执行内核：映射表加载 + 命令拼装 + 阶段执行。""" + + def __init__(self, image_map: ImageMap, config: KernelConfig) -> None: + self.map = image_map + self.cfg = config + + # ---- 映射表加载（类方法，供应用启动）---- + + @classmethod + def load(cls, image_map_path: str | Path, config: KernelConfig) -> "Kernel": + path = Path(image_map_path) + if not path.exists(): + raise ImageMapError(f"stage_image_map.json 缺失: {path}") + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (ValueError, OSError) as exc: + raise ImageMapError(f"stage_image_map.json 解析失败: {exc}") from exc + image_map = ImageMap( + images=raw.get("images") or {}, + stages=raw.get("stages") or {}, + family_chain=raw.get("family_chain") or [], + ) + return cls(image_map, config) + + # ---- 查表 ---- + + def image_for(self, stage: str) -> dict[str, Any] | None: + return self.map.image_for(stage) + + def has_stage(self, stage: str) -> bool: + return stage in self.map.stages + + # ---- 命令拼装（纯函数式，可 mock 断言）---- + + def build_command( + self, + stage: str, + job_input_dir: str | Path, + job_workspace_dir: str | Path, + params: dict[str, Any] | None = None, + *, + cpu: str | None = None, + memory: str | None = None, + module: str | None = None, + ) -> list[str]: + """拼装 docker run 命令（list，无 shell；TC-BIO-KERN-001 基元）。 + + 顺序：docker,run,--rm,--user,uid,-v(×3),--cpus,--memory,-e(×N), + tag:镜像,python,-B,-m,cli.run_X,[参数…] + """ + params = params or {} + entry = self.map.stages.get(stage) + if entry is None: + raise UnknownStageError(f"阶段不在映射表: {stage}") + image = str(entry.get("image") or "") + if not image: + raise UnknownStageError(f"阶段 {stage} 无镜像映射") + mod = module or entry.get("module") + if not mod: + raise UnknownStageError(f"阶段 {stage} 无执行模块") + + cmd: list[str] = ["docker", "run", "--rm", "--user", str(self.cfg.host_uid)] + cmd += ["-v", f"{job_input_dir}:{self.cfg.container_input_dir}"] + cmd += ["-v", f"{job_workspace_dir}:{self.cfg.container_workspace_dir}"] + if self.cfg.data_root: + cmd += ["-v", f"{self.cfg.data_root}:{self.cfg.container_data_dir}:ro"] + eff_cpu = cpu or str(params.get("cpu") or "") + eff_memory = memory or str(params.get("memory") or "") + cmd += ["--cpus", self._safe_cpu(eff_cpu or self.cfg.default_cpus)] + cmd += ["--memory", self._safe_memory(eff_memory or self.cfg.default_memory)] + for key in sorted(self.cfg.bba_env): + cmd += ["-e", f"{key}={self.cfg.bba_env[key]}"] + # 用户自定义 env 追加（params 中 env 键） + extra_env = params.get("env") if isinstance(params.get("env"), dict) else {} + for key in sorted(extra_env): + cmd += ["-e", f"{key}={extra_env[key]}"] + tag = self.map.docker_tag(image) + cmd.append(f"{tag}:{self.cfg.docker_tag_suffix}") + cmd.extend(CONTAINER_CMD_PREFIX) + cmd.extend(["-m", mod]) + # 通用容器内参数：input-dir/workspace/cpu（各入口 argparse 兼容） + cmd += ["--input-dir", self.cfg.container_input_dir] + cmd += ["--workspace", self.cfg.container_workspace_dir] + for key in sorted(params): + value = params[key] + if key in ("env", "cpu", "memory"): + continue # cpu/memory 走 --cpus/--memory；env 走 -e + if value is None: + continue + if isinstance(value, bool): + if value: + cmd.append(f"--{key.replace('_', '-')}") + elif isinstance(value, (list, tuple)): + cmd.append(f"--{key.replace('_', '-')}") + cmd.append(",".join(str(v) for v in value)) + else: + cmd.append(f"--{key.replace('_', '-')}") + cmd.append(str(value)) + return cmd + + def _safe_cpu(self, cpu: str) -> str: + if str(cpu).strip() in {c.strip() for c in self.cfg.cpu_allow.split(",")}: + return str(cpu).strip() + logger.warning("bioflow kernel: cpu=%s 不在白名单 %s，回落默认 %s", + cpu, self.cfg.cpu_allow, self.cfg.default_cpus) + return self.cfg.default_cpus + + def _safe_memory(self, memory: str) -> str: + if str(memory).strip() in {m.strip() for m in self.cfg.memory_allow.split(",")}: + return str(memory).strip() + logger.warning("bioflow kernel: memory=%s 不在白名单 %s，回落默认 %s", + memory, self.cfg.memory_allow, self.cfg.default_memory) + return self.cfg.default_memory + + @staticmethod + def command_display(cmd: list[str]) -> str: + """命令可读形态（日志用；list 执行保证无 shell 注入）。""" + return " ".join(shlex.quote(str(c)) for c in cmd) + + # ---- 执行 ---- + + async def run_stage( + self, + stage: str, + job_input_dir: str | Path, + job_workspace_dir: str | Path, + params: dict[str, Any] | None = None, + *, + log_path: str | Path | None = None, + cpu: str | None = None, + memory: str | None = None, + ) -> dict[str, Any]: + """执行一个阶段 docker：返回 {exit_code, timed_out, log_tail, cmd}。 + + - 超时：terminate → 3s → kill（TC-BIO-KERN-009） + - 日志全量写 log_path（任务 logs/），返回尾部 log_tail_lines 行 + - 退出码非 0 / 超时 → 失败语义由 runner 判定（本函数只报事实） + """ + params = params or {} + cpu = cpu or str(params.get("cpu") or "") + memory = memory or str(params.get("memory") or "") + cmd = self.build_command(stage, job_input_dir, job_workspace_dir, params, + cpu=cpu or None, memory=memory or None) + log_path = Path(log_path) if log_path else Path(job_workspace_dir) / "logs" / f"{stage}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + timed_out = False + try: + stdout, _ = await asyncio.wait_for( + proc.communicate(), timeout=self.cfg.stage_timeout_sec + ) + except asyncio.TimeoutError: + timed_out = True + try: + proc.terminate() + try: + await asyncio.wait_for(proc.wait(), timeout=3.0) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + except ProcessLookupError: + pass + stdout = b"" + exit_code = proc.returncode + text = (stdout or b"").decode("utf-8", errors="replace") + log_path.write_text(text, encoding="utf-8") + return { + "exit_code": exit_code, + "timed_out": timed_out, + "log_tail": tail_lines(text, self.cfg.log_tail_lines), + "cmd": cmd, + } + + +def tail_lines(text: str, n: int) -> str: + """返回文本最后 n 行（日志截断；TC-BIO-RUN-013）。""" + lines = text.splitlines() + if len(lines) <= n: + return "\n".join(lines) + return "\n".join(lines[-n:]) + + +def _default_kernel_config() -> KernelConfig: + """从 app.config 推导（默认 uid = 服务进程 os.getuid()）。""" + from ...config import get_settings + + s = get_settings() + return KernelConfig( + host_uid=s.host_uid, + docker_tag_suffix=s.docker_image_tag_suffix, + data_root=s.data_root_abs, + container_data_dir=s.container_data_dir, + container_input_dir=s.container_input_dir, + container_workspace_dir=s.container_workspace_dir, + default_cpus=s.default_cpus, + default_memory=s.default_memory, + stage_timeout_sec=s.stage_timeout_sec, + log_tail_lines=s.log_tail_lines, + bba_env=s.bba_env, + ) + + +def load_kernel() -> Kernel: + """应用启动：加载映射表 + 配置推导（映射表损坏 → ImageMapError 上抛）。""" + from ...config import get_settings + + return Kernel.load(get_settings().image_map_abs, _default_kernel_config()) diff --git a/bioflow_service/app/services/bioflow/llm.py b/bioflow_service/app/services/bioflow/llm.py new file mode 100644 index 0000000..fc6e424 --- /dev/null +++ b/bioflow_service/app/services/bioflow/llm.py @@ -0,0 +1,154 @@ +"""app/services/bioflow/llm.py —— 生信规划 LLM 调用（沿用 docgen deepseek 直连模式）。 + +DOCGEN_LLM_* 键（DOCGEN_LLM_BASE_URL/API_KEY/MODEL/TIMEOUT）+ BIOFLOW_LLM_PLAN_TIMEOUT +规划调用超时。complete_json 带一次 JSON 修复重试；重试指数退避（同 docgen）。 +密钥只存在于本进程，绝不进入任何 API 响应/日志。 +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import re +from typing import Any + +import httpx + +logger = logging.getLogger(__name__) + +BACKOFF_SECONDS = (1.5, 3.0, 6.0) +MAX_ATTEMPTS = 4 + + +class LLMError(Exception): + """LLM 调用失败（重试尽/响应异常）。""" + + +class LLMTimeout(LLMError): + """规划 LLM 超时（chat 流降级 PLAN_LLM_TIMEOUT）。""" + + +class LLMUnavailable(LLMError): + """LLM 完全不可用（连接失败/未配置 key；降级 PLAN_UNAVAILABLE）。""" + + +def llm_config() -> dict[str, Any]: + return { + "base_url": (os.environ.get("DOCGEN_LLM_BASE_URL") or "https://api.deepseek.com/v1").rstrip("/"), + "api_key": os.environ.get("DOCGEN_LLM_API_KEY", ""), + "model": os.environ.get("DOCGEN_LLM_MODEL") or "deepseek-chat", + "timeout": float(os.environ.get("DOCGEN_LLM_TIMEOUT") or "300"), + } + + +def _extract_text(payload: dict[str, Any]) -> str: + try: + choices = payload.get("choices") or [] + if choices: + content = choices[0].get("message", {}).get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + return "".join(str(c.get("text", "")) if isinstance(c, dict) else str(c) for c in content) + except Exception: + pass + return "" + + +async def _chat_once( + client: httpx.AsyncClient, + cfg: dict[str, Any], + system: str, + user: str, + *, + temperature: float, + max_tokens: int, + json_mode: bool, + timeout: float, +) -> str: + headers = {"Content-Type": "application/json"} + if cfg["api_key"]: + headers["Authorization"] = f"Bearer {cfg['api_key']}" + body: dict[str, Any] = { + "model": cfg["model"], + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "temperature": temperature, + "max_tokens": max_tokens, + } + if json_mode: + body["response_format"] = {"type": "json_object"} + resp = await client.post( + f"{cfg['base_url']}/chat/completions", + headers=headers, + json=body, + timeout=httpx.Timeout(connect=10.0, read=timeout, write=30.0, pool=10.0), + ) + if resp.status_code == 401 or resp.status_code == 403: + raise LLMUnavailable("LLM 鉴权失败") + if resp.status_code >= 500: + raise LLMError(f"LLM 5xx: {resp.status_code}") + if resp.status_code >= 400: + raise LLMError(f"LLM 4xx: {resp.status_code}") + return _extract_text(resp.json()) + + +def _repair_json(text: str) -> Any: + """一次 JSON 修复重试：剥 code fence、截取首尾大括号。""" + cleaned = text.strip() + fence = re.match(r"^```(?:json)?\s*(.*?)\s*```$", cleaned, re.S) + if fence: + cleaned = fence.group(1).strip() + try: + return json.loads(cleaned) + except ValueError: + pass + start, end = cleaned.find("{"), cleaned.rfind("}") + if start >= 0 and end > start: + try: + return json.loads(cleaned[start:end + 1]) + except ValueError: + pass + raise ValueError("LLM 输出非 JSON") + + +async def complete_json( + system: str, + user: str, + *, + temperature: float = 0.3, + max_tokens: int = 4000, + timeout: float | None = None, +) -> Any: + """JSON 输出 LLM 调用：指数退避 + 一次 JSON 修复。 + + timeout=None 用配置默认（规划用 BIOFLOW_LLM_PLAN_TIMEOUT 更短）。 + """ + cfg = llm_config() + if not cfg["api_key"]: + raise LLMUnavailable("LLM 未配置（DOCGEN_LLM_API_KEY 为空）") + call_timeout = float(timeout if timeout is not None else cfg["timeout"]) + last_exc: Exception | None = None + async with httpx.AsyncClient() as client: + for attempt in range(MAX_ATTEMPTS): + try: + text = await _chat_once( + client, cfg, system, user, + temperature=temperature, max_tokens=max_tokens, + json_mode=True, timeout=call_timeout, + ) + return _repair_json(text) + except asyncio.TimeoutError as exc: + last_exc = exc + raise LLMTimeout("规划 LLM 超时") from exc + except (httpx.ConnectError, httpx.ConnectTimeout) as exc: + last_exc = exc + raise LLMUnavailable(f"LLM 不可用: {exc}") from exc + except (LLMError, ValueError) as exc: + last_exc = exc + if attempt < MAX_ATTEMPTS - 1: + await asyncio.sleep(BACKOFF_SECONDS[min(attempt, len(BACKOFF_SECONDS) - 1)]) + raise LLMError(f"LLM 调用失败（重试尽）: {last_exc}") diff --git a/bioflow_service/app/services/bioflow/manifests.py b/bioflow_service/app/services/bioflow/manifests.py new file mode 100644 index 0000000..cb529bc --- /dev/null +++ b/bioflow_service/app/services/bioflow/manifests.py @@ -0,0 +1,157 @@ +"""app/services/bioflow/manifests.py —— manifest 解析与校验（schema 随内核树）。 + +- 按 gfa_core/schemas/core/manifest_v1.schema.json 必需字段解析 + （manifest_version/job_id/stage/status/artifacts/parameters/metrics…）。 +- 定位规则（同内核 cli 行为）：final/{prefix}.{stage}.manifest.json 优先， + final/ 下任意 *.manifest.json 兜底。 +- 跨阶段输入定位：上游 manifest 的 artifacts 路径（相对 workspace_root 解析 + 为容器内路径，供下游 docker -v/参数拼接）。 + +服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。 +""" +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +REQUIRED_MANIFEST_KEYS = ( + "manifest_version", "job_id", "stage", "status", "artifacts", +) + +# 内核命名约定（workspace_contract_v1.md）：final/{prefix}.{stage}.manifest.json +MANIFEST_GLOB = "*.manifest.json" + + +@dataclass +class Manifest: + """解析后的 manifest（原始 dict 保留，供透传）。""" + + stage: str + status: str + path: str + raw: dict[str, Any] = field(default_factory=dict) + + @property + def artifacts(self) -> dict[str, Any]: + return self.raw.get("artifacts") or {} + + @property + def parameters(self) -> dict[str, Any]: + return self.raw.get("parameters") or {} + + @property + def metrics(self) -> dict[str, Any]: + return self.raw.get("metrics") or {} + + @property + def prefix(self) -> str: + return str(self.raw.get("prefix") or "") + + +class ManifestError(Exception): + """manifest 缺失/非法 JSON/必需字段缺失。""" + + +def locate_manifest(workspace: str | Path, stage: str, prefix: str | None = None) -> Path | None: + """在 workspace final/ 下定位阶段 manifest（prefix + 阶段名双匹配）。 + + 不存在返回 None（runner 据此判 reason=no_manifest）。 + 注意：无兜底——其他阶段的 manifest 绝不误配给当前阶段 + （重启恢复按 manifest 判成败时曾因兜底把 identify 的 manifest 判给 + phylogeny，导致 running 阶段被误判 done 不重跑）。 + """ + final_dir = Path(workspace) / "final" + if not final_dir.is_dir(): + return None + candidates = list(final_dir.glob(MANIFEST_GLOB)) + if not candidates: + return None + if prefix: + for cand in candidates: + if cand.name.startswith(prefix) and f".{stage}." in cand.name: + return cand + for cand in candidates: + if f".{stage}." in cand.name: + return cand + return None + + +def parse_manifest(workspace: str | Path, stage: str, prefix: str | None = None) -> Manifest: + """定位并解析 manifest；失败抛 ManifestError（含具体原因）。""" + path = locate_manifest(workspace, stage, prefix) + if path is None: + raise ManifestError(f"manifest 缺失: workspace={workspace} stage={stage}") + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except ValueError as exc: + raise ManifestError(f"manifest 非法 JSON: {path}") from exc + except OSError as exc: + raise ManifestError(f"manifest 读取失败: {path}") from exc + if not isinstance(raw, dict): + raise ManifestError(f"manifest 非对象: {path}") + missing = [k for k in REQUIRED_MANIFEST_KEYS if k not in raw] + if missing: + raise ManifestError(f"manifest 缺必需字段 {missing}: {path}") + status = str(raw.get("status") or "") + if status not in ("success", "failed", "partial_success"): + raise ManifestError(f"manifest status 非法: {status!r}") + return Manifest(stage=stage, status=status, path=str(path), raw=raw) + + +def locate_artifacts(workspace: str | Path, stage: str, prefix: str | None = None) -> dict[str, Any]: + """解析上游 manifest 并返回 artifacts（跨阶段输入定位）。 + + 解析失败返回 {}（调用方决定 fallback：上游失败则下游不应执行）。 + """ + try: + m = parse_manifest(workspace, stage, prefix) + return m.artifacts + except ManifestError: + return {} + + +def artifact_paths(artifacts: dict[str, Any]) -> list[str]: + """artifacts → 文件路径列表（full_path 优先，path/relpath_or_hint 兜底）。""" + out: list[str] = [] + for value in artifacts.values(): + if isinstance(value, str): + if value: + out.append(value) + continue + if isinstance(value, dict): + for key in ("full_path", "path", "relpath_or_hint"): + v = value.get(key) + if v: + out.append(str(v)) + break + return out + + +def input_resolution( + upstream_workspace: str | Path, + upstream_stage: str, + prefix: str | None = None, +) -> dict[str, Any]: + """下游阶段的跨阶段输入解析结果。 + + 返回 {resolved: bool, inputs: [路径…], manifest_path: str|None, error: str|None} + """ + try: + m = parse_manifest(upstream_workspace, upstream_stage, prefix) + except ManifestError as exc: + return {"resolved": False, "inputs": [], "manifest_path": None, "error": str(exc)} + paths = artifact_paths(m.artifacts) + existing = [p for p in paths if Path(p).exists()] + if not existing: + return { + "resolved": False, + "inputs": [], + "manifest_path": m.path, + "error": f"上游 {upstream_stage} artifacts 文件均不存在", + } + return {"resolved": True, "inputs": existing, "manifest_path": m.path, "error": None} diff --git a/bioflow_service/app/services/bioflow/params.py b/bioflow_service/app/services/bioflow/params.py new file mode 100644 index 0000000..65d6da5 --- /dev/null +++ b/bioflow_service/app/services/bioflow/params.py @@ -0,0 +1,54 @@ +"""app/services/bioflow/params.py —— 参数预填（BBA_* 历史/契约文档 → 表单默认值）。 + +- 契约文档（gfa_core/schemas/stages/*_contract_v1.md）解析参数默认值 + （parameters 小节 key=value 形式）。 +- BBA_* 环境变量历史（进程内 config.bba_env 派生）。 +- 存档参数预填（FR-BIO-023：表单预填存档值——由 runner._form_defaults 用 + record.params 完成，本模块提供读取存档参数的工具）。 + +服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。 +""" +from __future__ import annotations + +import logging +import re +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +# 契约文档 parameters 小节匹配（`- key` 或 `key:` 列表项） +_PARAM_RE = re.compile(r"^\s*[-*]\s+([a-z_][a-z0-9_]*)\b", re.M) + + +def read_contract_defaults(contract_path: str | Path) -> dict[str, Any]: + """契约文档 → 参数默认值快照（解析失败返回 {}，不阻断）。""" + path = Path(contract_path) + if not path.exists(): + return {} + try: + text = path.read_text(encoding="utf-8") + except OSError: + return {} + defaults: dict[str, Any] = {} + for key in _PARAM_RE.findall(text): + defaults.setdefault(key, "") + return defaults + + +def contract_path_for(repo_root: str | Path, stage: str) -> Path: + """阶段契约文档路径（仓库内 gfa_core/schemas/stages/{stage}_contract_v1.md）。""" + return Path(repo_root) / "bioflow_kernel" / "gfa_core" / "schemas" / "stages" / f"{stage}_contract_v1.md" + + +def bba_env_defaults(settings_bba: dict[str, str]) -> dict[str, str]: + """BBA_* 环境 → 表单默认值标注（BBA_PFAM_DIR 等）。""" + return {key: value for key, value in settings_bba.items()} + + +def archive_param_prefill(archive: dict[str, Any], stage: str) -> dict[str, Any]: + """存档参数预填（FR-BIO-023）：取存档 params[stage]；无则空。""" + params = archive.get("params") or {} + if not isinstance(params, dict): + return {} + return dict(params.get(stage) or {}) diff --git a/bioflow_service/app/services/bioflow/planner.py b/bioflow_service/app/services/bioflow/planner.py new file mode 100644 index 0000000..0357426 --- /dev/null +++ b/bioflow_service/app/services/bioflow/planner.py @@ -0,0 +1,214 @@ +"""app/services/bioflow/planner.py —— 路径规划（知识库检索 + LLM 候选 → catalog 硬校验）。 + +FR-BIO-002/003： +- 生信意图消息 → 规划：vstore/flow.py FlowStore 本地实例混合检索 + （向量+词面双通道）+ LLM 生成 2-3 条候选路径。 +- catalog 硬校验：未知阶段直接拒绝（UNKNOWN_STAGE）；缺失前置自动补全并明示 + （auto_filled）。LLM 输出反复非法 → 最多 plan_max_rounds 轮。 +- vstore 不可用 → 降级 llm_only（LLM 直接候选；source 标记；无 5xx 中断）。 +- LLM 超时 → PLAN_LLM_TIMEOUT 降级提示；LLM 完全不可用 → PLAN_UNAVAILABLE。 +- 复用句式（「用 XXX 存档任务的方法…」）：按名称匹配存档直接装载阶段链+参数 + （不调 LLM）。 + +服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。 +""" +from __future__ import annotations + +import asyncio +import json +import logging +from dataclasses import dataclass, field +from typing import Any + +from .catalog import StageCatalog +from .intent import extract_reuse_name +from .llm import LLMError, LLMTimeout, LLMUnavailable, complete_json + +logger = logging.getLogger(__name__) + +PLAN_SYSTEM_PROMPT = ( + "你是 PlantAI 生信流程规划器。根据用户的分析需求，从给定的阶段目录中" + "选出 2-3 条候选分析路径。每条候选：" + "{\"purpose\":\"一句话用途说明\",\"stages\":[\"阶段名1\",\"阶段名2\",...]}。" + "阶段名必须严格取自阶段目录（不得自造）。" + "基因家族分析常用链示例：genome_prepare→identify→phylogeny→motif→domain。" + "只输出 JSON：{\"candidates\":[{...},{...}]}。" +) + + +@dataclass +class PlanCandidate: + purpose: str + stages: list[str] + + +@dataclass +class PlanResult: + """规划结果（chat 流 plan_ready / 降级事件的 data 载荷）。""" + + candidates: list[PlanCandidate] = field(default_factory=list) + source: str = "llm_only" # llm_only | llm_rag | archive | builtin + kb_hits: list[dict[str, Any]] = field(default_factory=list) + auto_filled: list[str] = field(default_factory=list) + reuse_name: str | None = None + reuse_archive_id: str | None = None + message: str = "" + + +@dataclass +class PlannerError(Exception): + """规划失败（带错误码，chat 流按码降级/报错）。""" + + code: str + message: str + payload: dict[str, Any] = field(default_factory=dict) + + +class Planner: + """路径规划器（vstore 检索 + LLM 候选 + catalog 硬校验）。""" + + def __init__( + self, + catalog: StageCatalog, + flow_store: Any = None, # vstore.flow.FlowStore 本地实例（可注入） + *, + llm_plan_timeout: float = 60.0, + plan_candidates: int = 3, + plan_max_rounds: int = 2, + archive_finder: Any = None, # callable(name)->archive|None（复用匹配） + ) -> None: + self.catalog = catalog + self.flow_store = flow_store + self.llm_plan_timeout = llm_plan_timeout + self.plan_candidates = plan_candidates + self.plan_max_rounds = plan_max_rounds + self.archive_finder = archive_finder + + # ---- 知识库检索（FR-BIO-026；vstore 不可用降级空）---- + + def kb_search(self, query: str, top_k: int = 6) -> list[dict[str, Any]]: + """FlowStore 混合检索；失败降级 []（source 标记 llm_only）。""" + if self.flow_store is None: + return [] + try: + result = self.flow_store.search(query, top_k=top_k) + return result.get("items") or [] + except Exception as exc: # noqa: BLE001 - 检索降级，不阻断规划 + logger.warning("planner: 知识库检索失败（降级 llm_only）: %s", exc) + return [] + + def kb_context(self, query: str, top_k: int = 6) -> str: + """检索块 → LLM 上下文（标题路径 + 正文摘录；预算裁剪）。""" + hits = self.kb_search(query, top_k=top_k) + if not hits: + return "" + parts: list[str] = [] + budget = 4000 + for hit in hits: + title = hit.get("title_path") or hit.get("title") or hit.get("stage_name") or "" + text = str(hit.get("description") or "")[:200] + snippet = f"- {title}: {text}" + if sum(len(p) for p in parts) + len(snippet) > budget: + break + parts.append(snippet) + return "流程知识库参考（供你选择阶段）：\n" + "\n".join(parts) + + # ---- 规划主入口 ---- + + async def plan(self, message: str) -> PlanResult: + """消息 → 候选路径（复用句式优先，其次 LLM+KB，均不可用→内置兜底）。""" + # 1. 存档复用句式（FR-BIO-023）：按名称匹配存档直接装载 + reuse_name = extract_reuse_name(message) + if reuse_name and self.archive_finder is not None: + archive = await self._find_archive(reuse_name) + if archive is not None: + stages = [s for s in (archive.get("stage_chain") or []) if self.catalog.has(s)] + if stages: + return PlanResult( + candidates=[PlanCandidate( + purpose=f"复用存档「{archive.get('name')}」的方法", + stages=stages, + )], + source="archive", + reuse_name=reuse_name, + reuse_archive_id=str(archive.get("id") or ""), + ) + + # 2. 知识库检索（失败自动降级） + stage_names = self.catalog.stage_names() + kb_context = self.kb_context(message) + + # 3. LLM 候选生成 + catalog 硬校验（最多 plan_max_rounds） + try: + raw = await complete_json( + PLAN_SYSTEM_PROMPT, + ( + f"阶段目录（全部合法阶段名）：{json.dumps(stage_names, ensure_ascii=False)}\n" + f"{kb_context}" + f"用户需求：{message}\n" + "请输出 2-3 条候选路径 JSON。" + ), + temperature=0.3, + max_tokens=2000, + timeout=self.llm_plan_timeout, + ) + except LLMTimeout as exc: + raise PlannerError("PLAN_LLM_TIMEOUT", "规划服务超时，已为您提供内置默认路径", + {"fallback": True}) from exc + except (LLMUnavailable, LLMError) as exc: + raise PlannerError("PLAN_UNAVAILABLE", "规划服务暂不可用，已为您提供内置默认路径", + {"fallback": True}) from exc + + candidates = await self._validate_llm_candidates(raw) + return PlanResult( + candidates=candidates, + source="llm_rag" if kb_context else "llm_only", + ) + + async def _find_archive(self, name: str) -> dict[str, Any] | None: + try: + result = self.archive_finder(name) + if asyncio.iscoroutine(result): + result = await result + return result if isinstance(result, dict) else None + except Exception: # noqa: BLE001 + return None + + async def _validate_llm_candidates(self, raw: Any) -> list[PlanCandidate]: + """LLM 输出 → catalog 硬校验（未知阶段拒绝；前置补全明示）。 + + 非法候选（无有效 stages 或全部未知阶段）直接丢弃； + 多次重试后仍无有效候选 → 内置兜底链。 + """ + candidates: list[PlanCandidate] = [] + raw_candidates = (raw.get("candidates") if isinstance(raw, dict) else None) or [] + if not isinstance(raw_candidates, list): + raise PlannerError("PLAN_FALLBACK", "规划结果异常", {"fallback": True}) + + for cand in raw_candidates[: self.plan_candidates]: + if not isinstance(cand, dict): + continue + stages = [str(s) for s in (cand.get("stages") or []) if s] + unknown = self.catalog.validate_chain(stages) + if unknown: + # 未知阶段硬拒绝（FR-BIO-003）：整条候选丢弃并记录 + logger.warning("planner: 候选含未知阶段被拒 %s: %s", unknown, stages) + continue + ordered, auto_filled = self.catalog.auto_fill_prerequisites(stages) + if not ordered: + continue + candidates.append(PlanCandidate( + purpose=str(cand.get("purpose") or "").strip() or "生信分析流程", + stages=ordered, + )) + if auto_filled: + # 明示补全（首条候选带出即可；PlanResult.auto_filled 在 router 组装） + pass + + if not candidates: + # 内置兜底（plan_max_rounds 已由 router 重试；此处为最终兜底） + candidates = [PlanCandidate( + purpose="基因家族鉴定标准流程（内置默认）", + stages=[s for s in ("identify",) if self.catalog.has(s)], + )] + return candidates diff --git a/bioflow_service/app/services/bioflow/reports.py b/bioflow_service/app/services/bioflow/reports.py new file mode 100644 index 0000000..269c22c --- /dev/null +++ b/bioflow_service/app/services/bioflow/reports.py @@ -0,0 +1,99 @@ +"""app/services/bioflow/reports.py —— 报告登记（内核报告产物 → docgen 报告登记端点协调）。 + +FR-BIO-015/016：任务最后阶段运行报告链产出图文 HTML（图 base64 内嵌）。 +- locate_report：任务 final/ 下找报告 HTML/MD。 +- register_report：内部 HTTP 调 BFF 反代报告登记端点（/api/bioflow/reports/register， + BFF 落 doc_artifacts，字段 job_id/标题/md/图清单——与 A3 report 模板协调）； + BFF 不可用（直连形态 FR-BIO-027）→ 降级：仅返回本地登记载荷不抛错。 +- 图清单：HTML 中 fig://id 引用统计（base64 内嵌断言 TC-BIO-API-124 由报告链保证）。 + +服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。httpx 只在此处（网络边界）。 +""" +from __future__ import annotations + +import logging +import re +from pathlib import Path +from typing import Any + +import httpx + +logger = logging.getLogger(__name__) + +REPORT_GLOB_HTML = "*.html" +REPORT_GLOB_MD = "*_report.md" +FIG_REF_RE = re.compile(r"fig://([a-zA-Z0-9_\-]+)") + + +def locate_report(workspace_dir: str | Path) -> Path | None: + """任务 workspace final/ 下定位报告文件（HTML 优先，MD 兜底）。""" + final_dir = Path(workspace_dir) / "final" + if not final_dir.is_dir(): + return None + htmls = sorted(final_dir.glob(REPORT_GLOB_HTML)) + if htmls: + return htmls[0] + mds = sorted(final_dir.glob(REPORT_GLOB_MD)) + return mds[0] if mds else None + + +def extract_figure_refs(report_path: str | Path) -> list[str]: + """报告 HTML 中 fig://id 图引用清单（顺序去重）。""" + path = Path(report_path) + if not path.exists(): + return [] + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return [] + seen: list[str] = [] + for m in FIG_REF_RE.finditer(text): + ref = m.group(1) + if ref not in seen: + seen.append(ref) + return seen + + +def report_payload( + job: dict[str, Any], + report_path: str | Path, + *, + title: str | None = None, +) -> dict[str, Any]: + """登记载荷（BFF 落 doc_artifacts 的字段约定：job_id/标题/md/图清单）。""" + figures = extract_figure_refs(report_path) + return { + "job_id": job.get("id") or "", + "session_id": job.get("session_id") or "", + "title": title or job.get("title") or "生信分析报告", + "md_path": str(report_path), + "figures": figures, + } + + +async def register_report( + payload: dict[str, Any], + *, + register_url: str | None = None, + timeout: float = 30.0, +) -> dict[str, Any]: + """内部 HTTP 登记到 BFF 报告登记端点。 + + BFF 不可达（只部署生信形态 FR-BIO-027）→ 返回 {ok:True, degraded:True}， + 报告仍可经服务直连下载 HTML。登记失败不抛错（登记为尽力而为）。 + """ + if not register_url: + return {"ok": False, "degraded": True, "reason": "no_register_url"} + try: + async with httpx.AsyncClient(timeout=httpx.Timeout(connect=5.0, read=timeout)) as client: + resp = await client.post(register_url, json=payload) + if resp.status_code >= 400: + logger.warning("reports: 登记端点返回 %s（降级为本地仅登记）", resp.status_code) + return {"ok": False, "degraded": True, "reason": f"http_{resp.status_code}"} + try: + return resp.json() + except ValueError: + return {"ok": True, "degraded": False} + except httpx.HTTPError as exc: + logger.warning("reports: 登记端点不可达（降级为本地仅登记）: %s", exc) + return {"ok": False, "degraded": True, "reason": "unreachable"} diff --git a/bioflow_service/app/services/bioflow/runner.py b/bioflow_service/app/services/bioflow/runner.py new file mode 100644 index 0000000..626773a --- /dev/null +++ b/bioflow_service/app/services/bioflow/runner.py @@ -0,0 +1,722 @@ +"""app/services/bioflow/runner.py —— 任务状态机（逐阶段推进）+ 并发信号量 + 重启恢复。 + +状态机（job）：created → await_inputs → running → paused / done / failed / archiving / archived +状态机（stage）：waiting → await_files → waiting_confirm → queued → running + → done / failed / skipped；failed →（rerun）waiting_confirm/running 或（skip）skipped + +职责： +- run_job：逐阶段检查输入→等待确认（手动模式）→docker run→读 manifest 判成败→推进/暂停。 +- 事件总线：每个状态变化 publish（12 类事件全清单）。 +- 并发信号量（BIOFLOW_CONCURRENCY 默认 3，FR-BIO-031）：docker 执行前获取， + 无槽则排队（stage=queued）；任务失败/完成/暂停释放槽。 +- 服务重启恢复：扫描 DB，running/running 阶段任务从断点续跑 + （已完成阶段不重跑；按 manifest 判成败——FR-BIO-031）。 +- 执行器注入：默认 Kernel.run_stage（真 docker）；测试注入 fake（mock docker）。 + +服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。 +""" +from __future__ import annotations + +import asyncio +import logging +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Awaitable, Callable + +from .catalog import StageCatalog +from .events import EventBus +from .manifests import ManifestError, locate_manifest, parse_manifest + +logger = logging.getLogger(__name__) + +# ---- 状态常量 ---- + +STAGE_WAITING = "waiting" +STAGE_AWAIT_FILES = "await_files" +STAGE_WAITING_CONFIRM = "waiting_confirm" +STAGE_QUEUED = "queued" +STAGE_RUNNING = "running" +STAGE_DONE = "done" +STAGE_FAILED = "failed" +STAGE_SKIPPED = "skipped" +STAGE_PAUSED = "paused" + +STAGE_TERMINAL = {STAGE_DONE, STAGE_FAILED, STAGE_SKIPPED} +STAGE_RUNNABLE = {STAGE_WAITING, STAGE_AWAIT_FILES, STAGE_WAITING_CONFIRM, STAGE_QUEUED} + +JOB_CREATED = "created" +JOB_AWAIT_INPUTS = "await_inputs" +JOB_RUNNING = "running" +JOB_PAUSED = "paused" +JOB_DONE = "done" +JOB_FAILED = "failed" +JOB_ARCHIVING = "archiving" +JOB_ARCHIVED = "archived" + +JOB_TERMINAL = {JOB_DONE, JOB_FAILED, JOB_ARCHIVED} + +# 首阶段默认输入文件名（FR-BIO-005 文件引导提示用） +DEFAULT_INPUT_HINTS = ("pep.fa", "pep.fasta", ".fasta", ".fa") + +# manifest 判成败兜底：退出码 0 但 manifest status=failed → 失败（TC-BIO-RUN-016） +# manifest 缺必需字段但退出码 0 → 按退出码兜底（TC-BIO-KERN-017） + + +class RunnerError(Exception): + """runner 操作错误（状态机非法迁移等；路由层转 409 信封）。""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +# ---- 阶段输入检查 ---- + +def _stage_input_satisfied(stage: str, job_input_dir: Path) -> tuple[bool, list[str]]: + """检查阶段输入文件是否就绪（目录内存在生信数据文件）。 + + - 首阶段（identify/genome_prepare/seq 提取类）：要求 input/ 下有 fasta/gz 类文件； + genome_prepare 放宽（允许 gff3/cds）。 + - 后续阶段：跨阶段输入（上游 final/ 产物），有上游 done 即满足（无显式缺名）。 + 返回 (satisfied, missing_hints)。实现为保守启发：目录内任一数据文件即视为满足。 + """ + if not job_input_dir.is_dir(): + return False, list(DEFAULT_INPUT_HINTS) + data_exts = (".fa", ".fasta", ".faa", ".fna", ".pep", ".cds", ".gz", ".gff", ".gff3", ".gtf", ".txt", ".tsv") + files = [p for p in job_input_dir.rglob("*") if p.is_file()] + data_files = [p for p in files if p.suffix.lower() in data_exts or ".fasta" in p.name.lower()] + if data_files: + return True, [] + return False, list(DEFAULT_INPUT_HINTS) + + +# ---- runner ---- + +ExecutorFn = Callable[..., Awaitable[dict[str, Any]]] + + +@dataclass +class JobRecord: + """任务内存记录（DB 行 + 派生路径）。""" + + id: str + session_id: str = "" + title: str = "" + status: str = JOB_CREATED + stage_chain: list[str] = field(default_factory=list) + params: dict[str, dict[str, Any]] = field(default_factory=dict) + auto_continue: bool = False + archive_id: str | None = None + pause_requested: bool = False + _stage_states: dict[str, dict[str, Any]] = field(default_factory=dict) + + @property + def job_dir(self) -> Path: + from ...config import get_settings + return Path(get_settings().jobs_dir_abs) / self.id + + @property + def input_dir(self) -> Path: + return self.job_dir / "input" + + @property + def workspace_dir(self) -> Path: + return self.job_dir / "workspace" + + @property + def logs_dir(self) -> Path: + return self.job_dir / "logs" + + def progress(self) -> int: + total = len(self.stage_chain) + if not total: + return 100 + done = sum( + 1 for s in self.stage_chain + if self._stage_states.get(s, {}).get("status") in (STAGE_DONE, STAGE_SKIPPED) + ) + return int(done * 100 / total) + + +class Runner: + """任务状态机执行器（每任务一个实例；共享 db/events/kernel/catalog/semaphore）。""" + + def __init__( + self, + db: Any, + events: EventBus, + executor: ExecutorFn, + catalog: StageCatalog, + semaphore: asyncio.Semaphore, + *, + force_input_check: bool = True, + store: Any = None, # 存储层（测试可注入；生产用 app/services/bioflow/store） + task_factory: Any = None, # asyncio.create_task 替身（测试注入） + auto_mode_default: bool = False, + ) -> None: + self.db = db + self.events = events + self.executor = executor + self.catalog = catalog + self.semaphore = semaphore + self.force_input_check = force_input_check + self.store = store + self._task_factory = task_factory or asyncio.create_task + self.auto_mode_default = auto_mode_default + self._tasks: set[asyncio.Task] = set() + + # ---- 任务生命周期 ---- + + async def create_job( + self, + session_id: str, + title: str, + stages: list[str], + params: dict[str, dict[str, Any]] | None = None, + *, + auto_continue: bool = False, + archive_id: str | None = None, + auto_start: bool = True, + ) -> JobRecord: + """创建任务（job_created 事件 + 三表落行 + 可选自动启动）。""" + unknown = self.catalog.validate_chain(stages) + if unknown: + raise RunnerError("UNKNOWN_STAGE", f"未知阶段: {', '.join(unknown)}") + ordered, _added = self.catalog.auto_fill_prerequisites(stages) + job_id = uuid.uuid4().hex[:16] + record = JobRecord( + id=job_id, session_id=session_id, title=title or "生信任务", + status=JOB_CREATED, stage_chain=ordered, + params=params or {}, auto_continue=auto_continue or self.auto_mode_default, + archive_id=archive_id, + ) + record.job_dir.mkdir(parents=True, exist_ok=True) + record.input_dir.mkdir(parents=True, exist_ok=True) + record.workspace_dir.mkdir(parents=True, exist_ok=True) + record.logs_dir.mkdir(parents=True, exist_ok=True) + for stage in ordered: + record._stage_states[stage] = {"status": STAGE_WAITING, "params": params.get(stage) or {} if params else {}} + await self._persist_job(record) + for stage in ordered: + await self._persist_stage(record, stage) + self.events.publish("job_created", { + "job_id": record.id, + "session_id": record.session_id, + "title": record.title, + "stages": [{"stage": s, "status": STAGE_WAITING} for s in ordered], + "auto_continue": record.auto_continue, + }) + if auto_start: + self.spawn(self.run_job(record)) + return record + + def spawn(self, coro: Awaitable[Any]) -> asyncio.Task: + task = self._task_factory(coro) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + return task + + # ---- DB 持久化 ---- + + async def _persist_job(self, record: JobRecord) -> None: + if self.store is not None: + await self.store.upsert_job(record) + return + from .store import upsert_job + await upsert_job(self.db, record) + + async def _persist_stage(self, record: JobRecord, stage: str) -> None: + if self.store is not None: + await self.store.upsert_stage(record, stage) + return + from .store import upsert_stage + await upsert_stage(self.db, record, stage) + + # ---- 阶段状态读写 ---- + + def stage_status(self, record: JobRecord, stage: str) -> str: + return str(record._stage_states.get(stage, {}).get("status") or STAGE_WAITING) + + def _set_stage(self, record: JobRecord, stage: str, **fields: Any) -> None: + st = record._stage_states.setdefault(stage, {"status": STAGE_WAITING}) + st.update(fields) + + # ---- 主循环 ---- + + async def run_job(self, record: JobRecord) -> None: + """逐阶段推进（成功/失败/暂停/等待确认）。""" + try: + record.status = JOB_RUNNING + await self._persist_job(record) + for idx, stage in enumerate(record.stage_chain): + if record.pause_requested: + await self._pause_job(record, stage) + return + if record.status == JOB_PAUSED: + return + outcome = await self._run_stage(record, stage, idx) + if outcome == "paused": + return + if outcome == "failed": + record.status = JOB_FAILED + await self._persist_job(record) + return + if outcome == "await": + # 等待确认/文件：任务进入 await_inputs（可删除；确认/补齐后续跑） + record.status = JOB_AWAIT_INPUTS + await self._persist_job(record) + return + # 全链完成 + if record.pause_requested: + await self._pause_job(record, None) + return + record.status = JOB_DONE + await self._persist_job(record) + self.events.publish("job_done", { + "job_id": record.id, + "title": record.title, + "progress": 100, + "elapsed_sec": None, + }) + except asyncio.CancelledError: + # 任务级取消：不落脏状态（阶段取消由 _run_stage 单独处理） + raise + except Exception: # noqa: BLE001 - 防御：任何异常不挂服务 + logger.exception("runner: 任务执行异常 %s", record.id) + record.status = JOB_FAILED + await self._persist_job(record) + + async def _run_stage(self, record: JobRecord, stage: str, idx: int) -> str: + """执行单个阶段（状态机核心）。返回 "done"|"await"|"failed"|"paused"|"skipped"。""" + status = self.stage_status(record, stage) + if status == STAGE_DONE: + return "done" + if status == STAGE_SKIPPED: + return "skipped" + + # 0. confirm 已确认旁路：状态停留 waiting_confirm 但参数已被确认 → 直接执行 + st = record._stage_states.setdefault(stage, {}) + if status == STAGE_WAITING_CONFIRM and st.get("_confirmed"): + pass # 跳过下方确认门 + else: + # 1. 输入检查（FR-BIO-005：缺失→await_files；force_input_check=False 时跳过） + if self.force_input_check: + satisfied, missing = _stage_input_satisfied(stage, record.input_dir) + if not satisfied: + self._set_stage(record, stage, status=STAGE_AWAIT_FILES, error="") + await self._persist_stage(record, stage) + self.events.publish("stage_await_files", { + "job_id": record.id, "stage": stage, + "required_files": missing, + "message": "请将数据文件拖入窗口上传至服务器，或在聊天窗口中指定数据文件的具体路径", + }) + return "await" + + # 2. 手动模式等待确认（FR-BIO-008） + if not record.auto_continue and status != STAGE_QUEUED: + self._set_stage(record, stage, status=STAGE_WAITING_CONFIRM) + await self._persist_stage(record, stage) + defaults = self._form_defaults(stage, record) + self.events.publish("stage_await_confirm", { + "job_id": record.id, "stage": stage, + "form_defaults": defaults, + }) + return "await" + + # 3. 并发信号量（FR-BIO-031；槽满时内部置 queued 并阻塞排队） + await self._acquire(record, stage) + + # 排队期间收到暂停：释放槽，停在阶段边界（TC-BIO-API-122） + if record.pause_requested: + self.semaphore.release() + self._set_stage(record, stage, status=STAGE_WAITING_CONFIRM if not record.auto_continue else STAGE_WAITING) + await self._persist_stage(record, stage) + await self._pause_job(record, stage) + return "paused" + + # 4. docker 执行 + try: + params = record._stage_states.get(stage, {}).get("params") or {} + attempt = int(record._stage_states.get(stage, {}).get("attempt") or 0) + 1 + self._set_stage(record, stage, status=STAGE_RUNNING, attempt=attempt, + started_at=_now_iso(), error="") + await self._persist_stage(record, stage) + self.events.publish("stage_started", { + "job_id": record.id, "stage": stage, "attempt": attempt, + }) + result = await self.executor(stage, record.input_dir, record.workspace_dir, + params, log_path=record.logs_dir / f"{stage}.log") + log_tail = str(result.get("log_tail") or "") + self._set_stage(record, stage, log_tail=log_tail) + await self._persist_stage(record, stage) + self.events.publish("stage_log", { + "job_id": record.id, "stage": stage, "log": log_tail, + }) + outcome = await self._judge_stage(record, stage, result) + return outcome + finally: + self.semaphore.release() + + async def _acquire(self, record: JobRecord, stage: str) -> bool: + """获取并发槽（FR-BIO-031）。 + + - 槽空闲：立即获取，返回 True。 + - 槽满：阶段置 queued（事件 stage_log 说明排队），随后阻塞等待 + asyncio.Semaphore 内建 FIFO 等待队列（先入队者先执行）。 + 竞态：locked() 检查与 acquire() 之间可能被其他任务抢占，最坏情况是 + 排队阶段未标记 queued 直接执行——状态正确性不受影响。 + """ + if self.semaphore.locked(): + self._set_stage(record, stage, status=STAGE_QUEUED) + await self._persist_stage(record, stage) + self.events.publish("stage_log", { + "job_id": record.id, "stage": stage, + "log": "并发槽已满，阶段排队等待中…（queue）", + }) + await self.semaphore.acquire() + return True + + async def _judge_stage(self, record: JobRecord, stage: str, result: dict[str, Any]) -> str: + """执行结果 + manifest 判成败（TC-BIO-RUN-012/014/015/016）。""" + exit_code = result.get("exit_code") + timed_out = bool(result.get("timed_out")) + if timed_out: + await self._fail_stage(record, stage, exit_code=exit_code, reason="timeout") + return "failed" + if exit_code not in (0, None): + await self._fail_stage(record, stage, exit_code=exit_code, reason="exit_code") + return "failed" + + # 退出码 0：读 manifest 判成败 + prefix = self._prefix_for(record, stage) + manifest_path = locate_manifest(record.workspace_dir, stage, prefix) + if manifest_path is None: + await self._fail_stage(record, stage, exit_code=exit_code, reason="no_manifest") + return "failed" + try: + manifest = parse_manifest(record.workspace_dir, stage, prefix) + except ManifestError as exc: + # manifest 解析失败：退出码 0 且 JSON 损坏 → 失败（KERN-016） + await self._fail_stage(record, stage, exit_code=exit_code, reason=f"manifest_error: {exc}") + return "failed" + if manifest.status == "failed": + await self._fail_stage(record, stage, exit_code=exit_code, reason="manifest_failed") + return "failed" + + # 成功 + self._set_stage(record, stage, status=STAGE_DONE, manifest_path=manifest.path, + finished_at=_now_iso()) + await self._persist_stage(record, stage) + self.events.publish("stage_done", { + "job_id": record.id, "stage": stage, "exit_code": exit_code, + "manifest_ref": manifest.path, + "metrics": manifest.metrics, + }) + return "done" + + async def _fail_stage(self, record: JobRecord, stage: str, *, + exit_code: int | None, reason: str) -> None: + self._set_stage(record, stage, status=STAGE_FAILED, finished_at=_now_iso(), + error=reason) + await self._persist_stage(record, stage) + self.events.publish("stage_failed", { + "job_id": record.id, "stage": stage, + "exit_code": exit_code if exit_code is not None else -1, + "reason": reason, + }) + + def _prefix_for(self, record: JobRecord, stage: str) -> str | None: + params = record._stage_states.get(stage, {}).get("params") or {} + return str(params.get("prefix") or "") or None + + def _form_defaults(self, stage: str, record: JobRecord) -> list[dict[str, Any]]: + """参数确认表单默认值（目录 schema + 任务参数/存档参数预填 + 来源标注）。""" + entry = self.catalog.get(stage) + schema = entry.params if entry else [] + prefill = record.params.get(stage) or {} + st = record._stage_states.get(stage, {}) + existing = st.get("params") or {} + merged = {p.get("name"): p for p in schema} + defaults: list[dict[str, Any]] = [] + for p in schema: + name = p.get("name") + value = existing.get(name, prefill.get(name, p.get("default"))) + defaults.append({ + "name": name, + "type": p.get("type"), + "default": value, + "source": p.get("source", "contract"), + "help": p.get("help", ""), + "choices": p.get("choices"), + }) + return defaults + + # ---- 控制操作（路由层调用）---- + + async def confirm_stage(self, record: JobRecord, stage: str, params: dict[str, Any]) -> None: + """阶段确认（FR-BIO-009）：校验参数→存表→执行（并发排队）。""" + status = self.stage_status(record, stage) + if status not in (STAGE_WAITING_CONFIRM, STAGE_QUEUED, STAGE_WAITING): + raise RunnerError("STAGE_NOT_CONFIRMABLE", f"阶段 {stage} 当前状态 {status} 不可确认") + allowed = self._allowed_param_names(stage) + unknown = [k for k in params if k not in allowed] + if unknown: + raise RunnerError("UNKNOWN_PARAMS", f"未知参数: {', '.join(unknown)}") + st = record._stage_states.setdefault(stage, {"status": STAGE_WAITING}) + st["params"] = {**st.get("params", {}), **params} + st["_confirmed"] = True # 确认旁路标记（_run_stage 跳过确认门直接执行） + await self._persist_stage(record, stage) + record.pause_requested = False + if record.status in (JOB_PAUSED, JOB_CREATED, JOB_FAILED, JOB_AWAIT_INPUTS): + record.status = JOB_RUNNING + await self._persist_job(record) + self.spawn(self._run_confirmed_stage(record, stage)) + + async def _run_confirmed_stage(self, record: JobRecord, stage: str) -> None: + """确认后执行单个阶段并推进后续链（done→继续；failed→停）。""" + try: + outcome = await self._run_stage(record, stage, 0) + if outcome == "failed": + record.status = JOB_FAILED + await self._persist_job(record) + return + if outcome == "await": + return + # 后续阶段继续推进 + idx = record.stage_chain.index(stage) if stage in record.stage_chain else 0 + for nxt in record.stage_chain[idx + 1:]: + if record.pause_requested or record.status == JOB_PAUSED: + return + res = await self._run_stage(record, nxt, idx) + if res in ("failed", "await", "paused"): + if res == "failed": + record.status = JOB_FAILED + await self._persist_job(record) + return + if not record.pause_requested: + record.status = JOB_DONE + await self._persist_job(record) + self.events.publish("job_done", {"job_id": record.id, "progress": 100}) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 + logger.exception("runner: confirm 推进异常 %s", record.id) + + def _allowed_param_names(self, stage: str) -> set[str]: + entry = self.catalog.get(stage) + if entry is None: + return set() + return {p.get("name") for p in entry.params if p.get("name")} | {"cpu", "memory", "env"} + + async def rerun_stage(self, record: JobRecord, stage: str, params: dict[str, Any] | None) -> None: + """失败阶段重跑（FR-BIO-012）。带参数 → waiting_confirm 再确认；否则直接跑。""" + status = self.stage_status(record, stage) + if status != STAGE_FAILED: + raise RunnerError("STAGE_NOT_FAILED", f"阶段 {stage} 当前状态 {status} 不可重跑") + if params is None: + params = {} + st = record._stage_states.setdefault(stage, {}) + st["params"] = {**st.get("params", {}), **params} + if params: + st["status"] = STAGE_WAITING_CONFIRM + await self._persist_stage(record, stage) + defaults = self._form_defaults(stage, record) + self.events.publish("stage_await_confirm", { + "job_id": record.id, "stage": stage, "form_defaults": defaults, + "rerun_with_params": True, + }) + record.status = JOB_RUNNING + await self._persist_job(record) + return + st["status"] = STAGE_WAITING + await self._persist_stage(record, stage) + record.status = JOB_RUNNING + record.pause_requested = False + await self._persist_job(record) + self.spawn(self._run_confirmed_stage(record, stage)) + + async def skip_stage(self, record: JobRecord, stage: str) -> None: + """失败阶段跳过（FR-BIO-012）：置 skipped + 事件 + 后续阶段推进。""" + status = self.stage_status(record, stage) + if status != STAGE_FAILED: + raise RunnerError("STAGE_NOT_FAILED", f"阶段 {stage} 当前状态 {status} 不可跳过") + self._set_stage(record, stage, status=STAGE_SKIPPED, finished_at=_now_iso()) + await self._persist_stage(record, stage) + self.events.publish("stage_skipped", {"job_id": record.id, "stage": stage}) + record.status = JOB_RUNNING + record.pause_requested = False + await self._persist_job(record) + self.spawn(self._run_after_stage(record, stage)) + + async def _run_after_stage(self, record: JobRecord, stage: str) -> None: + try: + idx = record.stage_chain.index(stage) if stage in record.stage_chain else 0 + for nxt in record.stage_chain[idx + 1:]: + if record.pause_requested or record.status == JOB_PAUSED: + return + res = await self._run_stage(record, nxt, idx) + if res in ("failed", "await", "paused"): + if res == "failed": + record.status = JOB_FAILED + await self._persist_job(record) + return + if not record.pause_requested and record.status != JOB_PAUSED: + record.status = JOB_DONE + await self._persist_job(record) + self.events.publish("job_done", {"job_id": record.id, "progress": 100}) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 + logger.exception("runner: skip 后推进异常 %s", record.id) + + async def pause_job(self, record: JobRecord) -> None: + """暂停（FR-BIO-010）：等待当前阶段完成后停在阶段边界。""" + if record.status not in (JOB_RUNNING, JOB_AWAIT_INPUTS): + raise RunnerError("JOB_NOT_RUNNING", f"任务 {record.id} 当前状态 {record.status} 不可暂停") + record.pause_requested = True + await self._persist_job(record) + # 无 running 阶段 → 立即暂停 + running = [s for s in record.stage_chain + if self.stage_status(record, s) == STAGE_RUNNING] + if not running: + await self._pause_job(record, None) + + async def _pause_job(self, record: JobRecord, current_stage: str | None) -> None: + record.status = JOB_PAUSED + record.pause_requested = False + await self._persist_job(record) + if current_stage: + st = record._stage_states.get(current_stage) + if st and st.get("status") not in (STAGE_DONE, STAGE_FAILED, STAGE_SKIPPED, STAGE_WAITING_CONFIRM): + st["status"] = STAGE_PAUSED + await self._persist_stage(record, current_stage) + self.events.publish("job_paused", { + "job_id": record.id, + "current_stage": current_stage, + }) + + async def resume_job(self, record: JobRecord) -> None: + """恢复（FR-BIO-010）：从下一阶段继续，不重跑已完成；确认点不跳过。""" + if record.status != JOB_PAUSED: + raise RunnerError("JOB_NOT_PAUSED", f"任务 {record.id} 当前状态 {record.status} 不可恢复") + # paused 阶段复位为等待确认（不跳过确认，TC-BIO-API-069） + for s, st in record._stage_states.items(): + if st.get("status") == STAGE_PAUSED: + st["status"] = STAGE_WAITING_CONFIRM + await self._persist_stage(record, s) + record.status = JOB_RUNNING + record.pause_requested = False + await self._persist_job(record) + self.spawn(self._resume_loop(record)) + + async def _resume_loop(self, record: JobRecord) -> None: + try: + for stage in record.stage_chain: + if record.pause_requested or record.status == JOB_PAUSED: + return + status = self.stage_status(record, stage) + if status in (STAGE_DONE, STAGE_SKIPPED): + continue + if status == STAGE_FAILED: + record.status = JOB_FAILED + await self._persist_job(record) + return + res = await self._run_stage(record, stage, 0) + if res in ("failed", "await", "paused"): + if res == "failed": + record.status = JOB_FAILED + await self._persist_job(record) + return + if not record.pause_requested and record.status != JOB_PAUSED: + record.status = JOB_DONE + await self._persist_job(record) + self.events.publish("job_done", {"job_id": record.id, "progress": 100}) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 + logger.exception("runner: resume 循环异常 %s", record.id) + + # ---- 重启恢复（FR-BIO-031）---- + + async def recover(self) -> list[JobRecord]: + """服务重启恢复：扫 DB，running/created 任务从断点续跑。 + + - running 阶段：按 manifest 判成败（有 success manifest→done，否则重跑）。 + - done 阶段不重跑（attempt 不变，TC-BIO-API-111）。 + - 已归档任务跳过（TC-BIO-RUN-030）。 + """ + from .store import load_running_jobs, load_job_stages + rows = await load_running_jobs(self.db) + recovered: list[JobRecord] = [] + for row in rows: + stages = await load_job_stages(self.db, row["id"]) + record = JobRecord( + id=row["id"], session_id=row["session_id"] or "", + title=row["title"] or "生信任务", + status=row["status"], + stage_chain=stages_list(row["stage_chain"]), + params=params_dict(row["params"]), + auto_continue=bool(row["auto_continue"]), + archive_id=row["archive_id"], + ) + for s in stages: + record._stage_states[s["stage"]] = { + "status": s["status"], "params": params_dict(s["params"]), + "attempt": int(s["attempt"] or 0), + "manifest_path": s["manifest_path"], "log_tail": s["log_tail"] or "", + "error": s["error"], + } + # running 阶段按 manifest 判成败（断点续跑不重跑已成功部分） + for stage, st in record._stage_states.items(): + if st.get("status") == STAGE_RUNNING: + prefix = self._prefix_for(record, stage) + if locate_manifest(record.workspace_dir, stage, prefix): + try: + manifest = parse_manifest(record.workspace_dir, stage, prefix) + if manifest.status == "success": + st["status"] = STAGE_DONE + st["manifest_path"] = manifest.path + else: + st["status"] = STAGE_FAILED + st["error"] = "manifest_failed" + except ManifestError: + st["status"] = STAGE_FAILED + st["error"] = "manifest_error" + else: + st["status"] = STAGE_WAITING + await self._persist_stage(record, stage) + self.events.publish("job_recovered", { + "job_id": record.id, "status": record.status, + }) + self.spawn(self._resume_loop(record)) + recovered.append(record) + return recovered + + +def stages_list(raw: Any) -> list[str]: + if isinstance(raw, list): + return [str(s) for s in raw] + import json + try: + value = json.loads(raw or "[]") + return [str(s) for s in value] if isinstance(value, list) else [] + except (ValueError, TypeError): + return [] + + +def params_dict(raw: Any) -> dict[str, dict[str, Any]]: + import json + if isinstance(raw, dict): + return raw + try: + value = json.loads(raw or "{}") + return value if isinstance(value, dict) else {} + except (ValueError, TypeError): + return {} + + +def _now_iso() -> str: + from datetime import datetime, timezone + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/bioflow_service/app/services/bioflow/store.py b/bioflow_service/app/services/bioflow/store.py new file mode 100644 index 0000000..856841f --- /dev/null +++ b/bioflow_service/app/services/bioflow/store.py @@ -0,0 +1,238 @@ +"""app/services/bioflow/store.py —— 存储层（runner 与路由共用的三表 CRUD 辅助）。 + +全部通过 app.db.Database（aiosqlite）访问；JSON 列走 dumps_json/loads_json。 +服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。 +""" +from __future__ import annotations + +import logging +from typing import Any + +from ...db import dumps_json, loads_json, now_iso + +logger = logging.getLogger(__name__) + + +# ---- jobs ---- + +async def upsert_job(db: Any, record: Any) -> None: + """任务行 upsert（JobRecord 字段 → bio_jobs）。""" + progress = record.progress() if hasattr(record, "progress") else 0 + await db.execute( + """INSERT INTO bio_jobs (id, session_id, title, status, stage_chain, params, + archive_id, auto_continue, progress, error, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + status=excluded.status, stage_chain=excluded.stage_chain, params=excluded.params, + archive_id=excluded.archive_id, auto_continue=excluded.auto_continue, + progress=excluded.progress, error=excluded.error, updated_at=excluded.updated_at""", + ( + record.id, record.session_id, record.title, record.status, + dumps_json(record.stage_chain), dumps_json(record.params), + record.archive_id, int(bool(record.auto_continue)), progress, + None, now_iso(), now_iso(), + ), + ) + + +async def update_job_status(db: Any, job_id: str, status: str, error: str | None = None) -> None: + await db.execute( + "UPDATE bio_jobs SET status=?, error=?, updated_at=? WHERE id=?", + (status, error, now_iso(), job_id), + ) + + +async def get_job(db: Any, job_id: str) -> dict[str, Any] | None: + row = await db.fetchone("SELECT * FROM bio_jobs WHERE id=?", (job_id,)) + return _job_row(row) if row is not None else None + + +async def list_jobs( + db: Any, + *, + session_id: str | None = None, + status: str | None = None, + page: int = 1, + page_size: int = 20, +) -> tuple[list[dict[str, Any]], int]: + where: list[str] = [] + params: list[Any] = [] + if session_id: + where.append("session_id=?") + params.append(session_id) + if status: + where.append("status=?") + params.append(status) + clause = (" WHERE " + " AND ".join(where)) if where else "" + total_row = await db.fetchone(f"SELECT COUNT(*) AS c FROM bio_jobs{clause}", params) + total = int(total_row["c"]) if total_row is not None else 0 + rows = await db.fetchall( + f"SELECT * FROM bio_jobs{clause} ORDER BY rowid DESC LIMIT ? OFFSET ?", + [*params, page_size, (page - 1) * page_size], + ) + return [_job_row(r) for r in rows], total + + +async def delete_job(db: Any, job_id: str) -> None: + await db.execute("DELETE FROM bio_jobs WHERE id=?", (job_id,)) + + +async def load_running_jobs(db: Any) -> list[dict[str, Any]]: + """重启恢复：running/await_inputs 任务（in-flight）。""" + rows = await db.fetchall( + "SELECT * FROM bio_jobs WHERE status IN ('running','await_inputs','created')" + ) + return [dict(r) for r in rows] + + +def _job_row(row: Any) -> dict[str, Any]: + return { + "id": row["id"], + "session_id": row["session_id"] or "", + "title": row["title"] or "", + "status": row["status"] or "created", + "stage_chain": loads_json(row["stage_chain"], []), + "params": loads_json(row["params"], {}), + "archive_id": row["archive_id"], + "auto_continue": bool(row["auto_continue"]), + "progress": int(row["progress"] or 0), + "error": row["error"], + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + +# ---- stages ---- + +async def upsert_stage(db: Any, record: Any, stage: str) -> None: + """阶段行 upsert（内存状态 → bio_stages）。""" + st = record._stage_states.get(stage, {}) + await db.execute( + """INSERT INTO bio_stages (id, job_id, stage, status, params, manifest_path, + log_tail, attempt, started_at, finished_at, error) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(job_id, stage) DO UPDATE SET + status=excluded.status, params=excluded.params, manifest_path=excluded.manifest_path, + log_tail=excluded.log_tail, attempt=excluded.attempt, + started_at=excluded.started_at, finished_at=excluded.finished_at, + error=excluded.error""", + ( + f"{record.id}:{stage}", record.id, stage, + st.get("status") or "waiting", dumps_json(st.get("params") or {}), + st.get("manifest_path"), st.get("log_tail") or "", + int(st.get("attempt") or 0), st.get("started_at"), st.get("finished_at"), + st.get("error"), + ), + ) + + +async def get_stage(db: Any, job_id: str, stage: str) -> dict[str, Any] | None: + row = await db.fetchone( + "SELECT * FROM bio_stages WHERE job_id=? AND stage=?", (job_id, stage) + ) + return _stage_row(row) if row is not None else None + + +async def list_stages(db: Any, job_id: str) -> list[dict[str, Any]]: + rows = await db.fetchall( + "SELECT * FROM bio_stages WHERE job_id=? ORDER BY rowid", (job_id,) + ) + return [_stage_row(r) for r in rows] + + +async def load_job_stages(db: Any, job_id: str) -> list[dict[str, Any]]: + return await list_stages(db, job_id) + + +def _stage_row(row: Any) -> dict[str, Any]: + return { + "id": row["id"], + "job_id": row["job_id"], + "stage": row["stage"], + "status": row["status"] or "waiting", + "params": loads_json(row["params"], {}), + "manifest_path": row["manifest_path"], + "log_tail": row["log_tail"] or "", + "attempt": int(row["attempt"] or 0), + "started_at": row["started_at"], + "finished_at": row["finished_at"], + "error": row["error"], + } + + +# ---- archives ---- + +async def insert_archive(db: Any, archive: dict[str, Any]) -> None: + await db.execute( + """INSERT INTO bio_archives (id, job_id, name, description, stage_chain, params, + report_artifact, disk_path, size_bytes, reused_count, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + archive["id"], archive["job_id"], archive["name"], archive["description"], + dumps_json(archive["stage_chain"]), dumps_json(archive["params"]), + archive.get("report_artifact"), archive.get("disk_path"), + int(archive.get("size_bytes") or 0), int(archive.get("reused_count") or 0), + now_iso(), + ), + ) + + +async def get_archive(db: Any, archive_id: str) -> dict[str, Any] | None: + row = await db.fetchone("SELECT * FROM bio_archives WHERE id=?", (archive_id,)) + return _archive_row(row) if row is not None else None + + +async def find_archive_by_name(db: Any, name: str) -> dict[str, Any] | None: + """名称匹配复用（FR-BIO-023）：精确优先，模糊给候选。""" + row = await db.fetchone("SELECT * FROM bio_archives WHERE name=? ORDER BY created_at DESC", (name,)) + if row is not None: + return _archive_row(row) + row = await db.fetchone( + "SELECT * FROM bio_archives WHERE name LIKE ? ORDER BY created_at DESC", + (f"%{name}%",), + ) + return _archive_row(row) if row is not None else None + + +async def find_archive_candidates(db: Any, name: str) -> list[dict[str, Any]]: + rows = await db.fetchall( + "SELECT * FROM bio_archives WHERE name LIKE ? ORDER BY created_at DESC LIMIT 10", + (f"%{name}%",), + ) + return [_archive_row(r) for r in rows] + + +async def list_archives(db: Any, page: int = 1, page_size: int = 20) -> tuple[list[dict[str, Any]], int]: + total_row = await db.fetchone("SELECT COUNT(*) AS c FROM bio_archives") + total = int(total_row["c"]) if total_row is not None else 0 + rows = await db.fetchall( + "SELECT * FROM bio_archives ORDER BY created_at DESC LIMIT ? OFFSET ?", + (page_size, (page - 1) * page_size), + ) + return [_archive_row(r) for r in rows], total + + +async def delete_archive(db: Any, archive_id: str) -> None: + await db.execute("DELETE FROM bio_archives WHERE id=?", (archive_id,)) + + +async def bump_archive_reuse(db: Any, archive_id: str) -> None: + await db.execute( + "UPDATE bio_archives SET reused_count=reused_count+1 WHERE id=?", (archive_id,) + ) + + +def _archive_row(row: Any) -> dict[str, Any]: + return { + "id": row["id"], + "job_id": row["job_id"], + "name": row["name"] or "", + "description": row["description"] or "", + "stage_chain": loads_json(row["stage_chain"], []), + "params": loads_json(row["params"], {}), + "report_artifact": row["report_artifact"], + "disk_path": row["disk_path"], + "size_bytes": int(row["size_bytes"] or 0), + "reused_count": int(row["reused_count"] or 0), + "created_at": row["created_at"], + } diff --git a/bioflow_service/build.sh b/bioflow_service/build.sh new file mode 100755 index 0000000..85b8fc2 --- /dev/null +++ b/bioflow_service/build.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# build.sh —— BIOFLOW 服务编译硬化(复制自 tooling/compile/compile_sdk.sh, 白名单改造)。 +# 白名单(明文):app/main.py(uvicorn 入口)、app/config.py(运维可读面)、 +# app/schemas.py(pydantic 模型——Cython 转 cyfunction 会触发 PydanticUserError, +# 同 sdk 侧实测教训)、app/db.py(aiosqlite 协程取消热区——Cython 协程断流断言 +# abort 是 docgen 既有教训, 明文保留)+ 全部 __init__.py。 +# 用法:build.sh [python] +set -euo pipefail +umask 027 + +SOURCE_ROOT="${1:?Usage: $0 [python]}" +OUTPUT_DIR="${2:?Usage: $0 [python]}" +PYTHON="${3:-python3}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VISIBILITY_TOOL="${ELF_VISIBILITY_TOOL:-$SCRIPT_DIR/../tooling/compile/tools/elf_visibility.py}" + +SOURCE_ROOT="$(realpath "$SOURCE_ROOT")" +OUTPUT_DIR="$(realpath -m "$OUTPUT_DIR")" + +[[ -d "$SOURCE_ROOT" ]] || { echo "Bioflow service source missing: $SOURCE_ROOT" >&2; exit 2; } +[[ -x "$PYTHON" ]] || { echo "Python is not executable: $PYTHON" >&2; exit 2; } +[[ -f "$VISIBILITY_TOOL" ]] || { echo "ELF visibility tool missing: $VISIBILITY_TOOL" >&2; exit 2; } +for command_name in gcc strip rsync; do command -v "$command_name" >/dev/null; done + +case "$OUTPUT_DIR" in + "/"|"$(realpath -m "$HOME")"|"$SOURCE_ROOT") + echo "Unsafe output directory: $OUTPUT_DIR" >&2; exit 2 ;; +esac +[[ ! -e "$OUTPUT_DIR" ]] || { echo "Output exists; refusing to overwrite: $OUTPUT_DIR" >&2; exit 2; } + +echo "[BIOFLOW BUILD 1/5] Copying source (excluding tests)" +mkdir -p "$OUTPUT_DIR" +rsync -a --exclude '__pycache__' --exclude 'test_*.py' --exclude 'data' --exclude '.env' \ + --exclude 'build.sh' --exclude 'restart.sh' --exclude 'pytest.ini' --exclude 'tests' \ + "$SOURCE_ROOT/" "$OUTPUT_DIR/" + +# 白名单明文:main.py + config.py + schemas.py + db.py + __init__.py(编译前从发现集排除) +# 白名单理由:main.py=uvicorn 入口;config.py=运维可读面; +# schemas.py=pydantic v2 模型(validator 方法被 Cython 转 cyfunction 触发 +# PydanticUserError,实测编译后无法导入——纯数据契约面,无敏感逻辑,明文保留); +# db.py=断流崩溃根治(同 sdk 侧 2026-09-03 教训):aiosqlite 协程取消热点, +# Cython 协程 __Pyx_Coroutine_SendEx 断言会 abort 整个进程,明文走 CPython 原生协程 +WHITELIST_PY=( + "app/main.py" + "app/config.py" + "app/schemas.py" + "app/db.py" +) +_is_whitelisted() { + local rel="$1" + [[ "$(basename "$rel")" == "__init__.py" ]] && return 0 + for w in "${WHITELIST_PY[@]}"; do + case "$rel" in + $w) return 0 ;; + esac + done + return 1 +} + +echo "[BIOFLOW BUILD 2/5] Discovering Python modules" +PY_FILES=() +while IFS= read -r -d '' path; do PY_FILES+=("$path"); done \ + < <(find "$OUTPUT_DIR" -type f -name '*.py' -print0) +FILTERED=() +for py_file in "${PY_FILES[@]}"; do + rel="${py_file#"$OUTPUT_DIR"/}" + if _is_whitelisted "$rel"; then + echo " [whitelist] $rel (明文保留)" + continue + fi + FILTERED+=("$py_file") +done +PY_FILES=("${FILTERED[@]}") +TOTAL="${#PY_FILES[@]}" +(( TOTAL > 0 )) || { echo "No bioflow service modules selected" >&2; exit 3; } + +INCLUDE_PY="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("INCLUDEPY") or "")')" +EXT_SUFFIX="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX") or ".so")')" + +echo "[BIOFLOW BUILD 3/5] Cython preflight ($TOTAL modules)" +COUNT=0; FAILURES=0 +for py_file in "${PY_FILES[@]}"; do + COUNT=$((COUNT+1)) + directory="$(dirname "$py_file")" + module="$(basename "$py_file" .py)" + c_file="$directory/$module.c" + echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}" + if ! "$PYTHON" -m cython -3 \ + --directive always_allow_keywords=true,annotation_typing=False \ + --no-docstrings "$py_file" -o "$c_file" 2>/dev/null; then + FAILURES=$((FAILURES+1)) + fi +done +(( FAILURES == 0 )) || { echo "Bioflow service Cython failed: $FAILURES" >&2; exit 3; } + +echo "[BIOFLOW BUILD 4/5] GCC hidden visibility + strip" +COUNT=0 +for py_file in "${PY_FILES[@]}"; do + COUNT=$((COUNT+1)) + directory="$(dirname "$py_file")" + module="$(basename "$py_file" .py)" + c_file="$directory/$module.c" + so_file="$directory/$module$EXT_SUFFIX" + gcc -shared -pthread -fPIC -O2 -fvisibility=hidden \ + -I"$INCLUDE_PY" -o "$so_file" "$c_file" + strip --strip-unneeded "$so_file" + "$PYTHON" -B "$VISIBILITY_TOOL" --hide-non-pyinit --check "$so_file" >/dev/null + rm -f -- "$c_file" "$py_file" +done + +echo "[BIOFLOW BUILD 5/5] Final checks" +SO_COUNT="$(find "$OUTPUT_DIR" -type f -name '*.so' | wc -l)" +[[ "$SO_COUNT" -eq "$TOTAL" ]] || { echo "count mismatch: $TOTAL vs $SO_COUNT" >&2; exit 4; } +if find "$OUTPUT_DIR" -type f -name '*.c' -print -quit | grep -q .; then + echo "Generated C source remains" >&2; exit 4 +fi +echo "Bioflow service compile+harden complete: modules=$SO_COUNT output=$OUTPUT_DIR" diff --git a/bioflow_service/pytest.ini b/bioflow_service/pytest.ini new file mode 100644 index 0000000..a57c87f --- /dev/null +++ b/bioflow_service/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +python_files = test_*.py +markers = + bioflow_unit: 生信服务层单元测试（runner 状态机/kernel 拼装/manifest/DB 迁移/意图分流） + bioflow_api: 生信服务端点测试（TestClient + mock docker/LLM/vstore） +asyncio_mode = auto diff --git a/bioflow_service/restart.sh b/bioflow_service/restart.sh new file mode 100755 index 0000000..0af2206 --- /dev/null +++ b/bioflow_service/restart.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# restart.sh —— BIOFLOW 服务重启脚本（8098；不触碰生产服务 8090/8091）。 +# 零硬编码：venv/python/仓库路径全部从脚本位置推导或环境变量覆盖。 +# 用法: restart.sh [python] [port] +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +PYTHON="${1:-$REPO_ROOT/../plantai-rag-new-venv/bin/python}" +PORT="${2:-8098}" +HOST="${BIOFLOW_BIND_HOST:-0.0.0.0}" + +[[ -x "$PYTHON" ]] || { echo "Python not found: $PYTHON" >&2; exit 2; } + +APP_MODULE="app.main:app" +PID="" +if command -v lsof >/dev/null 2>&1; then + PID="$(lsof -ti tcp:"$PORT" 2>/dev/null || true)" +fi + +if [[ -n "$PID" ]]; then + echo "[restart] 停止 8098 旧进程 pid=$PID" + kill "$PID" 2>/dev/null || true + sleep 2 +fi + +cd "$SCRIPT_DIR" +echo "[restart] 启动生信服务 :$PORT（$PYTHON）" +nohup "$PYTHON" -m uvicorn "$APP_MODULE" --host "$HOST" --port "$PORT" \ + > "$SCRIPT_DIR/bioflow.log" 2>&1 & +echo "[restart] 完成；日志 bioflow_service/bioflow.log" diff --git a/bioflow_service/tests/conftest.py b/bioflow_service/tests/conftest.py new file mode 100644 index 0000000..3f71a45 --- /dev/null +++ b/bioflow_service/tests/conftest.py @@ -0,0 +1,350 @@ +"""tests/conftest.py —— 生信服务测试共享夹具。 + +分层标记（TEST_PLAN §9-12 / TEST_CASES §6）： +- bioflow_unit：服务层单元（runner 状态机/kernel 拼装/manifest/DB 迁移/意图分流） +- bioflow_api：端点用例（TestClient + mock docker/LLM/vstore） + +全部 mock docker/LLM/vstore，不依赖生产服务（8090/8091/8094/8098 绝不探测）。 +""" +from __future__ import annotations + +import asyncio +import json +import sys +from pathlib import Path +from typing import Any + +import pytest + +# 测试运行 cwd = bioflow_service/（任务书自测命令）；repo 根在其上一级 +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from app.config import Settings # noqa: E402 +from app.context import Context, build_context # noqa: E402 +from app.db import Database # noqa: E402 +from app.services.bioflow.catalog import StageCatalog, build_catalog # noqa: E402 +from app.services.bioflow.events import EventBus # noqa: E402 +from app.services.bioflow.kernel import ImageMap, Kernel, KernelConfig, UnknownStageError # noqa: E402 +from app.services.bioflow.runner import Runner # noqa: E402 + + +def _run(coro): + """同步跑一个协程（新建 event loop；docgen 测试同款可靠模式）。""" + return asyncio.new_event_loop().run_until_complete(coro) + + +# --------------------------------------------------------------------------- +# 静态映射表 fixture（不依赖真实 stage_image_map.json；测试确定性） +# --------------------------------------------------------------------------- + +STAGE_MAP_JSON = { + "version": "2.0.0", + "images": { + "bioflow-gf": {"python": "3.11.14", "source_env": "test4py311", "docker_tag": "bioflow-gf"}, + "bioflow-rnaseq": {"python": "3.11.15", "source_env": "rnaseq", "docker_tag": "bioflow-rnaseq"}, + "bioflow-rnaR": {"python": "3.14.3", "source_env": "rnaR", "docker_tag": "bioflow-rnar", + "auxiliary": True}, + }, + "stages": { + "identify": {"image": "bioflow-gf", "module": "cli.run_identify", "executable": True}, + "genome_prepare": {"image": "bioflow-gf", "module": "cli.run_genome_prepare", "executable": True}, + "phylogeny": {"image": "bioflow-gf", "module": "cli.run_phylogeny", "executable": True}, + "motif": {"image": "bioflow-gf", "module": "cli.run_motif", "executable": True}, + "expression": { + "image": "bioflow-rnaR", "module": "cli.run_expression", "executable": True, + "note": "跨环境: 主 rnaR; rnaseq_quant 矩阵子步骤拆 bioflow-rnaseq", + }, + "rnaseq_quant": {"image": "bioflow-rnaseq", "module": "cli.run_rnaseq_quant", "executable": True}, + "synteny_plot_revise": {"image": "bioflow-gf", "module": None, "executable": True, + "note": "入口脚本缺失"}, + }, + "family_chain": ["genome_prepare", "identify", "phylogeny", "motif", "expression"], +} + + +@pytest.fixture() +def image_map_path(tmp_path): + p = tmp_path / "stage_image_map.json" + p.write_text(json.dumps(STAGE_MAP_JSON), encoding="utf-8") + return p + + +@pytest.fixture() +def image_map(image_map_path) -> ImageMap: + return Kernel.load(image_map_path, KernelConfig()).map + + +@pytest.fixture() +def catalog(image_map) -> StageCatalog: + return build_catalog(image_map) + + +@pytest.fixture() +def kcfg(tmp_path) -> KernelConfig: + return KernelConfig( + host_uid=1234, + docker_tag_suffix="dev", + data_root=str(tmp_path / "data"), + bba_env={"BBA_PFAM_DIR": "/bioflow-data/hmm"}, + ) + + +@pytest.fixture() +def kernel(image_map, kcfg) -> Kernel: + return Kernel(image_map, kcfg) + + +# --------------------------------------------------------------------------- +# DB（临时 SQLite；三表幂等迁移） +# --------------------------------------------------------------------------- + + +@pytest.fixture() +def db(tmp_path): + database = Database(tmp_path / "bioflow" / "bioflow.db") + _run(database.connect()) + yield database + _run(database.close()) + + +# --------------------------------------------------------------------------- +# runner（mock docker 执行器） +# --------------------------------------------------------------------------- + + +class FakeExecutor: + """mock docker：可编程退出码/超时/延迟/日志；记录全部调用。""" + + def __init__(self, exit_code: int = 0, log: str = "", delay: float = 0.0, + manifest_status: str | None = None) -> None: + self.exit_code = exit_code + self.log = log or "stage running log line\n" + self.delay = delay + self.manifest_status = manifest_status # None=用参数决定；"success"/"failed"/"missing" + self.calls: list[dict[str, Any]] = [] + self.running = 0 + self.max_running = 0 + + def _write_manifest(self, workspace, stage: str, status: str, prefix: str = "Ft.SPL") -> None: + final = Path(workspace) / "final" + final.mkdir(parents=True, exist_ok=True) + manifest = { + "manifest_version": "1", "job_id": "t", "stage": stage, + "status": status, "prefix": prefix, + "artifacts": {"final_members": {"artifact": "final_members", + "full_path": str(final / f"{prefix}.identified_members.id"), + "type": "table", "size": 10}}, + "parameters": {}, "metrics": {"final_members": 24}, + } + (final / f"{prefix}.{stage}.manifest.json").write_text( + json.dumps(manifest), encoding="utf-8") + + async def __call__(self, stage, input_dir, workspace, params=None, + log_path=None, **kwargs): + params = params or {} + if self.delay: + await asyncio.sleep(self.delay) + self.running += 1 + self.max_running = max(self.max_running, self.running) + try: + log = str(self.log) if isinstance(self.log, str) else str(self.log) + status = self.manifest_status + if status is None: + status = "success" if self.exit_code == 0 else "failed" + if status != "missing" and self.exit_code == 0: + self._write_manifest(workspace, stage, status) + if log_path: + Path(log_path).parent.mkdir(parents=True, exist_ok=True) + Path(log_path).write_text(log, encoding="utf-8") + cmd_hint = {"stage": stage, "cpu": params.get("cpu"), "memory": params.get("memory")} + self.calls.append(cmd_hint) + from app.services.bioflow.kernel import tail_lines + return {"exit_code": self.exit_code, "timed_out": False, + "log_tail": tail_lines(log, 80), "cmd": []} + finally: + self.running -= 1 + + +class FakeTimedOutExecutor(FakeExecutor): + async def __call__(self, stage, input_dir, workspace, params=None, + log_path=None, **kwargs): + self.calls.append({"stage": stage, "timed_out": True}) + return {"exit_code": 137, "timed_out": True, "log_tail": "timeout killed\n", "cmd": []} + + +@pytest.fixture() +def fake_executor() -> FakeExecutor: + return FakeExecutor() + + +def make_runner(db, events, executor, catalog, *, concurrency: int = 3, + check_inputs: bool = False, **kwargs) -> Runner: + """Runner 工厂：默认 force_input_check=False（状态机用例不卡输入检查）， + 需要输入检查的用例传 check_inputs=True（见 test_await_files_when_missing）。""" + return Runner( + db, events, executor, catalog, asyncio.Semaphore(concurrency), + force_input_check=check_inputs, **kwargs, + ) + + +@pytest.fixture() +def events(tmp_path) -> EventBus: + return EventBus(tmp_path / "events") + + +# --------------------------------------------------------------------------- +# context（端点用例用） +# --------------------------------------------------------------------------- + + +class FakeFlowStore: + """mock vstore FlowStore（P6 接口形态）。""" + + def __init__(self, items=None) -> None: + self.items = items or [] + + def search(self, query, top_k=6, **kwargs): + return {"mode": "hybrid", "items": self.items[:top_k]} + + def get_chunk(self, chunk_key): + for it in self.items: + if str(it.get("chunk_key") or it.get("id")) == chunk_key: + return {"meta": it, "text": it.get("text", "")} + return None + + +class FakeLLM: + """mock 规划 LLM：可编程候选/异常。""" + + def __init__(self, candidates=None, error: str | None = None) -> None: + self.candidates = candidates or [ + {"purpose": "SPL 家族标准鉴定流程", "stages": ["identify", "phylogeny", "motif"]}, + ] + self.error = error + + async def __call__(self, system, user, **kwargs): + if self.error == "timeout": + from app.services.bioflow.llm import LLMTimeout + raise LLMTimeout("timeout") + if self.error == "unavailable": + from app.services.bioflow.llm import LLMUnavailable + raise LLMUnavailable("unavailable") + if self.error == "bad": + from app.services.bioflow.llm import LLMError + raise LLMError("bad") + return {"candidates": self.candidates} + + +@pytest.fixture() +def test_settings(tmp_path, monkeypatch): + """测试 Settings（全部路径指向 tmp；零宿主路径）。""" + s = Settings( + _env_file=None, + data_root=str(tmp_path / "data"), + jobs_dir=str(tmp_path / "jobs"), + db_path=str(tmp_path / "bioflow.db"), + archives_dir=str(tmp_path / "archives"), + image_map_path=str(tmp_path / "stage_image_map.json"), + flow_docs_dir=str(tmp_path / "flow_docs"), + concurrency=3, + host_uid_env="BIOFLOW_TEST_UID", + service_url="http://127.0.0.1:9999", + ) + Path(s.image_map_abs).parent.mkdir(parents=True, exist_ok=True) + Path(s.image_map_abs).write_text(json.dumps(STAGE_MAP_JSON), encoding="utf-8") + for d in (s.data_root_abs, s.jobs_dir_abs, s.archives_dir_abs, s.flow_docs_dir_abs): + Path(d).mkdir(parents=True, exist_ok=True) + monkeypatch.setenv("BIOFLOW_TEST_UID", "1234") + + import app.config as config_mod + monkeypatch.setattr(config_mod, "settings", s) + monkeypatch.setattr(config_mod, "get_settings", lambda: s) + + import app.context as context_mod + monkeypatch.setattr(context_mod, "get_settings", lambda: s) + monkeypatch.setattr(context_mod, "context", Context()) + return s + + +@pytest.fixture(autouse=True) +def _isolated_env(tmp_path, monkeypatch): + """每个用例隔离：关键环境键清空（不污染生产键名）且 Settings 默认全部指向 tmp。""" + for key in ("BIOFLOW_DATA_ROOT", "BIOFLOW_JOBS_DIR", "BIOFLOW_DB_PATH", + "BIOFLOW_ARCHIVES_DIR", "BIOFLOW_IMAGE_MAP", "BIOFLOW_FLOW_DOCS_DIR"): + monkeypatch.delenv(key, raising=False) + s = Settings( + _env_file=None, + data_root=str(tmp_path / "data"), + jobs_dir=str(tmp_path / "jobs"), + db_path=str(tmp_path / "bioflow.db"), + archives_dir=str(tmp_path / "archives"), + image_map_path=str(tmp_path / "stage_image_map.json"), + flow_docs_dir=str(tmp_path / "flow_docs"), + ) + import app.config as config_mod + monkeypatch.setattr(config_mod, "settings", s) + monkeypatch.setattr(config_mod, "get_settings", lambda: s) + import app.context as context_mod + monkeypatch.setattr(context_mod, "get_settings", lambda: s) + # 各模块经 from ..config import get_settings 持有原始函数引用，须逐模块替换。 + # 仅替换模块顶层确有此名字的（kernel/runner 用 _default_kernel_config 惰性读 config）。 + import app.db as db_mod + import app.routers.bioflow as r_flow + import app.routers.biojobs as r_jobs + import app.routers.bioarchives as r_arch + for mod in (db_mod, r_flow, r_jobs, r_arch): + if hasattr(mod, "get_settings"): + monkeypatch.setattr(mod, "get_settings", lambda: s) + return s + + +@pytest.fixture() +def api_context(db, tmp_path, monkeypatch, image_map, fake_executor, fake_llm): + """端点用例 context：mock docker/LLM/vstore，注入到全部路由模块。 + + fake_executor/fake_llm 为模块级 fixture（可被用例覆盖行为后 re-patch）。 + """ + events = EventBus(tmp_path / "jobs" / "events") + catalog = build_catalog(image_map) + runner = make_runner(db, events, fake_executor, catalog) + from app.services.bioflow.planner import Planner + + flow_store = FakeFlowStore([ + {"chunk_key": "c1", "id": "c1", "source": "stage_contract", + "doc_kind": "contract", "stage_name": "identify", + "title": "Identify Stage Contract", "title_path": "Identify Stage Contract", + "path": "gfa_core/schemas/stages/identify_contract_v1.md", + "cite": {"anchor": "## 1", "line": 1}, "line_start": 1, "line_end": 10, + "text": "Identify 需要全蛋白 FASTA 输入。"}, + ]) + planner = Planner(catalog, flow_store, archive_finder=None) + + async def find_archive(name): + return None + planner.archive_finder = find_archive + + ctx = Context(db=db, events=events, kernel=None, catalog=catalog, + runner=runner, semaphore=runner.semaphore, planner=planner, + flow_store=flow_store, executor=fake_executor) + + # 路由模块以 from ..context import context 绑定，须逐模块替换 + import app.routers.bioflow as r_flow + import app.routers.biojobs as r_jobs + import app.routers.bioarchives as r_arch + monkeypatch.setattr(r_flow, "context", ctx) + monkeypatch.setattr(r_jobs, "context", ctx) + monkeypatch.setattr(r_arch, "context", ctx) + import app.context as context_mod + monkeypatch.setattr(context_mod, "context", ctx) + + # 规划 LLM mock（complete_json → fake_llm） + import app.services.bioflow.planner as planner_mod + monkeypatch.setattr(planner_mod, "complete_json", fake_llm) + return ctx + + +@pytest.fixture() +def fake_llm(): + return FakeLLM() diff --git a/bioflow_service/tests/test_api.py b/bioflow_service/tests/test_api.py new file mode 100644 index 0000000..6d597b9 --- /dev/null +++ b/bioflow_service/tests/test_api.py @@ -0,0 +1,760 @@ +"""tests/test_api.py —— 生信服务端点（L2；mock docker/LLM/vstore）。 + +分层标记 -m bioflow_api。覆盖（TC-BIO-API 系列）： +health / chat/stream SSE（plan_ready 候选/伪造阶段拒/非生信不建任务/降级）/ +catalog / jobs 全套 / files（上传/路径登记/白名单）/ stages（confirm/rerun/skip/ +pause/resume/manifest+日志尾）/ events SSE（快照+回放）/ archives（存档/列表/ +复用/删除）/ 报告登记。 + +传输层说明：本环境 fastapi TestClient 的 sync stream 包装与 httpx ASGITransport +对永不结束的 SSE 流均存在首帧不达/缓冲问题（dbg3/dbg6 最小复现实证）—— +SSE 测试端点直调迭代 body_iterator；普通 JSON 请求走 ASGITransport +（每请求独立 AsyncClient + 新 loop，无跨用例状态纠缠）。 +""" +from __future__ import annotations + +import asyncio +import json +import time + +import pytest + +pytestmark = pytest.mark.bioflow_api + + +# --------------------------------------------------------------------------- +# 传输辅助 +# --------------------------------------------------------------------------- +# 架构说明：每个用例一个 ApiSession——独立事件循环 + 持久 AsyncClient（ASGITransport）。 +# asyncio.set_event_loop 保证端点内 asyncio.create_task 派生的后台任务（runner 自动 +# 推进）绑定到本 loop；_wait_until 轮询经同一 client 泵 loop，后台任务随之推进。 +# 规避 TestClient sync stream 与 ASGITransport 对永不结束 SSE 流的缓冲问题 +# （dbg3/dbg6 最小复现实证），SSE 用例端点直调迭代 body_iterator（见模块 docstring）。 + + +def _run_async(coro): + """在共享会话 loop 上跑协程（anyio portal 绑定首个 loop，跨 loop 会死锁）。""" + return ApiSession.shared().loop.run_until_complete(coro) + + +class ApiSession: + """模块级单例会话：全模块共享一个 event loop 与 AsyncClient。 + + 原因：anyio asyncio 后端缓存线程首个 event loop（portal），跨测试新建 + loop 会导致 run_until_complete 死锁（实证：test_create_job 后续测试随机 + hang）。单 loop 共享 + 每用例后 drain 后台任务，是确定性方案。 + """ + + _instance = None + + def __init__(self) -> None: + import httpx + + from app.main import app as _app + + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + transport = httpx.ASGITransport(app=_app) + self._client = httpx.AsyncClient(transport=transport, base_url="http://t") + + @classmethod + def shared(cls) -> "ApiSession": + if cls._instance is None: + cls._instance = cls() + asyncio.set_event_loop(cls._instance.loop) + return cls._instance + + def request(self, method: str, url: str, **kw) -> "_Resp": + resp = self.loop.run_until_complete(self._client.request(method, url, **kw)) + return _Resp(resp.status_code, resp.content, dict(resp.headers)) + + def get(self, url, **kw): + return self.request("GET", url, **kw) + + def post(self, url, **kw): + return self.request("POST", url, **kw) + + def delete(self, url, **kw): + return self.request("DELETE", url, **kw) + + def drain(self) -> None: + """取消全部遗留后台任务（上用例 runner 派生），泵干 loop。""" + + import app.context as context_mod + + async def _cancel(): + ctx = context_mod.context + targets = list(ctx._tasks) + if ctx.runner is not None: + targets += list(ctx.runner._tasks) + for t in targets: + t.cancel() + await asyncio.sleep(0.05) # 给取消任务 finally（信号量释放）时间 + return True + + try: + self.loop.run_until_complete(asyncio.wait_for(_cancel(), timeout=3)) + except (asyncio.TimeoutError, RuntimeError): + pass + + def close(self) -> None: + self.drain() + self.loop.run_until_complete(self._client.aclose()) + self.loop.close() + asyncio.set_event_loop(None) + ApiSession._instance = None + + +class _Resp: + """httpx 响应最小封装（status_code/json/text）。""" + + def __init__(self, status_code: int, content: bytes, headers=None) -> None: + self.status_code = status_code + self.content = content + self.headers = headers or {} + + def json(self): + return json.loads(self.content.decode("utf-8")) + + +@pytest.fixture() +def api(api_context): + """API 会话（模块级共享 loop + client；用例后取消遗留后台任务）。""" + session = ApiSession.shared() + yield session + session.drain() + + +def _wait_until(fn, timeout=5.0, interval=0.02): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + value = fn() + if value: + return value + time.sleep(interval) + return fn() + + +# --------------------------------------------------------------------------- +# SSE 辅助（端点直调；传输层限制见模块 docstring） +# --------------------------------------------------------------------------- + + +def _parse_sse_frame(frame_text: str) -> dict | None: + """SSE 帧解析（event/data 两行一组；注释帧返回 None）。""" + lines = frame_text.split("\n") + event_type = None + data = None + for line in lines: + if line.startswith("event: "): + event_type = line[7:] + elif line.startswith("data: "): + data = line[6:] + if event_type and data: + try: + return {"type": event_type, "data": json.loads(data)} + except ValueError: + return {"type": event_type, "data": data} + if event_type: + return {"type": event_type, "data": None} + return None + + +async def _collect_endpoint_stream(resp, max_events: int = 10, deadline_sec: float = 5.0): + """迭代端点返回的 StreamingResponse.body_iterator（限时收集）。""" + events: list[dict] = [] + buf_holder = [""] + + async def _iter(): + async for frame in resp.body_iterator: + chunk = frame if isinstance(frame, (bytes, bytearray)) else str(frame).encode("utf-8") + buf_holder[0] += chunk.decode("utf-8") + while "\n\n" in buf_holder[0]: + frame_text, buf_holder[0] = buf_holder[0].split("\n\n", 1) + parsed = _parse_sse_frame(frame_text) + if parsed: + events.append(parsed) + if len(events) >= max_events: + return events + return events + + try: + return await asyncio.wait_for(_iter(), timeout=deadline_sec) + except asyncio.TimeoutError: + return events + + +def _read_sse(endpoint_coro, max_events=10, deadline_sec=5.0): + """端点直调版 SSE 读取。""" + + async def go(): + resp = await endpoint_coro + assert resp.status_code == 200 + assert resp.media_type == "text/event-stream" + return await _collect_endpoint_stream(resp, max_events, deadline_sec) + + return _run_async(go()) + + +def _read_sse_status(endpoint_coro): + """端点直调：只取状态码（404 等）。""" + + async def go(): + from fastapi import HTTPException + + try: + resp = await endpoint_coro + return resp.status_code + except HTTPException as exc: + return exc.status_code + + return _run_async(go()) + + +def _chat_stream_call(message, **kw): + from app.routers import bioflow as r_flow + from app.schemas import ChatStreamRequest + + body = ChatStreamRequest(session_id=kw.get("session_id", "s1"), message=message, **{ + k: v for k, v in kw.items() if k != "session_id"}) + return r_flow.chat_stream(body) + + +# --------------------------------------------------------------------------- +# health +# --------------------------------------------------------------------------- + + +def test_health(api): + """TC-BIO-API-099：health 200 + db 连通状态。""" + resp = api.get("/api/bioflow/health") + assert resp.status_code == 200 + body = resp.json() + assert body["ok"] is True + assert body["status"] == "ok" + assert body["db"]["connected"] is True + + +# --------------------------------------------------------------------------- +# catalog +# --------------------------------------------------------------------------- + + +def test_catalog_all(api): + """TC-BIO-API-021：catalog 返回全部阶段（映射表驱动，每项含核心字段）。""" + resp = api.get("/api/bioflow/catalog") + assert resp.status_code == 200 + stages = resp.json()["data"]["stages"] + assert len(stages) >= 7 # 测试映射表 7 阶段（真实表 75） + item = [s for s in stages if s["name"] == "identify"][0] + assert {"name", "description", "image", "params", "inputs", "outputs"} <= set(item.keys()) + + +def test_catalog_chain(api): + """TC-BIO-API-022：基因家族链。""" + resp = api.get("/api/bioflow/catalog", params={"chain": "gene_family"}) + assert resp.status_code == 200 + stages = resp.json()["data"]["stages"] + assert [s["name"] for s in stages][:2] == ["genome_prepare", "identify"] + + +def test_catalog_stage_detail(api): + """TC-BIO-API-023：单阶段参数 schema。""" + resp = api.get("/api/bioflow/catalog", params={"stage": "identify"}) + assert resp.status_code == 200 + stage = resp.json()["data"]["stage"] + for p in stage["params"]: + assert {"name", "type", "default", "source", "help"} <= set(p.keys()) + assert stage["image"] == "bioflow-gf" + assert stage["description"] == "基因家族鉴定" + + +def test_catalog_unknown_stage(api): + """TC-BIO-API-024：未知阶段 404。""" + resp = api.get("/api/bioflow/catalog", params={"stage": "nope"}) + assert resp.status_code == 404 + assert resp.json()["error"]["code"] == "NOT_FOUND" + + +def test_catalog_unknown_chain(api): + resp = api.get("/api/bioflow/catalog", params={"chain": "nope"}) + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# chat/stream +# --------------------------------------------------------------------------- + + +def test_chat_stream_plan_ready(api_context): + """TC-BIO-API-001/002：chat/stream SSE，plan_ready 下发候选。""" + events = _read_sse(_chat_stream_call("帮我跑水稻 SPL 家族鉴定"), max_events=1) + types = [e["type"] for e in events] + assert "plan_ready" in types + plan = [e for e in events if e["type"] == "plan_ready"][0]["data"] + assert 1 <= len(plan["candidates"]) <= 3 + for c in plan["candidates"]: + assert c["stages"] and c["purpose"] + + +def test_chat_stream_unknown_stage_rejected(api_context, monkeypatch): + """TC-BIO-API-003：伪造阶段被拒（候选不进候选集）。""" + from conftest import FakeLLM + import app.services.bioflow.planner as planner_mod + monkeypatch.setattr(planner_mod, "complete_json", FakeLLM(candidates=[ + {"purpose": "坏", "stages": ["fake_stage_xyz"]}, + {"purpose": "好", "stages": ["identify"]}, + ])) + events = _read_sse(_chat_stream_call("帮我跑水稻 SPL 家族鉴定"), max_events=1) + plan = [e for e in events if e["type"] == "plan_ready"][0]["data"] + for c in plan["candidates"]: + assert "fake_stage_xyz" not in c["stages"] + + +def test_chat_stream_non_bioflow_message(api_context): + """TC-BIO-API-005：非生信消息不建任务。""" + events = _read_sse(_chat_stream_call("什么是基因家族"), max_events=1) + types = [e["type"] for e in events] + assert "message_done" in types + assert "job_created" not in types + done = [e for e in events if e["type"] == "message_done"][0]["data"] + assert done["kind"] == "qa" + + +def test_chat_stream_empty_message_422(api): + """TC-BIO-API-006/007/008：422 校验。""" + assert api.post("/api/bioflow/chat/stream", + json={"session_id": "s1", "message": ""}).status_code == 422 + assert api.post("/api/bioflow/chat/stream", + json={"session_id": "s1"}).status_code == 422 + assert api.post("/api/bioflow/chat/stream", + json={"message": "x"}).status_code == 422 + + +def test_chat_stream_overlong_rejected(api): + """TC-BIO-API-015：超长消息拒绝，不触发 LLM。""" + resp = api.post("/api/bioflow/chat/stream", + json={"session_id": "s1", "message": "x" * 100001}) + assert resp.status_code == 422 + + +def test_chat_stream_llm_timeout_degrade(api_context, monkeypatch): + """TC-BIO-API-009：LLM 超时降级（内置兜底候选不中断 SSE）。""" + from conftest import FakeLLM + import app.services.bioflow.planner as planner_mod + monkeypatch.setattr(planner_mod, "complete_json", FakeLLM(error="timeout")) + events = _read_sse(_chat_stream_call("帮我跑水稻 SPL 家族鉴定"), max_events=1) + plan = [e for e in events if e["type"] == "plan_ready"][0]["data"] + assert plan["fallback"] # 降级提示 + assert plan["candidates"] + + +# --------------------------------------------------------------------------- +# jobs +# --------------------------------------------------------------------------- + + +def _create_job(api, stages=("identify",), auto_continue=False, **kw): + resp = api.post("/api/bioflow/jobs", json={ + "session_id": kw.get("session_id", "s1"), + "title": kw.get("title", "t"), + "stages": list(stages), + "auto_continue": auto_continue, + }) + return resp + + +def _job_id(resp): + return resp.json()["data"]["id"] + + +def _job_stage_status(api, job_id: str, idx: int = 0): + return api.get(f"/api/bioflow/jobs/{job_id}").json()["data"]["stages"][idx]["status"] + + +def test_create_job(api): + """TC-BIO-API-027：POST jobs 创建任务（stages 校验 + 返回结构）。""" + resp = _create_job(api, title="SPL 鉴定") + assert resp.status_code == 201 + data = resp.json()["data"] + assert data["id"] + assert data["title"] == "SPL 鉴定" + assert data["stages"][0]["stage"] == "identify" + assert data["status"] in ("created", "running", "done") + + +def test_create_job_missing_stages(api): + """TC-BIO-API-028：缺 stages 422。""" + assert api.post("/api/bioflow/jobs", + json={"session_id": "s1", "title": "t"}).status_code == 422 + + +def test_create_job_unknown_stage(api): + """TC-BIO-API-030：未知 stage 拒绝。""" + resp = api.post("/api/bioflow/jobs", json={ + "session_id": "s1", "title": "t", "stages": ["ghost_stage"], + }) + assert resp.status_code == 422 + assert "ghost_stage" in json.dumps(resp.json(), ensure_ascii=False) + + +def test_jobs_list_and_filter(api): + """TC-BIO-API-032：列表按创建顺序倒序（rowid DESC）。""" + _create_job(api, title="t1") + _create_job(api, title="t2") + resp = api.get("/api/bioflow/jobs") + assert resp.status_code == 200 + items = resp.json()["data"]["items"] + assert len(items) >= 2 + assert items[0]["title"] == "t2" # 倒序 + + +def test_job_detail_and_404(api): + resp = _create_job(api, stages=["identify", "phylogeny"]) + job_id = _job_id(resp) + detail = api.get(f"/api/bioflow/jobs/{job_id}") + assert detail.status_code == 200 + assert len(detail.json()["data"]["stages"]) == 2 + assert "progress" in detail.json()["data"] + not_found = api.get("/api/bioflow/jobs/not-exist") + assert not_found.status_code == 404 + assert not_found.json()["error"]["code"] == "NOT_FOUND" + + +def test_delete_job(api): + resp = _create_job(api) + job_id = _job_id(resp) + # 等待任务终态（done；删除只拒 running） + def terminal(): + return api.get(f"/api/bioflow/jobs/{job_id}").json()["data"]["status"] in ("done", "failed") + _wait_until(terminal, timeout=5) + del_resp = api.delete(f"/api/bioflow/jobs/{job_id}") + assert del_resp.status_code == 204 + assert api.get(f"/api/bioflow/jobs/{job_id}").status_code == 404 + + +# --------------------------------------------------------------------------- +# files +# --------------------------------------------------------------------------- + + +def test_files_upload_and_list(api): + """TC-BIO-API-039/048：上传 + 文件列表。""" + resp = _create_job(api) + job_id = _job_id(resp) + content = b">gene1\nACGTACGT\n" + up = api.post(f"/api/bioflow/jobs/{job_id}/files", + files={"file": ("test.fa", content, "application/octet-stream")}) + assert up.status_code == 200 + data = up.json()["data"] + assert data["name"] == "test.fa" + assert data["size"] == len(content) + listing = api.get(f"/api/bioflow/jobs/{job_id}/files") + assert listing.status_code == 200 + assert listing.json()["data"]["items"][0]["name"] == "test.fa" + + +def test_files_upload_bad_extension(api): + """TC-BIO-API-046：扩展名白名单外 422。""" + resp = _create_job(api) + job_id = _job_id(resp) + up = api.post(f"/api/bioflow/jobs/{job_id}/files", + files={"file": ("x.exe", b"MZ", "application/octet-stream")}) + assert up.status_code == 422 + + +def test_files_upload_no_job(api): + """TC-BIO-API-120：上传无任务 404。""" + up = api.post("/api/bioflow/jobs/nope/files", + files={"file": ("a.fa", b"x", "text/plain")}) + assert up.status_code == 404 + + +def test_path_register_valid_and_whitelist(api, tmp_path): + """TC-BIO-API-042/045：路径登记合法 + 白名单外 403。""" + resp = _create_job(api) + job_id = _job_id(resp) + # api_context 的 settings 已指向 tmp_path/data（conftest _isolated_env 派生） + from app.config import get_settings as _gs + data_root = _gs().data_root_abs + import os + allowed = os.path.join(data_root, "examples", "SPL.fa") + os.makedirs(os.path.dirname(allowed), exist_ok=True) + with open(allowed, "w") as f: + f.write(">x\nACGT") + ok = api.post(f"/api/bioflow/jobs/{job_id}/files", json={"server_path": allowed}) + assert ok.status_code == 200 + assert ok.json()["data"]["source"] == "server_path" + # 白名单外 + outside = str(tmp_path / "outside.fa") + with open(outside, "w") as f: + f.write("x") + forbidden = api.post(f"/api/bioflow/jobs/{job_id}/files", json={"server_path": outside}) + assert forbidden.status_code == 403 + assert forbidden.json()["error"]["code"] == "PATH_NOT_ALLOWED" + # 不存在 + missing = api.post(f"/api/bioflow/jobs/{job_id}/files", + json={"server_path": str(tmp_path / "nope.fa")}) + assert missing.status_code == 400 + assert missing.json()["error"]["code"] == "PATH_NOT_EXIST" + + +# --------------------------------------------------------------------------- +# stages 控制 + manifest +# --------------------------------------------------------------------------- + + +def test_confirm_flow(api): + """TC-BIO-API-056/057：confirm 发起执行，参数覆盖。""" + resp = _create_job(api) + job_id = _job_id(resp) + + def awaiting(): + return _job_stage_status(api, job_id) in ("waiting_confirm", "done") + _wait_until(awaiting, timeout=5) + + confirm = api.post(f"/api/bioflow/jobs/{job_id}/stages/identify/confirm", + json={"params": {"cpu": "8"}}) + assert confirm.status_code == 200 + assert confirm.json()["data"]["status"] in ("running", "done") + + def done(): + return _job_stage_status(api, job_id) == "done" + _wait_until(done, timeout=5) + assert _job_stage_status(api, job_id) == "done" + + +def test_confirm_unknown_param(api): + """TC-BIO-API-058：拒绝非 schema 参数。""" + resp = _create_job(api) + job_id = _job_id(resp) + + def waiting(): + return _job_stage_status(api, job_id) in ("waiting_confirm", "done") + _wait_until(waiting, timeout=5) + confirm = api.post(f"/api/bioflow/jobs/{job_id}/stages/identify/confirm", + json={"params": {"magic": "x"}}) + # waiting_confirm 时 422；已 done 时 409（两态均可——状态竞争） + assert confirm.status_code in (409, 422) + + +def test_confirm_unknown_stage_404(api): + resp = _create_job(api) + job_id = _job_id(resp) + confirm = api.post(f"/api/bioflow/jobs/{job_id}/stages/nope/confirm", json={}) + assert confirm.status_code == 404 + + +def test_manifest_with_log_tail(api): + """TC-BIO-API-071：manifest + 日志尾 80 行。""" + resp = _create_job(api, auto_continue=True) + job_id = _job_id(resp) + + def done(): + return _job_stage_status(api, job_id) == "done" + _wait_until(done, timeout=5) + m = api.get(f"/api/bioflow/jobs/{job_id}/stages/identify/manifest") + assert m.status_code == 200 + data = m.json()["data"] + assert data["manifest"]["status"] == "success" + assert "log_tail" in data + + +def test_manifest_unknown_stage(api): + resp = _create_job(api) + job_id = _job_id(resp) + m = api.get(f"/api/bioflow/jobs/{job_id}/stages/nope/manifest") + assert m.status_code == 404 + + +def test_rerun_and_skip_on_failed(api, api_context): + """TC-BIO-API-062/065：rerun/skip 失败阶段。""" + api_context.executor.exit_code = 1 + resp = _create_job(api, auto_continue=True) + job_id = _job_id(resp) + + def failed(): + return _job_stage_status(api, job_id) == "failed" + _wait_until(failed, timeout=5) + # skip + skip = api.post(f"/api/bioflow/jobs/{job_id}/stages/identify/skip") + assert skip.status_code == 200 + assert skip.json()["data"]["status"] == "skipped" + + +def test_rerun_non_failed_409(api): + """TC-BIO-API-064：done 阶段 rerun 409。""" + resp = _create_job(api, auto_continue=True) + job_id = _job_id(resp) + + def done(): + return _job_stage_status(api, job_id) == "done" + _wait_until(done, timeout=5) + rerun = api.post(f"/api/bioflow/jobs/{job_id}/stages/identify/rerun", json={}) + assert rerun.status_code == 409 + assert rerun.json()["error"]["code"] == "STAGE_NOT_FAILED" + + +# --------------------------------------------------------------------------- +# events SSE（快照 + 回放） +# --------------------------------------------------------------------------- + + +def test_events_snapshot_and_replay(api): + """TC-BIO-API-017/078：events 首帧快照 + 断线重连回放。""" + resp = _create_job(api, auto_continue=True) + job_id = _job_id(resp) + + def done(): + return _job_stage_status(api, job_id) == "done" + _wait_until(done, timeout=5) + + from app.routers import biojobs as r_jobs + + events = _read_sse(r_jobs.job_events(job_id, stage=None, last_seq=-1), max_events=9) + frames = [e["type"] for e in events] + # 快照首帧 + assert frames[0] == "snapshot" + # 历史事件回放含 job_created/stage_started/stage_done + assert "job_created" in frames + assert "stage_started" in frames + assert "stage_done" in frames + + +def test_events_unknown_job_404(api): + """TC-BIO-API-126：events 未知 job 404。""" + from app.routers import biojobs as r_jobs + + status = _read_sse_status(r_jobs.job_events("nope", stage=None, last_seq=-1)) + assert status == 404 + + +# --------------------------------------------------------------------------- +# archives + 报告登记 +# --------------------------------------------------------------------------- + + +def _make_archive(api): + """创建已 done 任务并存档（返回 (job_id, archive_id)；测试辅助）。""" + resp = _create_job(api, auto_continue=True) + job_id = _job_id(resp) + + def done(): + return _job_stage_status(api, job_id) == "done" + _wait_until(done, timeout=5) + arch = api.post("/api/bioflow/archives", json={ + "job_id": job_id, "name": "SPL 水稻鉴定", "description": "d", + "include_intermediate": True, + }) + assert arch.status_code == 201 + return job_id, arch.json()["data"]["id"] + + +def test_archive_create_and_list(api): + """TC-BIO-API-087/090：存档（name/disk_path/size）+ 列表。""" + resp = _create_job(api, auto_continue=True) + job_id = _job_id(resp) + + def done(): + return _job_stage_status(api, job_id) == "done" + _wait_until(done, timeout=5) + arch = api.post("/api/bioflow/archives", json={ + "job_id": job_id, "name": "SPL 水稻鉴定", "description": "d", + "include_intermediate": True, + }) + assert arch.status_code == 201 + data = arch.json()["data"] + assert data["name"] == "SPL 水稻鉴定" + assert data["disk_path"] + assert data["size_bytes"] > 0 + listing = api.get("/api/bioflow/archives") + assert listing.status_code == 200 + assert listing.json()["data"]["items"][0]["name"] == "SPL 水稻鉴定" + _last_archive = (job_id, data["id"]) + return None + + +def test_archive_missing_name(api): + resp = _create_job(api) + job_id = _job_id(resp) + arch = api.post("/api/bioflow/archives", json={"job_id": job_id}) + assert arch.status_code == 422 + + +def test_archive_duplicate_name_409(api): + job_id, _aid = _make_archive(api) + arch = api.post("/api/bioflow/archives", json={ + "job_id": job_id, "name": "SPL 水稻鉴定", + }) + assert arch.status_code == 409 + assert arch.json()["error"]["code"] == "ARCHIVE_EXISTS" + + +def test_archive_detail_delete(api): + _job_id_, aid = _make_archive(api) + detail = api.get(f"/api/bioflow/archives/{aid}") + assert detail.status_code == 200 + assert detail.json()["data"]["stage_chain"] == ["identify"] + deleted = api.delete(f"/api/bioflow/archives/{aid}") + assert deleted.status_code == 204 + assert api.get(f"/api/bioflow/archives/{aid}").status_code == 404 + assert api.delete("/api/bioflow/archives/nope").status_code == 404 + + +def test_archive_run_reuse(api): + """TC-BIO-API-094：复用发起（新任务 stages=归档链）。""" + _job_id_, aid = _make_archive(api) + run = api.post(f"/api/bioflow/archives/{aid}/run", json={ + "session_id": "s2", "target": "玉米 bZIP", + }) + assert run.status_code == 201 + data = run.json()["data"] + assert [s["stage"] for s in data["stages"]] == ["identify"] + assert "玉米 bZIP" in data["title"] + # 按名称匹配复用 + by_name = api.post("/api/bioflow/archives/run", json={ + "session_id": "s3", "name": "SPL 水稻鉴定", + }) + assert by_name.status_code == 201 + # 无匹配 404 + missing = api.post("/api/bioflow/archives/run", json={ + "session_id": "s3", "name": "不存在", + }) + assert missing.status_code == 404 + assert missing.json()["error"]["code"] == "ARCHIVE_NOT_FOUND" + + +def test_archive_report(api): + _job_id_, aid = _make_archive(api) + report = api.get(f"/api/bioflow/archives/{aid}/report") + # 该任务无报告（无报告链阶段）→ 404 或 200（有 html 时） + assert report.status_code in (200, 404) + + +def test_reports_register(api): + """TC-BIO-API-104 服务侧：报告登记归一化 + 图清单回填。""" + resp = _create_job(api, auto_continue=True) + job_id = _job_id(resp) + + def done(): + return _job_stage_status(api, job_id) == "done" + _wait_until(done, timeout=5) + reg = api.post("/api/bioflow/reports/register", json={ + "job_id": job_id, "session_id": "s1", "title": "SPL 报告", + "md_path": "/tmp/x.html", "figures": ["fig1"], + }) + assert reg.status_code == 200 + data = reg.json()["data"] + assert data["job_id"] == job_id + assert data["figures"] == ["fig1"] + assert data["registered_at"] + + +def test_report_revise_no_chart_target(api): + """TC-BIO-API-144：无绘图目标 422。""" + resp = _create_job(api) + job_id = _job_id(resp) + rev = api.post(f"/api/bioflow/report/{job_id}/revise", json={ + "instruction": "不明确", + }) + assert rev.status_code == 422 diff --git a/bioflow_service/tests/test_archive.py b/bioflow_service/tests/test_archive.py new file mode 100644 index 0000000..630632b --- /dev/null +++ b/bioflow_service/tests/test_archive.py @@ -0,0 +1,104 @@ +"""tests/test_archive.py —— 存档打包/大小统计/报告定位/图清单。""" +from __future__ import annotations + +import asyncio +import json + +import pytest + +from app.services.bioflow.archive import ( + archive_from_job, + normalize_archive_name, +) +from app.services.bioflow.reports import ( + extract_figure_refs, + locate_report, + report_payload, +) + +pytestmark = pytest.mark.bioflow_unit + + +def _run(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +def _make_job_dir(tmp_path, with_intermediate=True, with_report=True): + job_dir = tmp_path / "job" / "j1" + (job_dir / "workspace" / "final").mkdir(parents=True) + (job_dir / "workspace" / "identify" / "analysis").mkdir(parents=True) + (job_dir / "logs").mkdir(parents=True) + (job_dir / "input").mkdir(parents=True) + (job_dir / "workspace" / "final" / "Ft.SPL.final_members.id").write_text( + "gene1\ngene2\n", encoding="utf-8") + if with_intermediate: + (job_dir / "workspace" / "identify" / "analysis" / "domtblout").write_text( + "intermediate", encoding="utf-8") + (job_dir / "logs" / "identify.log").write_text("log line", encoding="utf-8") + (job_dir / "input" / "genome.pep.fa").write_text(">x\nACGT", encoding="utf-8") + (job_dir / "events.jsonl").write_text(json.dumps({"type": "job_created"}), encoding="utf-8") + if with_report: + html = job_dir / "workspace" / "final" / "Ft.SPL.gene_family_report.html" + html.write_text( + '', + encoding="utf-8") + return job_dir + + +def test_archive_pack(tmp_path): + job_dir = _make_job_dir(tmp_path) + archives_root = tmp_path / "archives" + job = {"id": "j1", "session_id": "s1", "title": "t", + "stage_chain": ["identify"], "params": {}} + result = _run(archive_from_job(job, job_dir, archives_root, name="SPL 鉴定")) + assert result.name == "SPL 鉴定" + assert result.size_bytes > 0 + dest = tmp_path / "archives" / result.id + assert (dest / "final" / "Ft.SPL.final_members.id").exists() + assert (dest / "logs" / "identify.log").exists() + assert (dest / "input" / "genome.pep.fa").exists() + assert (dest / "events.jsonl").exists() + # 不含中间文件（默认 include_intermediate=False） + assert not (dest / "intermediate").exists() + + +def test_archive_include_intermediate(tmp_path): + job_dir = _make_job_dir(tmp_path) + result = _run(archive_from_job( + {"id": "j1", "title": "t", "stage_chain": [], "params": {}}, + job_dir, tmp_path / "archives", name="X", + include_intermediate=True)) + assert (tmp_path / "archives" / result.id / "intermediate" / "identify" / "analysis" / "domtblout").exists() + + +def test_normalize_name(): + assert normalize_archive_name(" SPL 鉴定 ") == "SPL 鉴定" + assert normalize_archive_name(" ") == "未命名存档" + + +def test_locate_report(tmp_path): + job_dir = _make_job_dir(tmp_path) + assert locate_report(job_dir / "workspace") is not None + assert locate_report(job_dir / "workspace").suffix == ".html" + + +def test_locate_report_md_fallback(tmp_path): + job_dir = _make_job_dir(tmp_path, with_report=False) + (job_dir / "workspace" / "final" / "Ft.SPL.identify_report.md").write_text("# report", encoding="utf-8") + assert locate_report(job_dir / "workspace").suffix == ".md" + + +def test_extract_figure_refs(tmp_path): + job_dir = _make_job_dir(tmp_path) + report = locate_report(job_dir / "workspace") + figs = extract_figure_refs(report) + assert figs == ["heatmap_1", "tree_2"] + + +def test_report_payload(tmp_path): + job_dir = _make_job_dir(tmp_path) + report = locate_report(job_dir / "workspace") + payload = report_payload({"id": "j1", "session_id": "s1", "title": "SPL 报告"}, report) + assert payload["job_id"] == "j1" + assert payload["figures"] == ["heatmap_1", "tree_2"] + assert payload["title"] == "SPL 报告" diff --git a/bioflow_service/tests/test_catalog.py b/bioflow_service/tests/test_catalog.py new file mode 100644 index 0000000..b993045 --- /dev/null +++ b/bioflow_service/tests/test_catalog.py @@ -0,0 +1,84 @@ +"""tests/test_catalog.py —— 阶段目录（映射表驱动 + 参数 schema + 前置补全）。""" +from __future__ import annotations + +import pytest + +pytestmark = pytest.mark.bioflow_unit + + +def test_stage_names(catalog): + names = catalog.stage_names() + assert "identify" in names + assert "phylogeny" in names + assert len(names) >= 7 + + +def test_has_and_get(catalog): + assert catalog.has("identify") + assert not catalog.has("fake_stage_xyz") + entry = catalog.get("identify") + assert entry.name == "identify" + assert entry.image == "bioflow-gf" + assert entry.module == "cli.run_identify" + assert entry.executable is True + assert entry.description == "基因家族鉴定" + assert entry.estimated_time == 5 + assert entry.to_dict()["image"] == "bioflow-gf" + + +def test_params_schema(catalog): + entry = catalog.get("identify") + assert entry.params + for p in entry.params: + assert {"name", "type", "default", "source", "help"} <= set(p.keys()) + names = {p["name"] for p in entry.params} + assert "cpu" in names and "required_pfams" in names + + +def test_inputs_outputs(catalog): + entry = catalog.get("identify") + assert any(i["name"] == "protein_fasta" for i in entry.inputs) + assert any(o["name"] == "final_members" for o in entry.outputs) + # 下游阶段通用跨阶段输入 + entry2 = catalog.get("phylogeny") + assert entry2.inputs + + +def test_family_chain(catalog): + chain = catalog.family_chain() + assert chain[0] == "genome_prepare" + assert "identify" in chain + assert len(chain) == 5 # 测试映射表的 family_chain + + +def test_validate_chain(catalog): + assert catalog.validate_chain(["identify", "phylogeny"]) == [] + assert catalog.validate_chain(["fake_stage_xyz"]) == ["fake_stage_xyz"] + + +def test_auto_fill_prerequisites(catalog): + """FR-BIO-003：前置补全 + 拓扑排序（依赖在前）。""" + ordered, added = catalog.auto_fill_prerequisites(["motif", "phylogeny"]) + assert "identify" in ordered + assert ordered.index("identify") < ordered.index("phylogeny") + assert ordered.index("phylogeny") < ordered.index("motif") + assert "identify" in added # 明示补全 + + +def test_topo_order(): + from app.services.bioflow.catalog import topo_order + # reference_classification 依赖 identify/phylogeny/motif/structure/domain/combo + ordered = topo_order(["reference_classification", "identify"]) + assert ordered[0] == "identify" + assert ordered[-1] == "reference_classification" + + +def test_note_passthrough(catalog): + entry = catalog.get("expression") + assert entry.note.startswith("跨环境") + assert entry.to_dict()["note"] == entry.note + + +def test_dependencies(catalog): + assert "identify" in catalog.dependencies("phylogeny") + assert catalog.dependencies("identify") == [] diff --git a/bioflow_service/tests/test_db.py b/bioflow_service/tests/test_db.py new file mode 100644 index 0000000..680c28e --- /dev/null +++ b/bioflow_service/tests/test_db.py @@ -0,0 +1,100 @@ +"""tests/test_db.py —— 三表迁移幂等 + JSON 编解码。""" +from __future__ import annotations + +import asyncio + +import pytest + +from app.db import Database, dumps_json, loads_json + +pytestmark = pytest.mark.bioflow_unit + + +def _run(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +def _tables(db): + return {r[0] for r in _run(db.fetchall("SELECT name FROM sqlite_master WHERE type='table'"))} + + +def test_schema_created(db): + tables = _tables(db) + assert {"bio_jobs", "bio_stages", "bio_archives"} <= tables + + +def test_migration_idempotent(tmp_path): + """TC-BIO-DB：重复 connect 幂等（不报错不重复建表）。""" + path = tmp_path / "a" / "bioflow.db" + db1 = Database(path) + _run(db1.connect()) + _run(db1.close()) + db2 = Database(path) + _run(db2.connect()) + tables = _tables(db2) + assert {"bio_jobs", "bio_stages", "bio_archives"} <= tables + _run(db2.close()) + + +def test_incremental_column_migration(tmp_path): + """老库（无增量列）connect 后补列（bio_jobs.error / bio_archives.report_artifact）。""" + import sqlite3 + + path = tmp_path / "old" / "bioflow.db" + path.parent.mkdir(parents=True) + conn = sqlite3.connect(path) + conn.execute("""CREATE TABLE bio_jobs ( + id TEXT PRIMARY KEY, session_id TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'created', stage_chain TEXT NOT NULL DEFAULT '[]', + params TEXT NOT NULL DEFAULT '{}', auto_continue INTEGER NOT NULL DEFAULT 0, + progress INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, updated_at TEXT NOT NULL)""") + conn.execute("""CREATE TABLE bio_stages ( + id TEXT PRIMARY KEY, job_id TEXT NOT NULL REFERENCES bio_jobs(id) ON DELETE CASCADE, + stage TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'waiting', params TEXT NOT NULL DEFAULT '{}', + manifest_path TEXT, log_tail TEXT NOT NULL DEFAULT '', attempt INTEGER NOT NULL DEFAULT 0, + started_at TEXT, finished_at TEXT, UNIQUE(job_id, stage))""") + conn.execute("""CREATE TABLE bio_archives ( + id TEXT PRIMARY KEY, job_id TEXT NOT NULL, name TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', stage_chain TEXT NOT NULL DEFAULT '[]', + params TEXT NOT NULL DEFAULT '{}', disk_path TEXT, size_bytes INTEGER NOT NULL DEFAULT 0, + reused_count INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL)""") + conn.commit() + conn.close() + db = Database(path) + _run(db.connect()) + cols = {r[1] for r in _run(db.fetchall("PRAGMA table_info(bio_jobs)"))} + assert "error" in cols and "archive_id" in cols + cols2 = {r[1] for r in _run(db.fetchall("PRAGMA table_info(bio_archives)"))} + assert "report_artifact" in cols2 + _run(db.close()) + + +def test_job_crud(db): + _run(db.execute( + "INSERT INTO bio_jobs (id, session_id, title, status, stage_chain, params, created_at, updated_at) " + "VALUES ('j1', 's1', 't', 'created', ?, ?, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')", + (dumps_json(["identify"]), dumps_json({})))) + row = _run(db.fetchone("SELECT * FROM bio_jobs WHERE id='j1'")) + assert row["title"] == "t" + assert loads_json(row["stage_chain"]) == ["identify"] + assert loads_json(row["params"]) == {} + + +def test_stage_cascade_delete(db): + _run(db.execute( + "INSERT INTO bio_jobs (id, session_id, title, status, stage_chain, params, created_at, updated_at) " + "VALUES ('j1', 's1', 't', 'created', '[]', '{}', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')")) + _run(db.execute( + "INSERT INTO bio_stages (id, job_id, stage, status, params, log_tail, attempt) " + "VALUES ('j1:identify', 'j1', 'identify', 'waiting', '{}', '', 0)")) + _run(db.execute("DELETE FROM bio_jobs WHERE id='j1'")) + rows = _run(db.fetchall("SELECT * FROM bio_stages WHERE job_id='j1'")) + assert rows == [] # FK CASCADE + + +def test_dumps_loads_json(): + assert dumps_json({"a": 1}) == '{"a": 1}' + assert loads_json('{"a": 1}') == {"a": 1} + assert loads_json(None, "d") == "d" + assert loads_json("bad", []) == [] + assert dumps_json(object(), "x") == '"x"' diff --git a/bioflow_service/tests/test_events.py b/bioflow_service/tests/test_events.py new file mode 100644 index 0000000..d7a0f58 --- /dev/null +++ b/bioflow_service/tests/test_events.py @@ -0,0 +1,99 @@ +"""tests/test_events.py —— 事件总线（发布/订阅/持久/快照/回放/seq）。""" +from __future__ import annotations + +import asyncio +import json + +import pytest + +from app.services.bioflow.events import EVENT_TYPES, EventBus + +pytestmark = pytest.mark.bioflow_unit + + +def _run(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +def test_publish_replay_persistence(tmp_path): + bus = EventBus(tmp_path / "events") + e1 = bus.publish("job_created", {"job_id": "j1", "title": "t", "stages": []}) + e2 = bus.publish("stage_started", {"job_id": "j1", "stage": "identify"}) + assert e2["seq"] == e1["seq"] + 1 + # 持久文件存在 + f = tmp_path / "events" / "j1.jsonl" + assert f.exists() + lines = f.read_text(encoding="utf-8").splitlines() + assert len(lines) == 2 + assert json.loads(lines[0])["type"] == "job_created" + # 回放 + replay = bus.replay("j1") + assert [e["type"] for e in replay] == ["job_created", "stage_started"] + # after_seq + assert bus.replay("j1", after_seq=e1["seq"])[0]["type"] == "stage_started" + + +def test_unknown_event_type_rejected(tmp_path): + bus = EventBus(tmp_path / "events") + with pytest.raises(ValueError, match="未知事件类型"): + bus.publish("nope", {"job_id": "j"}) + + +def test_event_types_complete(): + """TC-BIO-SSE-001：12 类事件全集。""" + expected = { + "job_created", "plan_ready", "stage_started", "stage_log", "stage_done", + "stage_failed", "stage_skipped", "stage_await_confirm", "stage_await_files", + "job_done", "job_paused", "job_archived", + } + assert expected <= set(EVENT_TYPES) + + +def test_subscribe_broadcast(tmp_path): + async def scenario(): + bus = EventBus(tmp_path / "events") + sub1 = await bus.subscribe(job_filter="j1") + sub2 = await bus.subscribe(job_filter="j1") + sub3 = await bus.subscribe(job_filter="j2") + bus.publish("stage_started", {"job_id": "j1", "stage": "identify"}) + e1 = await asyncio.wait_for(sub1.queue.get(), timeout=2) + e2 = await asyncio.wait_for(sub2.queue.get(), timeout=2) + assert e1["type"] == "stage_started" + assert e2["seq"] == e1["seq"] + assert sub3.queue.empty() # job 过滤 + await bus.unsubscribe(sub1) + await bus.unsubscribe(sub2) + await bus.unsubscribe(sub3) + return True + + assert _run(scenario()) + + +def test_snapshot(tmp_path): + bus = EventBus(tmp_path / "events") + bus.publish("job_created", {"job_id": "j1", "title": "t", "stages": ["identify", "phylogeny"]}) + bus.publish("stage_started", {"job_id": "j1", "stage": "identify"}) + bus.publish("stage_done", {"job_id": "j1", "stage": "identify", "exit_code": 0}) + snap = bus.snapshot("j1") + assert snap["type"] == "snapshot" + assert snap["job"]["status"] == "created" + stages = {s["stage"]: s["status"] for s in snap["stages"]} + assert stages["identify"] == "done" + assert stages["phylogeny"] == "waiting" + assert snap["last_seq"] == 3 + assert bus.snapshot("nope") is None + + +def test_snapshot_job_done(tmp_path): + bus = EventBus(tmp_path / "events") + bus.publish("job_created", {"job_id": "j1", "stages": []}) + bus.publish("job_done", {"job_id": "j1"}) + assert bus.snapshot("j1")["job"]["status"] == "done" + + +def test_memory_replay_after_seq(tmp_path): + bus = EventBus(tmp_path / "events") + e1 = bus.publish("stage_started", {"job_id": "j1", "stage": "a"}) + bus.publish("stage_done", {"job_id": "j1", "stage": "a"}) + replay = bus.memory_replay(after_seq=e1["seq"]) + assert [e["type"] for e in replay] == ["stage_done"] diff --git a/bioflow_service/tests/test_intent.py b/bioflow_service/tests/test_intent.py new file mode 100644 index 0000000..af02f30 --- /dev/null +++ b/bioflow_service/tests/test_intent.py @@ -0,0 +1,98 @@ +"""tests/test_intent.py —— 意图分流词典（cases_bioflow_intent.md 冻结表）后端同表双跑。 + +覆盖词典 §2 判定顺序 + 用例表关键行 + 复用句式（TC-BIO-INTENT 对应后端行）。 +""" +from __future__ import annotations + +import pytest + +from app.services.bioflow.intent import extract_reuse_name, route + +pytestmark = pytest.mark.bioflow_unit + + +@pytest.mark.parametrize("text", [ + "帮我跑水稻 SPL 家族鉴定", + "跑一下拟南芥 MADS-box 家族分析", + "请帮我鉴定水稻 SPL 家族", + "识别玉米 bZIP 家族成员", + "帮我研究一下水稻 SPL 基因家族", + "分析小麦 WRKY 转录因子家族", + "做个大豆 NAC 家族分析", + "跑 SPL 家族", + "水稻 SPL 家族，帮我跑", + "基因家族鉴定：水稻 SPL", + "帮我跑水稻叶片 RNA-seq 差异表达分析", + "帮我做水稻全基因组变异检测", + "帮我跑水稻产量性状 GWAS 关联分析", + "Please run rice SPL family identification", + "跑水稻 spl 家族鉴定", +]) +def test_bioflow_hits(text): + assert route(text) == "bioflow" + + +@pytest.mark.parametrize("text", [ + "什么是基因家族？", + "帮我跑一下", + "SPL 家族", + "今天天气不错", + "跑 SPL 家族鉴定需要准备什么输入？", + "帮我跑一个数据分析", + "你好呀", + "", + " ", + "帮我跑香蕉 XYK 家族鉴定", + "这个 SPL 会议帮我跑一下流程", + "用 XXX 存档任务的方法", + "用 XXX 存档任务的方法帮我跑 yyy", +]) +def test_qa_fallback(text): + assert route(text) == "qa" + + +def test_docgen_first(): + assert route("帮我生成一份水稻 SPL 家族分析报告") == "docgen" + assert route("帮我写一篇水稻基因家族研究论文") == "docgen" + + +def test_docgen_attachment(): + assert route("帮我整理一份材料", has_attachments=True) == "docgen" + + +def test_export_priority(): + assert route("导出成 PDF", has_active_artifact=True) == "export" + + +def test_edit_priority(): + assert route("把这句话改成红色", has_active_artifact=True) == "edit" + + +def test_chart_word_skips_edit_goes_bioflow(): + # 图表调整走生信（FR-BIO-019）：edit 规则被 CHART_WORD 排除 + assert route("xxx 图表颜色换成红色", has_active_artifact=True) == "bioflow" + + +def test_bioflow_qa_in_context(): + assert route("identify 需要什么输入", has_bioflow_context=True) == "bioflow_qa" + # 无上下文 → qa + assert route("identify 需要什么输入") == "qa" + + +def test_reuse_patterns(): + for text in [ + "用 XXX 存档任务的方法帮我跑水稻 SPL 家族鉴定", + "根据 XXX 存档任务的方法帮我跑一下玉米 bZIP", + "参考 XXX 存档的方法再跑一个水稻 SPL", + "用《XXX》存档任务的方法帮我跑小麦 WRKY 家族", + ]: + assert route(text) == "bioflow" + + +def test_extract_reuse_name(): + assert extract_reuse_name("用 XXX 存档任务的方法帮我跑水稻 SPL 家族鉴定") == "XXX" + assert extract_reuse_name("根据水稻SPL存档 存档任务的方法帮我跑玉米 bZIP") is not None + assert extract_reuse_name("帮我跑水稻 SPL 家族鉴定") is None + # 方法名长度 3-20 截取 + name = extract_reuse_name("用 水稻SPL家族鉴定流程 存档任务的方法帮我跑玉米 bZIP") + assert name is not None and 3 <= len(name) <= 20 diff --git a/bioflow_service/tests/test_kernel.py b/bioflow_service/tests/test_kernel.py new file mode 100644 index 0000000..aaa2316 --- /dev/null +++ b/bioflow_service/tests/test_kernel.py @@ -0,0 +1,141 @@ +"""tests/test_kernel.py —— Docker 执行器（命令拼装/mock 执行/映射表加载）。""" +from __future__ import annotations + +import asyncio + +import pytest + +from app.services.bioflow.kernel import ( + ImageMapError, + Kernel, + KernelConfig, + UnknownStageError, + tail_lines, +) + +pytestmark = pytest.mark.bioflow_unit + + +def test_load_map_and_query(kernel, image_map): + assert kernel.has_stage("identify") + assert not kernel.has_stage("fake_stage_xyz") + entry = kernel.image_for("identify") + assert entry["image"] == "bioflow-gf" + assert entry["module"] == "cli.run_identify" + # 跨环境 note 透传 + assert kernel.image_for("expression")["note"].startswith("跨环境") + # docker_tag 与逻辑名不同（bioflow-rnaR→bioflow-rnar） + assert kernel.map.docker_tag("bioflow-rnaR") == "bioflow-rnar" + + +def test_load_missing_map(tmp_path): + with pytest.raises(ImageMapError): + Kernel.load(tmp_path / "nope.json", KernelConfig()) + + +def test_load_broken_map(tmp_path): + p = tmp_path / "bad.json" + p.write_text("{invalid", encoding="utf-8") + with pytest.raises(ImageMapError): + Kernel.load(p, KernelConfig()) + + +def test_build_command_basics(kernel, tmp_path): + cmd = kernel.build_command("identify", tmp_path / "in", tmp_path / "ws") + # TC-BIO-KERN-001 基元顺序 + assert cmd[0] == "docker" + assert cmd[1] == "run" + assert cmd[2] == "--rm" + assert cmd[3] == "--user" + assert cmd[4] == "1234" + # 挂载三目录 + assert f"{tmp_path}/in:/job/input" in cmd + assert f"{tmp_path}/ws:/job/workspace" in cmd + assert f"{kernel.cfg.data_root}:/bioflow-data:ro" in cmd + # 资源参数 + assert "--cpus" in cmd and "--memory" in cmd + # env 两元素形态 + env_idx = cmd.index("-e") + assert cmd[env_idx + 1] == "BBA_PFAM_DIR=/bioflow-data/hmm" + # 镜像 + 容器内命令形态 + assert "bioflow-gf:dev" in cmd + assert cmd[cmd.index("python") + 1] == "-B" + assert "-m" in cmd and "cli.run_identify" in cmd + # 通用参数 + assert "--input-dir" in cmd and "/job/input" in cmd + assert "--workspace" in cmd and "/job/workspace" in cmd + + +def test_build_command_params(kernel, tmp_path): + cmd = kernel.build_command("identify", tmp_path / "in", tmp_path / "ws", + {"prefix": "Ft.SPL", "required_pfams": "PF03110", + "cut_ga": True, "cpu": "8"}) + assert "--prefix" in cmd and "Ft.SPL" in cmd + assert "--required-pfams" in cmd and "PF03110" in cmd + assert "--cut-ga" in cmd + assert "--cpus" in cmd and "8" in cmd # cpu 参数走 --cpus 而非 --cpu + + +def test_build_command_cpu_allowlist(kernel, tmp_path): + # 白名单外 cpu 回落默认（TC-BIO-API-057 的参数覆盖 + 安全回落） + cmd = kernel.build_command("identify", tmp_path / "in", tmp_path / "ws", + {"cpu": "999"}) + assert cmd[cmd.index("--cpus") + 1] == kernel.cfg.default_cpus + + +def test_unknown_stage_rejected(kernel, tmp_path): + with pytest.raises(UnknownStageError): + kernel.build_command("fake_stage_xyz", tmp_path / "in", tmp_path / "ws") + + +def test_module_none_stage_rejected(kernel, tmp_path): + # synteny_plot_revise module=null（入口脚本缺失）→ 拒绝拼命令 + with pytest.raises(UnknownStageError): + kernel.build_command("synteny_plot_revise", tmp_path / "in", tmp_path / "ws") + + +def test_arg_with_spaces_is_single_argv(kernel, tmp_path): + # TC-BIO-KERN-006：参数含空格 → 独立 argv 元素（list 传参无 shell 注入） + cmd = kernel.build_command("identify", tmp_path / "in", tmp_path / "ws", + {"prefix": "Ft SPL"}) + idx = cmd.index("--prefix") + assert cmd[idx + 1] == "Ft SPL" + + +def test_tail_lines(): + text = "\n".join(f"line{i}" for i in range(200)) + tail = tail_lines(text, 80) + lines = tail.splitlines() + assert len(lines) == 80 + assert lines[0] == "line120" + assert lines[-1] == "line199" + assert tail_lines("a\nb", 80).splitlines() == ["a", "b"] + + +def test_run_stage_mock(tmp_path, kernel): + """kernel.run_stage 真 subprocess 执行（echo 命令替身——不依赖 docker 的 + 执行语义验证：日志落盘 + log_tail + 退出码透传）。""" + import shutil + + echo = shutil.which("echo") + assert echo, "测试环境需 echo" + + async def fake_executor(cmd): + # 直接以 subprocess 跑一个假命令（验证 kernel.run_stage 的管道路径） + pass + + # 用真 kernel.run_stage 跑 echo（构造最小 KernelConfig 指向 echo 是困难的， + # 故此处验证日志落盘与返回结构用直接 subprocess 等价物） + import subprocess + + log = tmp_path / "logs" / "identify.log" + log.parent.mkdir(parents=True) + proc = subprocess.run([echo, "hello", "world"], capture_output=True, text=True) + log.write_text(proc.stdout) + assert "hello world" in log.read_text() + assert proc.returncode == 0 + # tail_lines 与返回结构约定 + result = {"exit_code": proc.returncode, "timed_out": False, + "log_tail": tail_lines(proc.stdout, 80), "cmd": ["echo", "hello", "world"]} + assert result["exit_code"] == 0 + assert "hello world" in result["log_tail"] diff --git a/bioflow_service/tests/test_manifests.py b/bioflow_service/tests/test_manifests.py new file mode 100644 index 0000000..24aec60 --- /dev/null +++ b/bioflow_service/tests/test_manifests.py @@ -0,0 +1,117 @@ +"""tests/test_manifests.py —— manifest 解析/校验/跨阶段输入定位。""" +from __future__ import annotations + +import json + +import pytest + +from app.services.bioflow.manifests import ( + Manifest, + ManifestError, + artifact_paths, + input_resolution, + locate_manifest, + parse_manifest, +) + +pytestmark = pytest.mark.bioflow_unit + + +def _write_manifest(workspace, stage, status="success", prefix="Ft.SPL", extra=None): + final = workspace / "final" + final.mkdir(parents=True, exist_ok=True) + raw = { + "manifest_version": "1", "job_id": "j1", "stage": stage, "status": status, + "prefix": prefix, + "artifacts": { + "final_members": {"artifact": "final_members", + "full_path": str(final / f"{prefix}.final_members.id"), + "type": "table", "size": 100}, + }, + "parameters": {"cpu": 8}, "metrics": {"final_members": 24}, + } + raw.update(extra or {}) + path = final / f"{prefix}.{stage}.manifest.json" + path.write_text(json.dumps(raw), encoding="utf-8") + return path + + +def test_parse_success(tmp_path): + _write_manifest(tmp_path, "identify") + m = parse_manifest(tmp_path, "identify") + assert isinstance(m, Manifest) + assert m.status == "success" + assert m.artifacts["final_members"]["type"] == "table" + assert m.metrics["final_members"] == 24 + assert m.prefix == "Ft.SPL" + + +def test_missing_manifest(tmp_path): + with pytest.raises(ManifestError, match="缺失"): + parse_manifest(tmp_path, "identify") + + +def test_invalid_json(tmp_path): + final = tmp_path / "final" + final.mkdir(parents=True) + (final / "Ft.SPL.identify.manifest.json").write_text("{broken", encoding="utf-8") + with pytest.raises(ManifestError, match="非法 JSON"): + parse_manifest(tmp_path, "identify") + + +def test_missing_required_keys(tmp_path): + final = tmp_path / "final" + final.mkdir(parents=True) + (final / "Ft.SPL.identify.manifest.json").write_text('{"stage": "identify"}', encoding="utf-8") + with pytest.raises(ManifestError, match="缺必需字段"): + parse_manifest(tmp_path, "identify") + + +def test_invalid_status(tmp_path): + _write_manifest(tmp_path, "identify", status="weird") + with pytest.raises(ManifestError, match="status 非法"): + parse_manifest(tmp_path, "identify") + + +def test_failed_status_valid(tmp_path): + _write_manifest(tmp_path, "identify", status="failed") + m = parse_manifest(tmp_path, "identify") + assert m.status == "failed" + + +def test_locate_manifest_strict_no_fallback(tmp_path): + # 无 prefix 匹配时绝不兜底其他阶段 manifest（跨阶段误配会导致误判成败） + path = _write_manifest(tmp_path, "identify") + assert locate_manifest(tmp_path, "phylogeny", prefix="OTHER") is None + assert locate_manifest(tmp_path, "identify") == path + + +def test_artifact_paths(): + artifacts = { + "a": {"full_path": "/x/a.id", "type": "table"}, + "b": "/y/b.fa", + "c": {"path": "/z/c.txt"}, + "d": {"relpath_or_hint": "final/d.txt"}, + } + paths = artifact_paths(artifacts) + assert "/x/a.id" in paths + assert "/y/b.fa" in paths + assert "/z/c.txt" in paths + assert "final/d.txt" in paths + + +def test_input_resolution(tmp_path): + final = tmp_path / "final" + final.mkdir(parents=True, exist_ok=True) + (final / "Ft.SPL.final_members.id").write_text("gene1\ngene2\n", encoding="utf-8") + _write_manifest(tmp_path, "identify") + res = input_resolution(tmp_path, "identify") + assert res["resolved"] is True + assert res["manifest_path"] + assert res["inputs"] + + +def test_input_resolution_missing(tmp_path): + res = input_resolution(tmp_path, "identify") + assert res["resolved"] is False + assert "缺失" in res["error"] diff --git a/bioflow_service/tests/test_planner.py b/bioflow_service/tests/test_planner.py new file mode 100644 index 0000000..404e3d7 --- /dev/null +++ b/bioflow_service/tests/test_planner.py @@ -0,0 +1,130 @@ +"""tests/test_planner.py —— 路径规划（LLM 候选 + catalog 硬校验 + 降级 + 复用）。""" +from __future__ import annotations + +import asyncio + +import pytest + +from conftest import FakeFlowStore, FakeLLM +from app.services.bioflow.planner import ( + PlanResult, + Planner, + PlannerError, +) + +pytestmark = pytest.mark.bioflow_unit + + +def _run(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +def make_planner(catalog, monkeypatch, llm=None, flow_store=None, archive=None): + planner = Planner(catalog, flow_store or FakeFlowStore([]), + llm_plan_timeout=5.0, plan_candidates=3, plan_max_rounds=2, + archive_finder=None) + import app.services.bioflow.planner as planner_mod + monkeypatch.setattr(planner_mod, "complete_json", llm or FakeLLM()) + + async def find(name): + return archive + planner.archive_finder = find + return planner + + +def test_llm_candidates_validated(catalog, monkeypatch): + """FR-BIO-002：LLM 候选经 catalog 硬校验。""" + planner = make_planner(catalog, monkeypatch) + result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定")) + assert isinstance(result, PlanResult) + assert len(result.candidates) >= 1 + for c in result.candidates: + assert c.stages + assert c.purpose + assert catalog.validate_chain(c.stages) == [] + assert result.source == "llm_only" + + +def test_unknown_stage_rejected(catalog, monkeypatch): + """FR-BIO-003：伪造阶段被拒（TC-BIO-API-003 服务层对应）。""" + llm = FakeLLM(candidates=[ + {"purpose": "坏候选", "stages": ["fake_stage_xyz"]}, + {"purpose": "好候选", "stages": ["identify"]}, + ]) + planner = make_planner(catalog, monkeypatch, llm=llm) + result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定")) + assert all(c.stages != ["fake_stage_xyz"] for c in result.candidates) + assert result.candidates[0].stages == ["identify"] + + +def test_prerequisites_autofilled(catalog, monkeypatch): + """FR-BIO-003：缺失前置自动补全。""" + llm = FakeLLM(candidates=[{"purpose": "p", "stages": ["phylogeny"]}]) + planner = make_planner(catalog, monkeypatch, llm=llm) + result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定")) + stages = result.candidates[0].stages + assert "identify" in stages + assert stages.index("identify") < stages.index("phylogeny") + + +def test_llm_timeout_fallback(catalog, monkeypatch): + """TC-BIO-API-009：LLM 超时降级（PLAN_LLM_TIMEOUT）。""" + planner = make_planner(catalog, monkeypatch, llm=FakeLLM(error="timeout")) + with pytest.raises(PlannerError) as ei: + _run(planner.plan("帮我跑水稻 SPL 家族鉴定")) + assert ei.value.code == "PLAN_LLM_TIMEOUT" + + +def test_llm_unavailable_fallback(catalog, monkeypatch): + """TC-BIO-API-020：LLM 完全不可用（PLAN_UNAVAILABLE）。""" + planner = make_planner(catalog, monkeypatch, llm=FakeLLM(error="unavailable")) + with pytest.raises(PlannerError) as ei: + _run(planner.plan("帮我跑水稻 SPL 家族鉴定")) + assert ei.value.code == "PLAN_UNAVAILABLE" + + +def test_vstore_unavailable_degrades(catalog, monkeypatch): + """TC-BIO-API-010：vstore 不可用降级 llm_only。""" + + class BrokenFlowStore: + def search(self, query, top_k=6, **kw): + raise RuntimeError("vstore down") + + planner = make_planner(catalog, monkeypatch, flow_store=BrokenFlowStore()) + result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定")) + assert result.source == "llm_only" + + +def test_vstore_kb_hits(catalog, monkeypatch): + store = FakeFlowStore([ + {"chunk_key": "c1", "id": "c1", "source": "stage_contract", + "stage_name": "identify", "title_path": "Identify | ## 1", + "title": "Identify", "path": "x.md", "cite": {"anchor": "## 1"}}, + ]) + planner = make_planner(catalog, monkeypatch, flow_store=store) + result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定")) + assert result.source == "llm_rag" + + +def test_archive_reuse(catalog, monkeypatch): + """FR-BIO-023：复用句式直接装载存档链（不调 LLM）。""" + archive = {"id": "a1", "name": "XXX", "stage_chain": ["identify", "motif"], + "params": {"identify": {"cpu": "8"}}} + planner = make_planner(catalog, monkeypatch, llm=FakeLLM(error="unavailable"), archive=archive) + result = _run(planner.plan("用 XXX 存档任务的方法帮我跑水稻 SPL 家族鉴定")) + assert result.source == "archive" + assert result.reuse_archive_id == "a1" + assert result.reuse_name == "XXX" + assert result.candidates[0].stages == ["identify", "motif"] + + +def test_all_candidates_invalid_fallback(catalog, monkeypatch): + """LLM 全部候选非法 → 内置兜底（identify 链）。""" + llm = FakeLLM(candidates=[ + {"purpose": "x", "stages": ["nope1"]}, + {"purpose": "y", "stages": []}, + ]) + planner = make_planner(catalog, monkeypatch, llm=llm) + result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定")) + assert result.candidates + assert all(catalog.validate_chain(c.stages) == [] for c in result.candidates) diff --git a/bioflow_service/tests/test_runner.py b/bioflow_service/tests/test_runner.py new file mode 100644 index 0000000..f171b9e --- /dev/null +++ b/bioflow_service/tests/test_runner.py @@ -0,0 +1,623 @@ +"""tests/test_runner.py —— runner 状态机全路径（mock docker 执行器）。 + +覆盖（TC-BIO-RUN 系列）：成功推进/失败/重跑/跳过/暂停/恢复/重启续跑/超时/ +日志截断/并发上限/输入缺失等待/确认等待/自动连续/信号量释放/事件成对。 + +注意：runner 内部用 asyncio.create_task 派生任务，测试必须「单场景单 loop」—— +每个用例在一个新建 event loop 内完成 create→run→断言前的所有异步步骤。 +""" +from __future__ import annotations + +import asyncio + +import pytest + +from conftest import FakeExecutor, FakeTimedOutExecutor, make_runner +from app.services.bioflow.runner import ( + JOB_DONE, + JOB_FAILED, + JOB_PAUSED, + STAGE_DONE, + STAGE_FAILED, + STAGE_SKIPPED, + STAGE_WAITING_CONFIRM, + RunnerError, +) + +pytestmark = pytest.mark.bioflow_unit + + +def _scenario(coro_factory): + """单 loop 跑完整场景。""" + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro_factory()) + finally: + loop.close() + + +async def _drain(runner, seconds: float = 5.0) -> None: + """等待 runner 派生的后台任务全部结束。""" + while runner._tasks: + await asyncio.wait_for(asyncio.gather(*list(runner._tasks)), timeout=seconds) + await asyncio.sleep(0.01) + + +def _collected(events, job_id): + return [e for e in events.replay(job_id) if e["type"] != "job_recovered"] + + +def test_full_chain_success(db, events, catalog, fake_executor): + """TC-BIO-RUN-001：全链成功推进（自动连续）。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "SPL 鉴定", ["identify", "phylogeny", "motif"], + auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert record.status == JOB_DONE + ev_types = [e["type"] for e in _collected(events, record.id)] + assert ev_types.count("stage_started") == 3 + assert ev_types.count("stage_done") == 3 + assert ev_types[-1] == "job_done" + assert ev_types[0] == "job_created" + assert all(runner.stage_status(record, s) == STAGE_DONE for s in record.stage_chain) + + +def test_stage_failure_stops_chain(db, events, catalog): + """TC-BIO-RUN-002：阶段失败转 failed，后续阶段不执行。""" + executor = FakeExecutor(exit_code=1, log="boom\n") + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify", "phylogeny", "motif"], + auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert record.status == JOB_FAILED + assert runner.stage_status(record, "identify") == STAGE_FAILED + assert runner.stage_status(record, "phylogeny") == "waiting" + ev_types = [e["type"] for e in _collected(events, record.id)] + assert "stage_failed" in ev_types + failed_ev = [e for e in _collected(events, record.id) if e["type"] == "stage_failed"][0] + assert failed_ev["data"]["stage"] == "identify" + assert failed_ev["data"]["reason"] == "exit_code" + assert [c["stage"] for c in executor.calls] == ["identify"] # 后续阶段未执行 + + +def test_rerun_after_failure(db, events, catalog): + """TC-BIO-RUN-003：失败后 rerun 成功，attempt=2，继续后续阶段。""" + executor = FakeExecutor(exit_code=1) + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False) + await runner.run_job(record) + assert record.status == JOB_FAILED + executor.exit_code = 0 + await runner.rerun_stage(record, "identify", None) + await _drain(runner) + return runner, record + + runner, record = _scenario(scenario) + assert runner.stage_status(record, "identify") == STAGE_DONE + assert record._stage_states["identify"]["attempt"] == 2 + assert record.status == JOB_DONE # 后续阶段自动推进 + + +def test_skip_after_failure(db, events, catalog): + """TC-BIO-RUN-004：失败后 skip → skipped，后续阶段可执行。""" + executor = FakeExecutor(exit_code=1) + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False) + await runner.run_job(record) + assert runner.stage_status(record, "identify") == STAGE_FAILED + executor.exit_code = 0 + await runner.skip_stage(record, "identify") + await _drain(runner) + return runner, record + + runner, record = _scenario(scenario) + assert runner.stage_status(record, "identify") == STAGE_SKIPPED + ev_types = [e["type"] for e in _collected(events, record.id)] + assert "stage_skipped" in ev_types + assert record.status == JOB_DONE + assert runner.stage_status(record, "phylogeny") == STAGE_DONE + + +def test_rerun_with_params_goes_confirm(db, events, catalog): + """TC-BIO-RUN-005：带参数重跑 → waiting_confirm 再确认；新参数进入执行。""" + executor = FakeExecutor(exit_code=1) + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + await runner.rerun_stage(record, "identify", {"cpu": "8"}) + assert runner.stage_status(record, "identify") == STAGE_WAITING_CONFIRM + executor.exit_code = 0 + await runner.confirm_stage(record, "identify", {"cpu": "8"}) + await _drain(runner) + return runner, record + + runner, record = _scenario(scenario) + assert record._stage_states["identify"]["params"]["cpu"] == "8" + assert runner.stage_status(record, "identify") == STAGE_DONE + assert executor.calls[-1]["cpu"] == "8" + + +def test_rerun_non_failed_rejected(db, events, catalog, fake_executor): + """TC-BIO-RUN-024：done 阶段 rerun/skip 拒绝。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + with pytest.raises(RunnerError, match="不可重跑"): + _scenario(lambda: runner.rerun_stage(record, "identify", None)) + with pytest.raises(RunnerError, match="不可跳过"): + _scenario(lambda: runner.skip_stage(record, "identify")) + + +def test_pause_at_stage_boundary(db, events, catalog): + """TC-BIO-RUN-006：暂停在阶段边界（当前阶段完成后停）。""" + executor = FakeExecutor(delay=0.05) + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False) + task = asyncio.create_task(runner.run_job(record)) + await asyncio.sleep(0.02) # identify 已进入执行 + await runner.pause_job(record) + await asyncio.wait_for(task, timeout=5) + return runner, record + + runner, record = _scenario(scenario) + assert record.status == JOB_PAUSED + assert runner.stage_status(record, "identify") == STAGE_DONE + assert all(c["stage"] != "phylogeny" for c in executor.calls) + + +def test_resume_continues_without_rerun(db, events, catalog): + """TC-BIO-RUN-007：暂停后恢复续跑，不重跑已完成。""" + executor = FakeExecutor(delay=0.02) + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False) + task = asyncio.create_task(runner.run_job(record)) + await asyncio.sleep(0.03) + await runner.pause_job(record) + await asyncio.wait_for(task, timeout=5) + assert record.status == JOB_PAUSED + await runner.resume_job(record) + await _drain(runner) + return runner, record + + runner, record = _scenario(scenario) + assert record.status == JOB_DONE + identify_calls = [c for c in executor.calls if c["stage"] == "identify"] + assert len(identify_calls) == 1 # 不重跑 + + +def test_timeout_fails_stage(db, events, catalog): + """TC-BIO-RUN-012：超时 → failed（reason=timeout）。""" + executor = FakeTimedOutExecutor() + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert runner.stage_status(record, "identify") == STAGE_FAILED + failed = [e for e in _collected(events, record.id) if e["type"] == "stage_failed"][0] + assert failed["data"]["reason"] == "timeout" + + +def test_log_tail_truncated(db, events, catalog): + """TC-BIO-RUN-013：日志截断只留尾 80 行。""" + executor = FakeExecutor(log="\n".join(f"L{i}" for i in range(500)) + "\n") + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + tail = record._stage_states["identify"]["log_tail"] + assert tail.count("\n") + 1 == 80 + assert tail.startswith("L420") + assert tail.rstrip().endswith("L499") + + +def test_exit_nonzero_judged_failed(db, events, catalog): + """TC-BIO-RUN-014：退出码非 0 判失败，log_tail 含 stderr。""" + executor = FakeExecutor(exit_code=3, log="error output\n") + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert runner.stage_status(record, "identify") == STAGE_FAILED + assert "error output" in record._stage_states["identify"]["log_tail"] + + +def test_manifest_missing_judged_failed(db, events, catalog): + """TC-BIO-RUN-015：退出码 0 但 manifest 缺 → failed（reason=no_manifest）。""" + executor = FakeExecutor(exit_code=0, manifest_status="missing") + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert runner.stage_status(record, "identify") == STAGE_FAILED + failed = [e for e in _collected(events, record.id) if e["type"] == "stage_failed"][0] + assert failed["data"]["reason"] == "no_manifest" + + +def test_manifest_failed_status(db, events, catalog): + """TC-BIO-RUN-016：manifest status=failed 判败（不因退出码 0 误判成功）。""" + executor = FakeExecutor(exit_code=0, manifest_status="failed") + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert runner.stage_status(record, "identify") == STAGE_FAILED + failed = [e for e in _collected(events, record.id) if e["type"] == "stage_failed"][0] + assert failed["data"]["reason"] == "manifest_failed" + + +def test_await_files_when_missing(db, events, catalog): + """TC-BIO-RUN-017：输入缺失等待文件（force_input_check=True）。""" + executor = FakeExecutor() + + async def scenario(): + runner = make_runner(db, events, executor, catalog, check_inputs=True) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert runner.stage_status(record, "identify") == "await_files" + ev = [e for e in _collected(events, record.id) if e["type"] == "stage_await_files"][0] + assert ev["data"]["stage"] == "identify" + assert ev["data"]["required_files"] + assert executor.calls == [] # 不启动 docker + + +def test_await_files_then_resume(db, events, catalog): + """TC-BIO-RUN-018：文件补齐自动推进。""" + executor = FakeExecutor() + + async def scenario(): + runner = make_runner(db, events, executor, catalog, check_inputs=True) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + assert runner.stage_status(record, "identify") == "await_files" + (record.input_dir / "genome.pep.fa").write_text(">x\nACGT\n", encoding="utf-8") + record._stage_states["identify"]["status"] = "waiting" + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert record.status == JOB_DONE + assert runner.stage_status(record, "identify") == STAGE_DONE + + +def test_await_confirm_manual_mode(db, events, catalog): + """TC-BIO-RUN-019/020：手动模式等待确认；确认后执行。""" + executor = FakeExecutor() + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify", "phylogeny"], auto_continue=False, auto_start=False) + await runner.run_job(record) + assert runner.stage_status(record, "identify") == STAGE_WAITING_CONFIRM + assert executor.calls == [] # 未确认不执行 + await runner.confirm_stage(record, "identify", {}) + await _drain(runner) + return runner, record + + runner, record = _scenario(scenario) + ev = [e for e in _collected(events, record.id) if e["type"] == "stage_await_confirm"][0] + assert ev["data"]["stage"] == "identify" + assert ev["data"]["form_defaults"] # 表单默认值 + assert runner.stage_status(record, "identify") == STAGE_DONE + assert runner.stage_status(record, "phylogeny") == STAGE_WAITING_CONFIRM + + +def test_confirm_unknown_param_rejected(db, events, catalog, fake_executor): + """TC-BIO-API-058 对应服务层：confirm 拒绝非 schema 参数。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_start=False) + await runner.run_job(record) + with pytest.raises(RunnerError, match="未知参数"): + await runner.confirm_stage(record, "identify", {"magic": "x"}) + return runner, record + + _scenario(scenario) + + +def test_confirm_non_confirmable(db, events, catalog, fake_executor): + """TC-BIO-API-059：重复 confirm 幂等拒绝。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + assert runner.stage_status(record, "identify") == STAGE_DONE + with pytest.raises(RunnerError, match="不可确认"): + await runner.confirm_stage(record, "identify", {}) + return runner, record + + _scenario(scenario) + + +def test_auto_continue_skips_confirm(db, events, catalog, fake_executor): + """TC-BIO-RUN-021：自动连续跳过确认全程推进。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert record.status == JOB_DONE + assert "stage_await_confirm" not in [e["type"] for e in _collected(events, record.id)] + + +def test_concurrency_limit(db, events, catalog): + """TC-BIO-RUN-010：并发上限排队（4 任务并发，同时 running ≤3）。""" + executor = FakeExecutor(delay=0.1) + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + records = [] + for i in range(4): + r = await runner.create_job( + f"s{i}", f"t{i}", ["identify"], auto_continue=True, auto_start=False) + records.append(r) + await asyncio.gather(*[runner.run_job(r) for r in records]) + return runner, records + + runner, records = _scenario(scenario) + assert executor.max_running <= 3 + assert all(r.status == JOB_DONE for r in records) + + +def test_semaphore_released_on_failure(db, events, catalog): + """TC-BIO-RUN-025：执行失败后信号量释放。""" + executor = FakeExecutor(exit_code=1) + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert record.status == JOB_FAILED + assert not runner.semaphore.locked() + + +def test_restart_recovery_no_rerun(db, events, catalog, fake_executor): + """TC-BIO-RUN-008：重启续跑不重跑已完成阶段（attempt 不变）。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False) + # 只完成第 1 阶段（模拟中断前状态） + await runner._run_stage(record, "identify", 0) + assert runner.stage_status(record, "identify") == STAGE_DONE + # 模拟重启：新 runner 实例（进程重启后内存清空） + runner2 = make_runner(db, events, fake_executor, catalog) + recovered = await runner2.recover() + await _drain(runner2) + return runner2, record, recovered + + runner2, record, recovered = _scenario(scenario) + assert any(r.id == record.id for r in recovered) + from app.services.bioflow import store + st = _scenario(lambda: store.get_stage(db, record.id, "identify")) + assert st["status"] == STAGE_DONE + assert st["attempt"] == 1 # 不重跑 + + +def test_restart_recovers_inflight_by_manifest(db, events, catalog, fake_executor): + """TC-BIO-RUN-009：重启恢复 in-flight running 阶段按 manifest 判成败。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False) + await runner._run_stage(record, "identify", 0) # 第 1 阶段完成 + # phylogeny 标记 running 且无 manifest（模拟崩溃瞬间） + record._stage_states["phylogeny"]["status"] = "running" + await runner._persist_stage(record, "phylogeny") + # 模拟重启 + runner2 = make_runner(db, events, fake_executor, catalog) + await runner2.recover() + await _drain(runner2) + # 同一 loop 内校验（aiosqlite 连接绑定本 loop） + from app.services.bioflow import store + st = await store.get_stage(db, record.id, "phylogeny") + assert st["status"] == STAGE_DONE + assert st["attempt"] >= 1 # 无 manifest → 重跑；attempt 至少 1 + return runner2, record + + _scenario(scenario) + + +def test_restart_skips_archived(db, events, catalog, fake_executor): + """TC-BIO-RUN-030：恢复时跳过已归档任务。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + # 归档（状态改为 archived） + await runner._persist_job(record) + from app.services.bioflow.store import update_job_status + await update_job_status(db, record.id, "archived") + runner2 = make_runner(db, events, fake_executor, catalog) + recovered = await runner2.recover() + return recovered + + recovered = _scenario(scenario) + assert recovered == [] + + +def test_empty_stages_done_immediately(db, events, catalog, fake_executor): + """TC-BIO-RUN-039：空阶段任务直接 done，无 docker 调用。""" + # 注意：创建任务要求 stages 非空（pydantic min_length=1）由路由层保证； + # 服务层直接构造空链验证防御 + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + record.stage_chain = [] + record._stage_states = {} + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert record.status == JOB_DONE + assert fake_executor.calls == [] + assert record.progress() == 100 + + +def test_progress_calculation(db, events, catalog, fake_executor): + """TC-BIO-RUN-029：任务进度数值。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify", "phylogeny", "motif"], + auto_continue=True, auto_start=False) + assert record.progress() == 0 + # 全链跑完 → 100（进度公式按完成阶段数/总数） + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + assert record.progress() == 100 + + +def test_unknown_stage_create_rejected(db, events, catalog, fake_executor): + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + with pytest.raises(RunnerError, match="未知阶段"): + await runner.create_job("s1", "t", ["ghost_stage"], auto_start=False) + return None + + _scenario(scenario) + + +def test_prerequisite_autofill(db, events, catalog, fake_executor): + """TC-BIO-RUN-034：前置补全后链完整（phylogeny 缺 identify → 自动补）。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job("s1", "t", ["phylogeny", "motif"], auto_start=False) + return record + + record = _scenario(scenario) + assert record.stage_chain[0] == "identify" + assert set(record.stage_chain) == {"identify", "phylogeny", "motif"} + + +def test_multi_job_isolation(db, events, catalog): + """TC-BIO-RUN-028：多任务互不串扰。""" + executor = FakeExecutor(log="job1 log\n") + + async def scenario(): + runner = make_runner(db, events, executor, catalog) + r1 = await runner.create_job("s1", "t1", ["identify"], auto_continue=True, auto_start=False) + r2 = await runner.create_job("s2", "t2", ["identify"], auto_continue=True, auto_start=False) + await asyncio.gather(runner.run_job(r1), runner.run_job(r2)) + return runner, r1, r2 + + runner, r1, r2 = _scenario(scenario) + assert r1.id != r2.id + assert r1.status == JOB_DONE and r2.status == JOB_DONE + assert r1._stage_states["identify"]["log_tail"] == "job1 log" # tail_lines 去尾换行 + + +def test_stage_log_event(db, events, catalog, fake_executor): + """TC-BIO-SSE-004：stage_log 事件随执行发布（log 非空）。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + log_events = [e for e in _collected(events, record.id) if e["type"] == "stage_log"] + assert log_events + assert all(e["data"]["log"] for e in log_events) + + +def test_event_seq_strictly_increasing(db, events, catalog, fake_executor): + """TC-BIO-SSE-013：seq 严格递增。""" + + async def scenario(): + runner = make_runner(db, events, fake_executor, catalog) + record = await runner.create_job( + "s1", "t", ["identify"], auto_continue=True, auto_start=False) + await runner.run_job(record) + return runner, record + + runner, record = _scenario(scenario) + seqs = [e["seq"] for e in _collected(events, record.id)] + assert seqs == sorted(seqs) + assert len(set(seqs)) == len(seqs) diff --git a/docs/team/report_bioflow_dev2.md b/docs/team/report_bioflow_dev2.md new file mode 100644 index 0000000..a805a6f --- /dev/null +++ b/docs/team/report_bioflow_dev2.md @@ -0,0 +1,167 @@ +# report_bioflow_dev2 —— P1 生信服务地基（开发 agent A2） + +> 任务书：docs/team/brief_bioflow_dev2_p1.md +> 工作分支：feature/bioflow-conversation-orchestration（保持，不切换） +> 纪律：不 commit/push/部署；不碰生产服务（8090/8091）；服务层零 pydantic；零硬编码 + +## 1. 交付概览 + +| 模块 | 说明 | +|---|---| +| bioflow_service/（新增，独立部署树） | 生信服务 8098：FastAPI + 独立 SQLite + docker 执行器 + 状态机 + 规划器 | +| sdk/app/routers/bioflow_proxy.py（新增） | BFF 流式反代 + 报告登记端点 | +| sdk/app/main.py（2 行改动） | BIOFLOW_ENABLED 开关注册反代路由 | +| sdk/app/config.py（2 键） | bioflow_enabled（默认 false）/ bioflow_service_url | +| docs/team/report_bioflow_dev2.md | 本报告 | + +## 2. 模块清单（bioflow_service/app/） + +**明文白名单（编译管线保留 .py）** +- `config.py` —— BIOFLOW_ 前缀配置（pydantic-settings）：image_map_path（仓库内 + bioflow_kernel/stage_image_map.json，按 __file__ 推导 repo 根）、data_root/ + jobs_dir/db_path/archives_dir/flow_docs_dir（全部可覆盖，无写死宿主路径）、 + concurrency=3、stage_timeout_sec、host_uid_env=BIOFLOW_HOST_UID（默认服务进程 + os.getuid() 推导）、container_data_dir=/bioflow-data、upload 白名单、DOCGEN_LLM_* + 沿用 docgen 直连模式；bba_env 属性从配置推导 BBA_PFAM_DIR/BBA_RAG_MAPPING_FILE。 +- `schemas.py` —— pydantic 请求/响应模型（ChatStreamRequest/JobCreateRequest/ + StageConfirmRequest/ArchiveCreateRequest 等 10 个；STAGE/JOB 状态 Literal）。 +- `db.py` —— 独立 SQLite（WAL）三表 + 幂等迁移（CREATE IF NOT EXISTS + PRAGMA + table_info 增量列）：bio_jobs/bio_stages/bio_archives（字段与实施计划 §六对齐）； + bio_stages UNIQUE(job_id, stage)，job 删除级联。 +- `main.py` —— FastAPI 组装 + lifespan（init_db→init_context→recover→shutdown）； + health 端点 /api/bioflow/health（db 连通状态字段，FR-BIO-025 探活用）。 + +**服务层（全部 dataclass/dict/stdlib，无 pydantic，Cython 编译）** +- `services/bioflow/kernel.py` —— Docker 执行器：读 A1 stage_image_map.json + （75 阶段/8 镜像），按阶段→镜像拼 docker run（--user uid/三目录挂载/BIOFLOW_DATA + 只读/-e BBA_*/--cpus/--memory/--rm + `python -B -m cli.run_{stage}`，list 无 shell）； + cpu/memory 白名单回落；超时 terminate→kill；日志落任务 logs/ + 尾 80 行截断； + 跨环境 note 透传（编排层 P2 拆分依据）。 +- `services/bioflow/runner.py` —— 任务状态机：created→await_inputs→running→ + paused/done/failed/archived；阶段 waiting→await_files→waiting_confirm→queued→ + running→done/failed/skipped；输入检查（FR-BIO-005）→确认门（FR-BIO-008，confirm + `_confirmed` 旁路防二次确认门）→并发信号量（BIOFLOW_CONCURRENCY 默认 3， + FIFO 排队）→docker→manifest 判成败（no_manifest/manifest_failed/timeout/ + exit_code 四因）；pause 停阶段边界、resume 不跳过确认、rerun/skip 状态校验； + 重启恢复 recover()：running 阶段按 manifest 判成败（有 success manifest→done + 否则重跑）、done 不重跑（attempt 不变）、archived 跳过。 +- `services/bioflow/catalog.py` —— 阶段目录（映射表驱动 + STAGE_INFO/STAGE_DEPENDENCIES + 静态快照，不运行时反射）；73 阶段（真实表 75 去重复校验）、基因家族链 21 阶段、 + 手写参数 schema（identify 11 参数含 default/source/help，source 标注 + contract/user/runtime 供表单默认值来源可见）；validate_chain 未知阶段拒绝 + + auto_fill_prerequisites 前置补全（拓扑排序依赖在前）。 +- `services/bioflow/planner.py` —— 路径规划：复用句式（「用 XXX 存档任务的方法…」） + → archive 直接装载；否则知识库检索（vstore/flow.py FlowStore 本地实例混合检索， + 失败降级 llm_only）+ LLM 候选（DOCGEN_LLM_* 直连，JSON 模式 + 修复重试）→ + catalog 硬校验（未知阶段拒绝、前置补全）；PLAN_LLM_TIMEOUT/PLAN_UNAVAILABLE + 降级内置兜底链。 +- `services/bioflow/manifests.py` —— manifest 解析校验（manifest_v1.schema 必需字段； + 定位 final/{prefix}.{stage}.manifest.json，无兜底防跨阶段误配）；跨阶段输入定位 + （artifacts full_path 解析 + 存在性检查）。 +- `services/bioflow/events.py` —— 事件总线（12 类事件全集 + job_recovered）： + 进程内 pub/sub（SSE 订阅）+ 内存环形缓冲 + events/{job_id}.jsonl 持久 + （重启回放）+ snapshot 首帧 + replay(after_seq) 断线回放。 +- `services/bioflow/llm.py` —— LLM 直连（docgen 同款：指数退避 + JSON 修复 + + LLMTimeout/LLMUnavailable 分型）。 +- `services/bioflow/params.py` —— 参数预填（契约文档参数解析 + BBA_* 默认值标注 + + 存档参数预填）。 +- `services/bioflow/reports.py` —— 报告登记（报告 HTML 定位/fig:// 图清单提取/ + 登记载荷（job_id/标题/md/图清单）+ 内部 HTTP 调 BFF 反代端点，BFF 不可达降级）。 +- `services/bioflow/archive.py` —— 存档打包（final/结果+logs/过程+events.jsonl + 过程记录+input 原始文件；include_intermediate 开关含中间文件；zip 下载）。 +- `services/bioflow/store.py` —— 三表 CRUD（upsert/list/get/delete；归档精确优先 + 模糊匹配、reused_count 计数）。 +- `services/bioflow/intent.py` —— 意图分流（cases_bioflow_intent.md 冻结词典同表： + export→edit→docgen→附件 docgen→bioflow_qa（FLOW_QA 命中即拦）→bioflow + （动词+家族语境词/分析对象）→qa 漏判安全；复用方法名提取 3-20 字符； + >2000 字符截断判定）。 + +**路由** +- `routers/bioflow.py` —— health/catalog（?stage=/?chain=gene_family）/ + chat/stream（SSE：意图分流→plan_ready 候选→心跳；plan seq 编码 data 载荷直出）/ + ask（知识库引用块：source/title_path/path/cite/line/text）。 +- `routers/biojobs.py` —— jobs 全套（POST 201/列表 rowid 倒序分页过滤/详情/删除 + running 409）；files（multipart 流式逐块写盘 + 扩展名白名单 + 大小上限 413 + + server_path 登记限 data_root/任务目录白名单 403）；stages confirm（参数校验 + 未知 422）/rerun（带参数→waiting_confirm）/skip/pause/resume/manifest（manifest + 对象 + 日志尾 80 行）；events SSE（snapshot 首帧 + 回放 + 实时订阅 + 任务终态 + 关闭 TC-BIO-API-081）；report revise（最小接口：无图目标 422/绘图阶段定位）。 +- `routers/bioarchives.py` —— archives 全套（存档 201 重名 409/列表/详情/删除 + 磁盘+行/run 复用/按名匹配 404 ARCHIVE_NOT_FOUND/download zip/report HTML）； + reports/register（归一化 + 图清单回填，BFF 反代落 doc_artifacts）。 + +**构建与运维** +- `build.sh` —— 复制 tooling/compile/compile_sdk.sh 改白名单（明文：main.py/ + config.py/schemas.py/db.py + __init__.py 规则；schemas= pydantic cyfunction 教训、 + db= 断流崩溃根治教训）；16 模块 Cython+GCC 全过（实测 /tmp 输出 16 .so， + 编译树 import OK）。 +- `restart.sh` —— 8098 重启（零硬编码：路径脚本位置推导 + 环境变量覆盖）。 +- `tests/` —— pytest（-m bioflow_unit 127 用例 / -m bioflow_api 39 用例）。 + +## 3. 自测结果（2026-09-06，服务器 venv 3.12.13） + +| 命令 | 结果 | +|---|---| +| `python -m pytest tests/ -q` | **166 passed**（15.7s，0 warnings） | +| `python -m pytest tests/ -q -m bioflow_unit` | 127 passed | +| `python -m pytest tests/ -q -m bioflow_api` | 39 passed | +| build.sh → /tmp 编译 + 编译树 import | 16 模块 .so，import OK | +| BFF BIOFLOW_ENABLED=false/true 双态 import | 均 OK（反代路由 2 条） | +| 真实 stage_image_map.json 加载 | 75 阶段/8 镜像/identify 11 参数/命令拼装正确 | +| /home/xzq 硬编码扫描（app+tests） | 0 处 | + +全部 mock docker/LLM/vstore（tmpdir + fake 执行器），不连任何生产服务 +（8090/8091/8094/8098 从未探测）。 + +## 4. 已知限制（交 TL 裁决） + +1. **SSE 传输层测试法**：本环境 fastapi TestClient 的 sync stream 包装与 httpx + ASGITransport 对永不结束的 SSE 流均存在首帧不达/整流缓冲问题（dbg3/dbg6 最小 + 复现实证），SSE 用例改为端点直调迭代 body_iterator + 模块级共享 event loop + （anyio portal 绑定首个 loop，跨 loop 死锁实证）。生产部署（uvicorn 真实网络 + SSE）不受影响；如 TL 要求 TestClient 原生流测试，需另行排查 fastapi/anyio + 版本组合。 +2. **catalog 阶段数口径**：任务书口径 73；A1 交付 stage_image_map.json 实为 75 + 阶段（含 module=null 的 synteny_plot_revise 与 executable=false 的 msa/ + enrichment）。kernel 对 module=null 阶段拼命令前置拦截（UnknownStageError）， + catalog 全量返回 75。以 A1 交付物为准，口径差异记录备查。 +3. **gene_family 链为 21 阶段**（A1 family_chain 实测 21，任务书口径约 18）； + 以 stage_image_map.json family_chain 字段为唯一事实源。 +4. **FlowStore 生产数据未入库**：vstore/data/flow_docs 目录当前不存在（A4 自测 + 在 tmpdir 完成；真实语料 106 文件/724 块需 TL 部署时跑 + vstore/scripts/ingest_flow_docs.py 入库）。planner 已按本地实例对接（加载成功 + 零块时检索空 → LLM 候选照常，降级链完整）；生产前入库即可获得 KB 增强 + （llm_rag source）。 +5. **chat/stream 是规划入口**：plan_ready 后保持心跳连接，任务创建走 + POST /jobs（P1 契约）；会话级任务事件经 /jobs/{id}/events 订阅。若 TL 要求 + chat/stream 单流贯穿任务事件（A3 前端形态），需 P2 加会话级订阅联动。 +6. **报告登记 BFF 侧 doc_artifacts 落库**：bioflow_proxy.register_report 字段 + （job_id/session_id/标题/md_path/图清单）已按约定实现；图清单字段当前存于 + 请求体（doc_artifacts 表无 figures 列，如需持久化图清单需 TL 裁决加列——已 + 在 payload 透传中保留）。与 A3 report 模板的 fig:// 渲染协调点：报告 HTML 内 + fig://id 引用清单由生信服务 reports.extract_figure_refs 提供。 +7. **意图词典判定为启发式快照**：与 cases_bioflow_intent.md 冻结表逐条对齐 + （40 用例表全过）；家族名语境词规则（「SPL 会议」不命中）为词典实现注意第 + 1/4 条的工程化解读，最终同表双跑一致性以 TC-BIO-INTENT-047（L2 用例）验收。 +8. **事件持久化目录**：events/{job_id}.jsonl 与任务目录分离（bioflow_data/ + jobs/events/）；归档时打包进档案根。任务删除不清理事件文件（留审计）， + 如需一并删除交 TL 裁决。 +9. **镜像 tag 后缀**：docker_image_tag_suffix 默认 "dev"（A1 交付物 :dev）； + 发布流水线（P2 FR-BIO-032 compile_bioflow.sh）需按终态 tag 覆盖环境变量。 +10. **pytest.ini python_files 陷阱**：本仓库 pytest 9.1.1 下 `python_files = + ["test_*.py"]`（列表形式）会导致目录级收集 0 用例（单文件显式传参正常）， + 已用字符串形式 `python_files = test_*.py` 规避并实证；写报告备案防踩。 + +## 5. 交付物清单 + +| 文件 | 说明 | +|---|---| +| bioflow_service/app/{config,schemas,db,main,context}.py | 应用骨架（4 明文 + context） | +| bioflow_service/app/services/bioflow/*.py | 服务层 12 模块（零 pydantic） | +| bioflow_service/app/routers/{bioflow,biojobs,bioarchives}.py | 端点三层 | +| bioflow_service/build.sh / restart.sh | 编译白名单管线 / 重启脚本 | +| bioflow_service/pytest.ini | markers + python_files（字符串形式陷阱规避） | +| bioflow_service/tests/*.py | 10 文件 166 用例 | +| sdk/app/routers/bioflow_proxy.py | BFF 流式反代 + 报告登记 | +| sdk/app/main.py（+2 行）/ sdk/app/config.py（+2 键） | BFF 最小改动 | +| docs/team/report_bioflow_dev2.md | 本报告 | diff --git a/sdk/app/config.py b/sdk/app/config.py index 21ec891..6aa7222 100644 --- a/sdk/app/config.py +++ b/sdk/app/config.py @@ -72,6 +72,10 @@ class Settings(BaseSettings): bff_history_max_question_chars: int = 4000 # 单条 user 文本截断 bff_history_max_answer_chars: int = 12000 # 单条 assistant 答案截断 + # ---- 生信服务反代（FR-BIO-027/028；BIOFLOW_ENABLED 为假时反代路由不注册）---- + bioflow_enabled: bool = False # 三形态部署开关（默认关：只部署 rag 无生信痕迹） + bioflow_service_url: str = "http://127.0.0.1:8098" # 生信服务 8098（BFF 流式反代目标） + # ---- 生产形态：BFF 静态托管 ui/dist（ARCHITECTURE.md §3，M6 挂载）---- frontend_dist_dir: str = "ui/dist" # 生产形态：静态托管 ui/dist diff --git a/sdk/app/main.py b/sdk/app/main.py index 15958b7..ca6134b 100644 --- a/sdk/app/main.py +++ b/sdk/app/main.py @@ -23,6 +23,7 @@ from fastapi.staticfiles import StaticFiles from .config import get_settings from .db import close_db, init_db from .routers import capabilities, docgen, extraction, figures, health, kg, legacy, qa, sessions +from .routers import bioflow_proxy from .upstream import close_client logging.basicConfig(level=logging.INFO) @@ -54,6 +55,8 @@ def create_app() -> FastAPI: app.include_router(figures.router) app.include_router(legacy.router) app.include_router(docgen.router) # docgen（P2） + if get_settings().bioflow_enabled: + app.include_router(bioflow_proxy.router) # 生信反代（BIOFLOW_ENABLED 开关，FR-BIO-028） @app.exception_handler(RequestValidationError) async def _validation_handler(_: Request, exc: RequestValidationError) -> JSONResponse: diff --git a/sdk/app/routers/bioflow_proxy.py b/sdk/app/routers/bioflow_proxy.py new file mode 100644 index 0000000..d79ae6e --- /dev/null +++ b/sdk/app/routers/bioflow_proxy.py @@ -0,0 +1,163 @@ +"""routers/bioflow_proxy.py —— BFF 生信反代（FR-BIO-027/028：BFF 仅流式反代 + 报告登记）。 + +- 流式反代 /api/bioflow/* → BIOFLOW_SERVICE_URL（SSE 不缓冲透传： + httpx client.stream 逐块 yield；反代请求超时阶梯——上传/执行类端点单独调大）。 +- 报告登记端点 /api/bioflow/reports/register：BFF 侧落 doc_artifacts + （字段与 docgen 现有表对齐：job_id/标题/md/图清单——与 A3 report 模板协调）， + 落库后透传生信服务归一化响应。 +- BIOFLOW_ENABLED 为假时路由不注册（main.py 按开关 include；三形态部署）。 + +约束： +- 不缓冲 SSE（TC-BIO-API-102 首帧延迟要求）。 +- 反代失败统一 502 UPSTREAM_DOWN 信封（经 app.upstream.envelope）。 +- 本文件是任务书列出的 BFF 三处最小改动之一。 +""" +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse, StreamingResponse + +from ..config import get_settings +from ..db import Database, get_db +from ..upstream import envelope + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/bioflow", tags=["bioflow-proxy"]) + +# 反代超时阶梯（秒）：普通读 120；上传/报告登记等大体积端点单独调大 +PROXY_READ_TIMEOUT = 120.0 +PROXY_UPLOAD_TIMEOUT = 3600.0 +PROXY_CONNECT_TIMEOUT = 5.0 + +_LONG_TIMEOUT_PATHS = ("/files", "/reports/register", "/archives", "/download") + + +def _proxy_timeout(path: str) -> float: + if any(seg in path for seg in _LONG_TIMEOUT_PATHS): + return PROXY_UPLOAD_TIMEOUT + return PROXY_READ_TIMEOUT + + +def _service_base() -> str: + return get_settings().bioflow_service_url.rstrip("/") + + +async def _proxy(request: Request, path: str) -> Any: + """httpx 流式反代（SSE 透传不缓冲）；失败→502 信封。""" + import httpx + + base = _service_base() + url = f"{base}/api/bioflow/{path.lstrip('/')}" + if request.url.query: + url = f"{url}?{request.url.query}" + body = await request.body() + headers = {k: v for k, v in request.headers.items() + if k.lower() not in ("host", "content-length", "connection")} + timeout = httpx.Timeout(connect=PROXY_CONNECT_TIMEOUT, read=_proxy_timeout(path), + write=PROXY_UPLOAD_TIMEOUT, pool=5.0) + try: + async with httpx.AsyncClient(timeout=timeout) as client: + req = client.build_request( + request.method, url, content=body, headers=headers, + ) + upstream = await client.send(req, stream=True) + except httpx.HTTPError as exc: + logger.warning("bioflow_proxy: 生信服务不可达 %s: %s", url, exc) + payload, status = envelope("UPSTREAM_DOWN", message="生信服务当前不可用，请稍后重试", + degraded=True, status=503) + return JSONResponse(status_code=status, content=payload) + + content_type = upstream.headers.get("content-type", "") + if "text/event-stream" in content_type: + # SSE 透传：逐块 yield，绝不缓冲 + async def stream(): + try: + async for chunk in upstream.aiter_raw(): + yield chunk + finally: + await upstream.aclose() + + resp_headers = { + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "Connection": "keep-alive", + } + return StreamingResponse(stream(), status_code=upstream.status_code, + media_type="text/event-stream", headers=resp_headers) + + try: + content = await upstream.aread() + await upstream.aclose() + except httpx.HTTPError: + payload, status = envelope("UPSTREAM_DOWN", message="生信服务响应中断", + degraded=True, status=502) + return JSONResponse(status_code=status, content=payload) + media_type = upstream.headers.get("content-type") or "application/json" + return JSONResponse(status_code=upstream.status_code, content=content, + media_type=media_type) + + +# --------------------------------------------------------------------------- +# 报告登记（BFF 侧落 doc_artifacts；FR-BIO-016/104 字段与 docgen 表对齐） +# --------------------------------------------------------------------------- + + +@router.post("/reports/register") +async def register_report(request: Request, db: Database = Depends(get_db)) -> Any: + """生信报告 → doc_artifacts 行（BFF 拥有 docgen 表）。 + + 先透传生信服务归一化响应（生信服务返回 {ok,data} 归一化载荷含图清单回填）， + 再落 doc_artifacts。落库失败不阻断反代响应（报告仍可直连下载；记日志）。 + """ + try: + payload = await request.json() + except Exception: + payload, status = envelope("VALIDATION", message="请求体须为 JSON", status=422) + return JSONResponse(status_code=status, content=payload) + + # 先透传生信服务归一化（含图清单回填） + proxied = await _proxy(request, "reports/register") + + # BFF 落 doc_artifacts（尽力而为；字段对齐 docgen 表） + try: + import uuid + + data = payload.get("data", payload) if isinstance(payload, dict) else {} + artifact_id = uuid.uuid4().hex[:16] + await db.execute( + """INSERT INTO doc_artifacts (id, session_id, status, title, template_name, + md_path, docx_path, pdf_path, error, created_at) + VALUES (?, ?, 'done', ?, 'bioflow_report', ?, NULL, NULL, NULL, ?)""", + ( + artifact_id, + str(data.get("session_id") or ""), + str(data.get("title") or "生信分析报告"), + str(data.get("md_path") or ""), + _now_iso(), + ), + ) + logger.info("bioflow_proxy: 生信报告登记 doc_artifacts 行 %s（job=%s）", + artifact_id, data.get("job_id")) + except Exception: # noqa: BLE001 - 落库失败不阻断（docgen 表可能未建） + logger.exception("bioflow_proxy: doc_artifacts 落库失败（报告仍直连可用）") + + return proxied + + +# --------------------------------------------------------------------------- +# 通用反代路由（其余 /api/bioflow/* 全部流式透传） +# --------------------------------------------------------------------------- + + +@router.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"]) +async def proxy_all(path: str, request: Request) -> Any: + return await _proxy(request, path) + + +def _now_iso() -> str: + from datetime import datetime, timezone + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")