feat(bioflow): P1 生信服务地基——bioflow_service 8098 独立部署树(12 服务模块+3 路由层+SQLite 三表+166 用例全绿)+BFF 流式反代/报告登记(BIOFLOW_ENABLED 开关默认关)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -0,0 +1 @@
|
||||
"""app/__init__.py —— bioflow-service 应用包。"""
|
||||
@@ -0,0 +1,133 @@
|
||||
"""config.py —— 生信服务（bioflow-service 8098）环境变量驱动配置（pydantic-settings）。
|
||||
|
||||
全部键可用环境变量覆盖（前缀 BIOFLOW_；LLM 沿用 docgen 的 DOCGEN_LLM_* 直连模式）。
|
||||
零硬编码纪律：不出现任何写死宿主路径；仓库内相对路径按 __file__ 推导 repo 根。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parent.parent.parent # bioflow_service/.. = 仓库根
|
||||
|
||||
|
||||
def _resolve_root(v: str) -> str:
|
||||
"""仓库内相对路径按 __file__ 推导 repo 根解析为绝对路径。"""
|
||||
p = Path(v).expanduser()
|
||||
if not p.is_absolute():
|
||||
p = _REPO_ROOT / p
|
||||
return str(p)
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
extra="ignore",
|
||||
env_prefix="BIOFLOW_",
|
||||
)
|
||||
|
||||
# ---- 应用 ----
|
||||
app_name: str = "plantai-bioflow-service"
|
||||
version: str = "0.1.0"
|
||||
|
||||
# ---- 阶段→镜像映射（A1 交付物，仓库内 bioflow_kernel/stage_image_map.json）----
|
||||
image_map_path: str = "bioflow_kernel/stage_image_map.json"
|
||||
|
||||
# ---- 数据与任务目录 ----
|
||||
data_root: str = "bioflow_data" # 生信数据根（hmm/rag_db 等，只读挂载进容器）
|
||||
jobs_dir: str = "bioflow_data/jobs" # 任务目录根（input/workspace/events）
|
||||
db_path: str = "bioflow_data/bioflow.db" # 独立 SQLite（与 BFF 完全隔离，FR-BIO-028）
|
||||
archives_dir: str = "bioflow_data/archives"
|
||||
|
||||
# ---- 执行 ----
|
||||
concurrency: int = 3 # 全局并发信号量上限（FR-BIO-031，可配）
|
||||
stage_timeout_sec: float = 36000.0 # 单阶段 docker 硬超时（秒，10h）
|
||||
docker_image_tag_suffix: str = "dev" # 镜像 tag 后缀（P0 交付物为 :dev）
|
||||
host_uid_env: str = "BIOFLOW_HOST_UID" # 宿主机 uid 环境变量名；未设置时回退服务进程 uid
|
||||
container_data_dir: str = "/bioflow-data"
|
||||
container_input_dir: str = "/job/input"
|
||||
container_workspace_dir: str = "/job/workspace"
|
||||
default_cpus: str = "4"
|
||||
default_memory: str = "32g"
|
||||
log_tail_lines: int = 80 # manifest 接口日志尾行数（≤80）
|
||||
max_upload_gb: float = 100.0 # 上传大小上限（GB；超限 413）
|
||||
upload_ext_allow: str = ".fasta,.fa,.faa,.fna,.pep,.cds,.gz,.gff,.gff3,.gtf,.txt,.tsv,.csv,.xlsx,.xls,.bed,.vcf,.vcf.gz,.bam,.cram,.sam,.hmm,.nwk,.newick,.treefile,.json,.yaml,.yml,.pdf"
|
||||
|
||||
# ---- 规划（planner）----
|
||||
llm_plan_timeout: float = 60.0 # 规划 LLM 单次调用超时（秒）
|
||||
plan_candidates: int = 3 # 候选路径条数（2~3，FR-BIO-002）
|
||||
plan_max_rounds: int = 2 # LLM 输出硬校验失败的最多重试轮数
|
||||
|
||||
# ---- 知识库（vstore/flow.py 本地实例；无 vstore 时自动降级 llm_only）----
|
||||
flow_docs_dir: str = "vstore/data/flow_docs"
|
||||
|
||||
# ---- LLM（沿用 docgen deepseek 直连模式，DOCGEN_LLM_* 键）----
|
||||
docgen_llm_base_url: str = "https://api.deepseek.com/v1"
|
||||
docgen_llm_api_key: str = ""
|
||||
docgen_llm_model: str = "deepseek-chat"
|
||||
docgen_llm_timeout: float = 300.0
|
||||
|
||||
# ---- 报告登记（BFF 反代端点；空=未配置，登记降级为仅落表）----
|
||||
service_url: str = "http://127.0.0.1:8098"
|
||||
|
||||
# ---- 容器环境变量（BBA_*；零硬编码：路径全部从本配置推导）----
|
||||
@property
|
||||
def bba_env(self) -> dict[str, str]:
|
||||
"""docker run 注入的 BBA_* 环境（值全部来自配置推导，无写死宿主路径）。"""
|
||||
return {
|
||||
"BBA_PFAM_DIR": f"{self.container_data_dir}/hmm",
|
||||
"BBA_RAG_MAPPING_FILE": f"{self.container_data_dir}/rag_db/pfam_mapping.json",
|
||||
"BBA_DISABLE_RAG_PFAM_LOOKUP": "0",
|
||||
}
|
||||
|
||||
@property
|
||||
def host_uid(self) -> int:
|
||||
"""--user 注入 uid：BIOFLOW_HOST_UID 环境变量优先，否则服务进程 uid。"""
|
||||
import os
|
||||
|
||||
raw = os.environ.get(self.host_uid_env, "").strip()
|
||||
if raw:
|
||||
try:
|
||||
return int(raw)
|
||||
except ValueError:
|
||||
pass
|
||||
return os.getuid()
|
||||
|
||||
@property
|
||||
def image_map_abs(self) -> str:
|
||||
return _resolve_root(self.image_map_path)
|
||||
|
||||
@property
|
||||
def data_root_abs(self) -> str:
|
||||
return _resolve_root(self.data_root)
|
||||
|
||||
@property
|
||||
def jobs_dir_abs(self) -> str:
|
||||
return _resolve_root(self.jobs_dir)
|
||||
|
||||
@property
|
||||
def db_path_abs(self) -> str:
|
||||
return _resolve_root(self.db_path)
|
||||
|
||||
@property
|
||||
def archives_dir_abs(self) -> str:
|
||||
return _resolve_root(self.archives_dir)
|
||||
|
||||
@property
|
||||
def flow_docs_dir_abs(self) -> str:
|
||||
return _resolve_root(self.flow_docs_dir)
|
||||
|
||||
@property
|
||||
def upload_ext_set(self) -> set[str]:
|
||||
return {ext.strip().lower() for ext in self.upload_ext_allow.split(",") if ext.strip()}
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_settings() -> Settings:
|
||||
return Settings()
|
||||
|
||||
|
||||
settings = get_settings()
|
||||
@@ -0,0 +1,213 @@
|
||||
"""app/context.py —— 生信服务共享实例容器（db/events/kernel/catalog/runner/semaphore/planner）。
|
||||
|
||||
- 生产：app 启动时 init_context() 加载真实依赖（SQLite、stage_image_map.json、
|
||||
vstore FlowStore 本地实例（存在时）、真 docker 执行器）。
|
||||
- 测试：build_context() 可注入 fake db/executor/flow_store（mock docker/LLM/vstore）。
|
||||
- vstore 不存在/不可用：FlowStore 为 None，planner 自动降级 llm_only（FR-BIO-010 降级）。
|
||||
- 并发信号量（BIOFLOW_CONCURRENCY，FR-BIO-031）每 context 一个。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .config import get_settings
|
||||
from .db import Database, db as default_db
|
||||
from .services.bioflow.catalog import StageCatalog, build_catalog
|
||||
from .services.bioflow.events import EventBus
|
||||
from .services.bioflow.kernel import Kernel, KernelConfig, load_kernel
|
||||
from .services.bioflow.planner import Planner
|
||||
from .services.bioflow.runner import Runner
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Context:
|
||||
db: Any = None
|
||||
events: EventBus | None = None
|
||||
kernel: Kernel | None = None
|
||||
catalog: StageCatalog | None = None
|
||||
runner: Runner | None = None
|
||||
semaphore: asyncio.Semaphore | None = None
|
||||
planner: Planner | None = None
|
||||
flow_store: Any = None
|
||||
executor: Any = None
|
||||
records: dict[str, Any] = field(default_factory=dict) # job_id → JobRecord（内存）
|
||||
_tasks: list[asyncio.Task] = field(default_factory=list)
|
||||
|
||||
def spawn(self, coro: Any) -> asyncio.Task:
|
||||
task = asyncio.create_task(coro)
|
||||
self._tasks.append(task)
|
||||
task.add_done_callback(lambda t: self._tasks.remove(t) if t in self._tasks else None)
|
||||
return task
|
||||
|
||||
async def close(self) -> None:
|
||||
for task in self._tasks:
|
||||
task.cancel()
|
||||
if self._tasks:
|
||||
await asyncio.gather(*self._tasks, return_exceptions=True)
|
||||
self._tasks.clear()
|
||||
|
||||
|
||||
# 全局 context（测试可整体替换）
|
||||
context: Context = Context()
|
||||
|
||||
|
||||
def init_events(jobs_dir: str) -> EventBus:
|
||||
from pathlib import Path
|
||||
|
||||
return EventBus(Path(jobs_dir) / "events")
|
||||
|
||||
|
||||
def init_flow_store(flow_docs_dir: str) -> Any | None:
|
||||
"""vstore FlowStore 本地实例（存在则加载；失败/缺失返回 None → planner 降级）。"""
|
||||
try:
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
repo_root = Path(__file__).resolve().parent.parent.parent
|
||||
if str(repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(repo_root))
|
||||
from vstore.flow import FlowStore # P6 A4 交付；缺失则降级
|
||||
|
||||
return FlowStore(flow_docs_dir)
|
||||
except Exception as exc: # noqa: BLE001 - 缺失/损坏降级 llm_only，不阻断服务
|
||||
logger.warning("context: vstore FlowStore 不可用（planner 降级 llm_only）: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def build_context(
|
||||
*,
|
||||
db: Database | None = None,
|
||||
events: EventBus | None = None,
|
||||
kernel: Kernel | None = None,
|
||||
catalog: StageCatalog | None = None,
|
||||
executor: Any = None,
|
||||
semaphore: asyncio.Semaphore | None = None,
|
||||
flow_store: Any = None,
|
||||
runner_kwargs: dict[str, Any] | None = None,
|
||||
) -> Context:
|
||||
"""构造 context（生产与测试共用；各依赖可注入）。
|
||||
|
||||
executor=None 时用 kernel.run_stage（真 docker）。
|
||||
"""
|
||||
settings = get_settings()
|
||||
database = db or default_db
|
||||
bus = events or init_events(settings.jobs_dir_abs)
|
||||
kern = kernel or load_kernel()
|
||||
cat = catalog or build_catalog(kern.map)
|
||||
sem = semaphore or asyncio.Semaphore(settings.concurrency)
|
||||
exec_fn = executor or kern.run_stage
|
||||
runner = Runner(
|
||||
database,
|
||||
bus,
|
||||
exec_fn,
|
||||
cat,
|
||||
sem,
|
||||
**(runner_kwargs or {}),
|
||||
)
|
||||
planner = Planner(
|
||||
cat,
|
||||
flow_store,
|
||||
llm_plan_timeout=settings.llm_plan_timeout,
|
||||
plan_candidates=settings.plan_candidates,
|
||||
plan_max_rounds=settings.plan_max_rounds,
|
||||
archive_finder=_make_archive_finder(database),
|
||||
)
|
||||
return Context(
|
||||
db=database,
|
||||
events=bus,
|
||||
kernel=kern,
|
||||
catalog=cat,
|
||||
runner=runner,
|
||||
semaphore=sem,
|
||||
planner=planner,
|
||||
flow_store=flow_store,
|
||||
executor=exec_fn,
|
||||
)
|
||||
|
||||
|
||||
def _make_archive_finder(database: Database):
|
||||
"""planner 复用归档查找回调（async；store 查询）。"""
|
||||
from .services.bioflow import store
|
||||
|
||||
async def find(name: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
return await store.find_archive_by_name(database, name)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
return find
|
||||
|
||||
|
||||
async def init_context(db_path: str | None = None) -> Context:
|
||||
"""应用启动：初始化全局 context（幂等；db_path 供测试替换）。"""
|
||||
global context
|
||||
from .db import init_db
|
||||
|
||||
database = context.db
|
||||
if db_path is not None:
|
||||
database = await init_db(db_path)
|
||||
elif context.db is None:
|
||||
database = await init_db()
|
||||
context = build_context(db=database)
|
||||
return context
|
||||
|
||||
|
||||
async def shutdown_context() -> None:
|
||||
await context.close()
|
||||
|
||||
|
||||
# ---- JobRecord 注册与恢复（路由层操作任务的内存记录）----
|
||||
|
||||
|
||||
def register_record(record: Any) -> None:
|
||||
context.records[record.id] = record
|
||||
|
||||
|
||||
def _record_from_job(job: dict[str, Any], stages: list[dict[str, Any]]) -> Any:
|
||||
"""DB 行 → JobRecord（无运行中内存记录时恢复，跨重启安全）。"""
|
||||
from .services.bioflow.runner import JobRecord, params_dict
|
||||
|
||||
record = JobRecord(
|
||||
id=job["id"],
|
||||
session_id=job["session_id"],
|
||||
title=job["title"],
|
||||
status=job["status"],
|
||||
stage_chain=list(job["stage_chain"] or []),
|
||||
params=params_dict(job.get("params")),
|
||||
auto_continue=bool(job.get("auto_continue")),
|
||||
archive_id=job.get("archive_id"),
|
||||
)
|
||||
for s in stages:
|
||||
record._stage_states[s["stage"]] = {
|
||||
"status": s["status"],
|
||||
"params": params_dict(s.get("params")),
|
||||
"attempt": int(s.get("attempt") or 0),
|
||||
"manifest_path": s.get("manifest_path"),
|
||||
"log_tail": s.get("log_tail") or "",
|
||||
"error": s.get("error"),
|
||||
"started_at": s.get("started_at"),
|
||||
"finished_at": s.get("finished_at"),
|
||||
}
|
||||
return record
|
||||
|
||||
|
||||
async def load_record(job_id: str) -> Any | None:
|
||||
"""取任务 JobRecord：内存优先，否则 DB 恢复（并注册回内存）。"""
|
||||
if job_id in context.records:
|
||||
return context.records[job_id]
|
||||
if context.db is None:
|
||||
return None
|
||||
from .services.bioflow import store
|
||||
|
||||
job = await store.get_job(context.db, job_id)
|
||||
if job is None:
|
||||
return None
|
||||
stages = await store.list_stages(context.db, job_id)
|
||||
record = _record_from_job(job, stages)
|
||||
context.records[job_id] = record
|
||||
return record
|
||||
@@ -0,0 +1,200 @@
|
||||
"""db.py —— 生信服务独立 SQLite（aiosqlite + WAL，无 ORM），三张表 + 幂等迁移。
|
||||
|
||||
FR-BIO-028：与 BFF 完全隔离（独立进程 + 独立 bioflow.db）。
|
||||
表（实施计划 §六三表清单）：
|
||||
- bio_jobs：id / session_id / title / status / stage_chain(JSON) / params(JSON) /
|
||||
archive_id / auto_continue / progress / error / created_at / updated_at
|
||||
- bio_stages：id / job_id(FK CASCADE) / stage / status / params(JSON) / manifest_path /
|
||||
log_tail / attempt / started_at / finished_at / error
|
||||
- bio_archives：id / job_id / name / description / stage_chain(JSON) / params(JSON) /
|
||||
report_artifact / disk_path / size_bytes / reused_count / created_at
|
||||
|
||||
时间一律 ISO 8601 UTC（由服务侧生成，同 docgen 模式）。
|
||||
模式沿用 docgen 同款：CREATE TABLE IF NOT EXISTS + PRAGMA table_info 幂等加列。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import AsyncIterator, Iterable
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiosqlite
|
||||
|
||||
from .config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS bio_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
session_id TEXT NOT NULL DEFAULT '',
|
||||
title TEXT NOT NULL DEFAULT '生信任务',
|
||||
status TEXT NOT NULL DEFAULT 'created',
|
||||
stage_chain TEXT NOT NULL DEFAULT '[]',
|
||||
params TEXT NOT NULL DEFAULT '{}',
|
||||
archive_id TEXT,
|
||||
auto_continue INTEGER NOT NULL DEFAULT 0,
|
||||
progress INTEGER NOT NULL DEFAULT 0,
|
||||
error TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bio_jobs_session ON bio_jobs(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_bio_jobs_status ON bio_jobs(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_bio_jobs_updated ON bio_jobs(updated_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bio_stages (
|
||||
id TEXT PRIMARY KEY,
|
||||
job_id TEXT NOT NULL REFERENCES bio_jobs(id) ON DELETE CASCADE,
|
||||
stage TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'waiting',
|
||||
params TEXT NOT NULL DEFAULT '{}',
|
||||
manifest_path TEXT,
|
||||
log_tail TEXT NOT NULL DEFAULT '',
|
||||
attempt INTEGER NOT NULL DEFAULT 0,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
error TEXT,
|
||||
UNIQUE(job_id, stage)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bio_stages_job ON bio_stages(job_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bio_archives (
|
||||
id TEXT PRIMARY KEY,
|
||||
job_id TEXT NOT NULL,
|
||||
name TEXT NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
stage_chain TEXT NOT NULL DEFAULT '[]',
|
||||
params TEXT NOT NULL DEFAULT '{}',
|
||||
report_artifact TEXT,
|
||||
disk_path TEXT,
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
reused_count INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bio_archives_name ON bio_archives(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_bio_archives_created ON bio_archives(created_at);
|
||||
"""
|
||||
|
||||
# 增量列幂等迁移（老库已建表时 CREATE IF NOT EXISTS 不会加列）
|
||||
_BIO_JOBS_MIGRATIONS = (
|
||||
("archive_id", "ALTER TABLE bio_jobs ADD COLUMN archive_id TEXT"),
|
||||
("error", "ALTER TABLE bio_jobs ADD COLUMN error TEXT"),
|
||||
)
|
||||
_BIO_ARCHIVES_MIGRATIONS = (
|
||||
("report_artifact", "ALTER TABLE bio_archives ADD COLUMN report_artifact TEXT"),
|
||||
)
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def dumps_json(value: Any, default: Any = None) -> str:
|
||||
try:
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
except (TypeError, ValueError):
|
||||
return json.dumps(default, ensure_ascii=False)
|
||||
|
||||
|
||||
def loads_json(text: str | None, default: Any = None) -> Any:
|
||||
if text is None:
|
||||
return default
|
||||
try:
|
||||
return json.loads(text)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
class Database:
|
||||
"""aiosqlite 单连接封装（WAL；写串行化，同 docgen 模式）。"""
|
||||
|
||||
def __init__(self, path: str | Path | None = None) -> None:
|
||||
self._path: Path | None = Path(path) if path is not None else None
|
||||
self._conn: aiosqlite.Connection | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
@property
|
||||
def path(self) -> Path:
|
||||
if self._path is None:
|
||||
self._path = Path(get_settings().db_path_abs)
|
||||
return self._path
|
||||
|
||||
async def connect(self) -> None:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._conn = await aiosqlite.connect(self.path)
|
||||
self._conn.row_factory = aiosqlite.Row
|
||||
await self._conn.execute("PRAGMA journal_mode=WAL")
|
||||
await self._conn.execute("PRAGMA foreign_keys=ON")
|
||||
await self._conn.execute("PRAGMA busy_timeout=5000")
|
||||
await self._conn.executescript(_SCHEMA)
|
||||
await self._migrate_columns()
|
||||
await self._conn.commit()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._conn is not None:
|
||||
await self._conn.close()
|
||||
self._conn = None
|
||||
|
||||
async def _migrate_columns(self) -> None:
|
||||
"""三表增量列幂等迁移（同 docgen _migrate_docgen_columns 模式）。"""
|
||||
for table, migrations in (
|
||||
("bio_jobs", _BIO_JOBS_MIGRATIONS),
|
||||
("bio_archives", _BIO_ARCHIVES_MIGRATIONS),
|
||||
):
|
||||
cur = await self._conn.execute(f"PRAGMA table_info({table})")
|
||||
cols = {row[1] for row in await cur.fetchall()}
|
||||
for name, ddl in migrations:
|
||||
if name not in cols:
|
||||
await self._conn.execute(ddl)
|
||||
|
||||
@property
|
||||
def conn(self) -> aiosqlite.Connection:
|
||||
if self._conn is None:
|
||||
raise RuntimeError("Database 未初始化：先 await init_db()")
|
||||
return self._conn
|
||||
|
||||
async def fetchall(self, sql: str, params: Iterable[Any] = ()) -> list[aiosqlite.Row]:
|
||||
async with self._conn.execute(sql, tuple(params)) as cursor:
|
||||
return list(await cursor.fetchall())
|
||||
|
||||
async def fetchone(self, sql: str, params: Iterable[Any] = ()) -> aiosqlite.Row | None:
|
||||
async with self._conn.execute(sql, tuple(params)) as cursor:
|
||||
return await cursor.fetchone()
|
||||
|
||||
async def execute(self, sql: str, params: Iterable[Any] = ()) -> int:
|
||||
async with self._lock:
|
||||
cur = await self._conn.execute(sql, tuple(params))
|
||||
await self._conn.commit()
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
db = Database()
|
||||
|
||||
|
||||
async def init_db(path: str | Path | None = None) -> Database:
|
||||
global db
|
||||
if path is not None:
|
||||
db = Database(path)
|
||||
await db.connect()
|
||||
return db
|
||||
|
||||
|
||||
async def get_db() -> AsyncIterator[Database]:
|
||||
yield db
|
||||
|
||||
|
||||
async def close_db() -> None:
|
||||
await db.close()
|
||||
|
||||
|
||||
def monotonic() -> float:
|
||||
return time.monotonic()
|
||||
@@ -0,0 +1,80 @@
|
||||
"""main.py —— 生信服务应用入口（uvicorn 入口，明文白名单文件）。
|
||||
|
||||
- 独立部署树 bioflow_service/（FR-BIO-028：独立进程 8098 + 独立 SQLite）。
|
||||
- lifespan：init_db（幂等迁移）→ init_context（kernel/catalog/planner/runner）
|
||||
→ recover（服务重启恢复 in-flight 续跑，FR-BIO-031）→ shutdown 清理。
|
||||
- 端点：health（/api/bioflow/health，前端探活 FR-BIO-025）。
|
||||
- 不依赖 BFF/生产服务（8090/8091 等绝不探测/连接）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from .config import get_settings
|
||||
from .context import context, init_context, shutdown_context
|
||||
from .db import close_db, init_db
|
||||
from .routers import bioarchives, bioflow, biojobs
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
settings = get_settings()
|
||||
app = FastAPI(title=settings.app_name, version=settings.version)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"], # 生信服务直连形态（FR-BIO-027）；BFF 反代为同源
|
||||
allow_credentials=False,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def _lifespan(_app: FastAPI):
|
||||
# 测试注入：context.db 已由测试预置（build_context 注入的临时库），
|
||||
# 跳过启动初始化（不替换全局 db、不加载真实映射表、不 recover）。
|
||||
# 注意经模块属性动态读取（monkeypatch 替换的是 context 模块的全局）。
|
||||
from . import context as context_mod
|
||||
|
||||
injected = context_mod.context.db is not None
|
||||
if not injected:
|
||||
try:
|
||||
await init_db()
|
||||
except Exception:
|
||||
# SQLite 初始化失败不阻断服务启动（health 报 db 降级；端点返回 INTERNAL）
|
||||
logger.exception("main: 数据库初始化失败（health 将报 db 降级）")
|
||||
try:
|
||||
await init_context()
|
||||
except Exception:
|
||||
logger.exception("main: context 初始化失败（kernel 映射表缺失时 docker 执行不可用）")
|
||||
try:
|
||||
if context_mod.context.runner is not None:
|
||||
recovered = await context_mod.context.runner.recover()
|
||||
if recovered:
|
||||
logger.info("main: 重启恢复 %d 个 in-flight 任务续跑", len(recovered))
|
||||
except Exception:
|
||||
logger.exception("main: 重启恢复扫描失败（任务不重跑，等待后续操作）")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await shutdown_context()
|
||||
try:
|
||||
await close_db()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
app.router.lifespan_context = _lifespan
|
||||
|
||||
app.include_router(bioflow.router)
|
||||
app.include_router(biojobs.router)
|
||||
app.include_router(bioarchives.router)
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1 @@
|
||||
"""app/routers/__init__.py —— bioflow-service routers 包。"""
|
||||
@@ -0,0 +1,240 @@
|
||||
"""app/routers/bioarchives.py —— 存档端点（archives 全套 + 报告登记接口）。
|
||||
|
||||
端点（实施计划 §六清单）：
|
||||
- POST /api/bioflow/archives 存档（FR-BIO-020/021）
|
||||
- GET /api/bioflow/archives 列表（FR-BIO-022）
|
||||
- GET /api/bioflow/archives/{id} 详情
|
||||
- DELETE /api/bioflow/archives/{id} 删除（磁盘目录 + 行）
|
||||
- POST /api/bioflow/archives/{id}/run 复用发起（FR-BIO-023）
|
||||
- POST /api/bioflow/archives/run 按名称匹配复用（精确优先/模糊候选）
|
||||
- GET /api/bioflow/archives/{id}/download 打包下载（zip）
|
||||
- GET /api/bioflow/archives/{id}/report 报告单独取（HTML）
|
||||
- POST /api/bioflow/reports/register 报告登记接口（BFF 反代调此端点
|
||||
→ 落 doc_artifacts 协调字段）
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
import uuid
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
|
||||
from ..config import get_settings
|
||||
from ..context import Context, context, load_record
|
||||
from ..schemas import ArchiveCreateRequest, ArchiveRunRequest
|
||||
from ..services.bioflow import store
|
||||
from ..services.bioflow.archive import (
|
||||
archive_from_job,
|
||||
normalize_archive_name,
|
||||
remove_archive_dir,
|
||||
)
|
||||
from ..services.bioflow.reports import extract_figure_refs, locate_report, report_payload
|
||||
from .bioflow import job_or_404
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/bioflow", tags=["bioflow-archives"])
|
||||
|
||||
|
||||
def _err(code: str, message: str, status: int) -> JSONResponse:
|
||||
return JSONResponse(status_code=status, content={
|
||||
"ok": False,
|
||||
"error": {"code": code, "message": message, "degraded": False},
|
||||
})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 报告登记（BFF 反代调此端点；FR-BIO-016 与 A3 report 模板协调）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/reports/register")
|
||||
async def reports_register(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""报告登记接口：BFF 把生信报告登记进 doc_artifacts（字段约定
|
||||
job_id/session_id/标题/md_path/图清单——写入报告交 TL 转达 A3）。
|
||||
|
||||
生信服务侧只做 payload 归一化与图清单回填；doc_artifacts 落库由 BFF
|
||||
反代路由 bioflow_proxy.py 完成（BFF 拥有 docgen 表）。
|
||||
"""
|
||||
job_id = str(payload.get("job_id") or "")
|
||||
record = await load_record(job_id) if job_id else None
|
||||
report_path = payload.get("md_path") or payload.get("report_path") or ""
|
||||
figures: list[str] = list(payload.get("figures") or [])
|
||||
if record is not None and not figures:
|
||||
located = locate_report(record.workspace_dir)
|
||||
if located:
|
||||
figures = extract_figure_refs(located)
|
||||
normalized = {
|
||||
"job_id": job_id,
|
||||
"session_id": payload.get("session_id") or "",
|
||||
"title": payload.get("title") or (record.title if record else "生信分析报告"),
|
||||
"md_path": str(report_path),
|
||||
"figures": figures,
|
||||
"registered_at": _now_iso(),
|
||||
}
|
||||
return {"ok": True, "data": normalized}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# archives
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/archives", status_code=201)
|
||||
async def create_archive(body: ArchiveCreateRequest) -> dict[str, Any]:
|
||||
"""存档（FR-BIO-020/021）：打包任务目录 + 落 bio_archives 行。"""
|
||||
record = await job_or_404(body.job_id)
|
||||
ctx: Context = context
|
||||
name = normalize_archive_name(body.name)
|
||||
# 重名 409（TC-BIO-API-098 两态断言取拒绝态）
|
||||
existing = await store.find_archive_by_name(ctx.db, name)
|
||||
if existing is not None:
|
||||
return _err("ARCHIVE_EXISTS", f"归档名已存在: {name}", 409)
|
||||
report_path = locate_report(record.workspace_dir)
|
||||
result = await archive_from_job(
|
||||
{
|
||||
"id": record.id, "session_id": record.session_id,
|
||||
"title": record.title, "stage_chain": record.stage_chain,
|
||||
"params": record.params,
|
||||
},
|
||||
record.job_dir,
|
||||
get_settings().archives_dir_abs,
|
||||
name=name,
|
||||
description=body.description,
|
||||
include_intermediate=body.include_intermediate,
|
||||
report_artifact=str(report_path) if report_path else None,
|
||||
)
|
||||
await store.insert_archive(ctx.db, {
|
||||
"id": result.id,
|
||||
"job_id": record.id,
|
||||
"name": name,
|
||||
"description": body.description,
|
||||
"stage_chain": record.stage_chain,
|
||||
"params": record.params,
|
||||
"report_artifact": result.report_artifact,
|
||||
"disk_path": result.disk_path,
|
||||
"size_bytes": result.size_bytes,
|
||||
})
|
||||
ctx.events.publish("job_archived", {"job_id": record.id, "archive_id": result.id})
|
||||
return {"ok": True, "data": {
|
||||
"id": result.id,
|
||||
"job_id": record.id,
|
||||
"name": name,
|
||||
"description": body.description,
|
||||
"disk_path": result.disk_path,
|
||||
"size_bytes": result.size_bytes,
|
||||
"report_artifact": result.report_artifact,
|
||||
}}
|
||||
|
||||
|
||||
@router.get("/archives")
|
||||
async def list_archives(
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
ctx: Context = context
|
||||
items, total = await store.list_archives(ctx.db, page=page, page_size=page_size)
|
||||
return {"ok": True, "data": {"items": items, "total": total, "page": page, "page_size": page_size}}
|
||||
|
||||
|
||||
@router.get("/archives/{archive_id}")
|
||||
async def get_archive(archive_id: str) -> dict[str, Any]:
|
||||
ctx: Context = context
|
||||
archive = await store.get_archive(ctx.db, archive_id)
|
||||
if archive is None:
|
||||
return _err("NOT_FOUND", "存档不存在", 404)
|
||||
return {"ok": True, "data": archive}
|
||||
|
||||
|
||||
@router.delete("/archives/{archive_id}", status_code=204)
|
||||
async def delete_archive(archive_id: str) -> JSONResponse:
|
||||
ctx: Context = context
|
||||
archive = await store.get_archive(ctx.db, archive_id)
|
||||
if archive is None:
|
||||
return _err("NOT_FOUND", "存档不存在", 404)
|
||||
if archive.get("disk_path"):
|
||||
remove_archive_dir(get_settings().archives_dir_abs, archive_id)
|
||||
await store.delete_archive(ctx.db, archive_id)
|
||||
return JSONResponse(status_code=204, content=None)
|
||||
|
||||
|
||||
@router.post("/archives/{archive_id}/run", status_code=201)
|
||||
async def archive_run(archive_id: str, body: ArchiveRunRequest) -> dict[str, Any]:
|
||||
"""复用发起（FR-BIO-023）：装载阶段链+参数 → 新建任务（表单预填存档值）。"""
|
||||
ctx: Context = context
|
||||
archive = await store.get_archive(ctx.db, archive_id)
|
||||
if archive is None:
|
||||
return _err("ARCHIVE_NOT_FOUND", f"存档不存在: {archive_id}", 404)
|
||||
from ..services.bioflow.runner import RunnerError
|
||||
|
||||
params = {**dict(archive.get("params") or {}), **dict(body.params or {})}
|
||||
title = f"{archive.get('name') or '复用任务'}（{body.target or '复用'}）"
|
||||
try:
|
||||
record = await ctx.runner.create_job(
|
||||
body.session_id, title, list(archive.get("stage_chain") or []),
|
||||
params, archive_id=archive_id,
|
||||
)
|
||||
except RunnerError as exc:
|
||||
return _err(exc.code, exc.message, 422)
|
||||
await store.bump_archive_reuse(ctx.db, archive_id)
|
||||
ctx.records[record.id] = record
|
||||
from .biojobs import _job_detail
|
||||
|
||||
return {"ok": True, "data": await _job_detail(record)}
|
||||
|
||||
|
||||
@router.post("/archives/run", status_code=201)
|
||||
async def archive_run_by_name(body: ArchiveRunRequest) -> dict[str, Any]:
|
||||
"""按名称匹配复用（FR-BIO-023）：精确优先；模糊给候选；无匹配 404。"""
|
||||
ctx: Context = context
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
return _err("VALIDATION", "缺少归档名 name", 422)
|
||||
archive = await store.find_archive_by_name(ctx.db, name)
|
||||
if archive is None:
|
||||
return _err("ARCHIVE_NOT_FOUND", f"未找到匹配存档: {name}", 404)
|
||||
return await archive_run(archive["id"], ArchiveRunRequest(
|
||||
session_id=body.session_id, target=body.target, params=body.params or {},
|
||||
))
|
||||
|
||||
|
||||
@router.get("/archives/{archive_id}/download")
|
||||
async def archive_download(archive_id: str) -> FileResponse:
|
||||
"""归档打包下载（zip；TC-BIO-API-141）。"""
|
||||
ctx: Context = context
|
||||
archive = await store.get_archive(ctx.db, archive_id)
|
||||
if archive is None:
|
||||
return _err("NOT_FOUND", "存档不存在", 404)
|
||||
disk = Path(archive.get("disk_path") or "")
|
||||
if not disk.is_dir():
|
||||
return _err("NOT_FOUND", "档案磁盘目录缺失", 404)
|
||||
zip_path = disk.parent / f"{archive_id}.zip"
|
||||
if not zip_path.exists():
|
||||
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
for p in disk.rglob("*"):
|
||||
if p.is_file():
|
||||
zf.write(p, p.relative_to(disk))
|
||||
return FileResponse(zip_path, filename=f"{archive.get('name') or archive_id}.zip")
|
||||
|
||||
|
||||
@router.get("/archives/{archive_id}/report")
|
||||
async def archive_report(archive_id: str) -> FileResponse:
|
||||
"""归档报告单独取（HTML；FR-BIO-022 详情回看）。"""
|
||||
ctx: Context = context
|
||||
archive = await store.get_archive(ctx.db, archive_id)
|
||||
if archive is None:
|
||||
return _err("NOT_FOUND", "存档不存在", 404)
|
||||
report = archive.get("report_artifact")
|
||||
if not report or not Path(report).exists():
|
||||
return _err("NOT_FOUND", "该存档无报告", 404)
|
||||
return FileResponse(report, media_type="text/html")
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
from datetime import datetime, timezone
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
@@ -0,0 +1,276 @@
|
||||
"""app/routers/bioflow.py —— 生信服务核心端点（health/chat 流/catalog/知识库咨询）。
|
||||
|
||||
端点（实施计划 §六清单）：
|
||||
- GET /api/bioflow/health 探活（前端显隐 FR-BIO-025）
|
||||
- POST /api/bioflow/chat/stream SSE 入口：意图分流→规划候选→事件流
|
||||
- GET /api/bioflow/catalog 阶段目录（?chain=gene_family / ?stage=identify）
|
||||
- POST /api/bioflow/ask 生信流内流程咨询（知识库引用 FR-BIO-026）
|
||||
|
||||
事件类型全集（12 类）与 seq 严格递增由 app/context 的 EventBus 保证。
|
||||
LLM/vstore/docker 全部 mock 时端点行为确定性可断言（L2 层用例）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from ..config import get_settings
|
||||
from ..context import Context, context, load_record
|
||||
from ..db import Database, get_db
|
||||
from ..schemas import ChatStreamRequest, FlowAskRequest
|
||||
from ..services.bioflow import store
|
||||
from ..services.bioflow.intent import route as route_intent
|
||||
from ..services.bioflow.planner import PlannerError, PlanResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/bioflow", tags=["bioflow"])
|
||||
|
||||
KEEPALIVE_INTERVAL = 15.0
|
||||
KEEPALIVE_COMMENT = b": keepalive\n\n"
|
||||
|
||||
|
||||
def _err(code: str, message: str, status: int) -> JSONResponse:
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
return JSONResponse(status_code=status, content={
|
||||
"ok": False,
|
||||
"error": {"code": code, "message": message, "degraded": False},
|
||||
})
|
||||
|
||||
|
||||
def _ctx() -> Context:
|
||||
return context
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 健康探活（FR-BIO-025；db 连通状态字段 TC-BIO-API-099）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def health() -> dict[str, Any]:
|
||||
ctx = _ctx()
|
||||
db_ok = True
|
||||
db_error: str | None = None
|
||||
if ctx.db is not None:
|
||||
try:
|
||||
await ctx.db.fetchone("SELECT 1 AS ok")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
db_ok = False
|
||||
db_error = str(exc)
|
||||
return {
|
||||
"ok": True,
|
||||
"status": "ok" if db_ok else "degraded",
|
||||
"db": {"connected": db_ok, "error": db_error},
|
||||
"version": get_settings().version,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 知识库流程咨询（FR-BIO-026；引用块携带 cite 供前端跳转——A3 渲染）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/ask")
|
||||
async def flow_ask(body: FlowAskRequest) -> dict[str, Any]:
|
||||
"""生信流内流程咨询：FlowStore 检索 + 引用块；无 vstore 时降级空引用。"""
|
||||
ctx = _ctx()
|
||||
hits: list[dict[str, Any]] = []
|
||||
if ctx.flow_store is not None:
|
||||
try:
|
||||
result = ctx.flow_store.search(body.question, top_k=8)
|
||||
hits = result.get("items") or []
|
||||
except Exception: # noqa: BLE001
|
||||
hits = []
|
||||
citations = []
|
||||
for hit in hits:
|
||||
cite = hit.get("cite") or {}
|
||||
citations.append({
|
||||
"source": hit.get("source") or hit.get("doc_kind") or "",
|
||||
"title_path": hit.get("title_path") or "",
|
||||
"path": hit.get("path") or "",
|
||||
"cite": cite,
|
||||
"line_start": hit.get("line_start"),
|
||||
"line_end": hit.get("line_end"),
|
||||
"text": _hit_text(hit),
|
||||
})
|
||||
return {
|
||||
"ok": True,
|
||||
"answer": {
|
||||
"question": body.question,
|
||||
"citations": citations,
|
||||
"degraded": not citations,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _hit_text(hit: dict[str, Any]) -> str:
|
||||
"""引用块原文（chunks 明文；检索 meta 不含正文时尝试 get_chunk）。"""
|
||||
text = str(hit.get("text") or "")
|
||||
if text:
|
||||
return text[:1200]
|
||||
ctx = _ctx()
|
||||
if ctx.flow_store is not None:
|
||||
doc = ctx.flow_store.get_chunk(str(hit.get("chunk_key") or hit.get("id") or ""))
|
||||
if doc:
|
||||
return str(doc.get("text") or "")[:1200]
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# chat/stream —— SSE 入口（FR-BIO-001/002/003/005/023/026 全链路）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _sse(event: dict[str, Any]) -> bytes:
|
||||
"""SSE 帧编码：event: 类型 + data: 载荷（不含信封壳，前端 parser 契约）。"""
|
||||
data = json.dumps(event.get("data") or {}, ensure_ascii=False)
|
||||
return f"event: {event['type']}\ndata: {data}\n\n".encode("utf-8")
|
||||
|
||||
|
||||
def _sse_error(code: str, message: str, seq: int = -1) -> bytes:
|
||||
payload = {"type": "error", "seq": seq, "data": {"code": code, "message": message}}
|
||||
return _sse(payload)
|
||||
|
||||
|
||||
@router.post("/chat/stream")
|
||||
async def chat_stream(body: ChatStreamRequest) -> StreamingResponse:
|
||||
"""对话 SSE：意图分流 → 规划（plan_ready）→ 事件订阅流。
|
||||
|
||||
非生信消息：message_done（不建任务，TC-BIO-API-005）。
|
||||
规划降级：PLAN_LLM_TIMEOUT / PLAN_UNAVAILABLE / PLAN_FALLBACK（内置兜底链）。
|
||||
客户端断开：生成器取消 → 订阅清理（TC-BIO-API-014）。
|
||||
"""
|
||||
ctx = _ctx()
|
||||
last_seq = body.last_seq if body.last_seq is not None else -1
|
||||
|
||||
async def gen():
|
||||
# 0. 生信流内流程咨询（has_bioflow_context + FLOW_QA_WORD）→ 知识库引用
|
||||
intent = route_intent(
|
||||
body.message,
|
||||
has_attachments=body.has_attachments,
|
||||
has_active_artifact=body.has_active_artifact,
|
||||
has_bioflow_context=body.has_bioflow_context,
|
||||
)
|
||||
if intent == "bioflow_qa":
|
||||
qa_payload = await flow_ask(FlowAskRequest(session_id=body.session_id, question=body.message))
|
||||
yield _sse({"type": "message_done", "seq": -2, "data": {
|
||||
"kind": "flow_qa",
|
||||
"citations": qa_payload["answer"]["citations"],
|
||||
"degraded": qa_payload["answer"]["degraded"],
|
||||
}})
|
||||
return
|
||||
if intent != "bioflow":
|
||||
yield _sse({"type": "message_done", "seq": -2, "data": {
|
||||
"kind": "qa", "message": "已转交知识问答处理", "routed": intent,
|
||||
}})
|
||||
return
|
||||
|
||||
# 1. 规划（候选路径；复用/LLM/降级）
|
||||
try:
|
||||
plan: PlanResult = await ctx.planner.plan(body.message)
|
||||
except PlannerError as exc:
|
||||
# 降级为内置兜底链（不中断 SSE；TC-BIO-API-009/020）
|
||||
from ..services.bioflow.planner import PlanCandidate
|
||||
|
||||
fallback = [s for s in ("identify",) if ctx.catalog.has(s)]
|
||||
plan = PlanResult(
|
||||
candidates=[PlanCandidate(
|
||||
purpose="基因家族鉴定标准流程（内置默认）",
|
||||
stages=fallback,
|
||||
)],
|
||||
source="llm_only",
|
||||
message=exc.message,
|
||||
)
|
||||
plan_payload = {
|
||||
"candidates": [{"purpose": c.purpose, "stages": c.stages} for c in plan.candidates],
|
||||
"source": plan.source,
|
||||
"auto_filled": plan.auto_filled,
|
||||
"reuse_archive": plan.reuse_archive_id,
|
||||
"reuse_name": plan.reuse_name,
|
||||
"fallback": plan.message or None,
|
||||
}
|
||||
# 若该会话已有历史事件流（断线重连走 events 接口），此处 job_id 待前端确认创建
|
||||
yield _sse({"type": "plan_ready", "seq": -2, "data": plan_payload})
|
||||
|
||||
# 2. 后续事件：规划完成后会话级任务可能在别处创建（本接口为规划入口），
|
||||
# 保持连接发送心跳；对 job_created 后的任务终态事件做会话级短订阅。
|
||||
# P1 语义：前端在此流上收到 plan_ready 后经 /jobs 创建任务，
|
||||
# 任务事件经 /jobs/{id}/events 订阅；本流保持心跳即可。
|
||||
try:
|
||||
while True:
|
||||
await asyncio.sleep(KEEPALIVE_INTERVAL)
|
||||
yield KEEPALIVE_COMMENT
|
||||
except asyncio.CancelledError:
|
||||
logger.info("chat_stream: 客户端断开 %s", body.session_id)
|
||||
|
||||
headers = {
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
return StreamingResponse(gen(), media_type="text/event-stream", headers=headers)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# catalog（FR-BIO-002/003；catalog 返回 73 阶段 TC-BIO-API-021）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/catalog")
|
||||
async def catalog(
|
||||
chain: str | None = Query(default=None),
|
||||
stage: str | None = Query(default=None),
|
||||
) -> dict[str, Any]:
|
||||
ctx = _ctx()
|
||||
cat = ctx.catalog
|
||||
|
||||
if stage:
|
||||
entry = cat.get(stage)
|
||||
if entry is None:
|
||||
return _err("NOT_FOUND", f"阶段不存在: {stage}", 404)
|
||||
return {"ok": True, "data": {"stage": entry.to_dict()}}
|
||||
|
||||
if chain:
|
||||
resolved = cat.resolve_chain(chain)
|
||||
if resolved is None:
|
||||
return _err("NOT_FOUND", f"链不存在: {chain}", 404)
|
||||
items = []
|
||||
for name in resolved:
|
||||
entry = cat.get(name)
|
||||
if entry:
|
||||
items.append(entry.to_dict())
|
||||
return {"ok": True, "data": {"chain": chain, "stages": items, "total": len(items)}}
|
||||
|
||||
items = []
|
||||
for name in cat.stage_names():
|
||||
entry = cat.get(name)
|
||||
if entry:
|
||||
items.append(entry.to_dict())
|
||||
return {"ok": True, "data": {"stages": items, "total": len(items)}}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# jobs 通用取记录依赖（供 biojobs/bioarchives 共用）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def get_record(job_id: str) -> Any | None:
|
||||
return await load_record(job_id)
|
||||
|
||||
|
||||
async def job_or_404(job_id: str) -> Any:
|
||||
record = await get_record(job_id)
|
||||
if record is None:
|
||||
from fastapi import HTTPException
|
||||
|
||||
raise HTTPException(status_code=404, detail={
|
||||
"ok": False, "error": {"code": "NOT_FOUND", "message": "任务不存在", "degraded": False},
|
||||
})
|
||||
return record
|
||||
@@ -0,0 +1,485 @@
|
||||
"""app/routers/biojobs.py —— 任务端点（jobs 全套 + files 上传 + stages 控制 + events SSE）。
|
||||
|
||||
端点（实施计划 §六清单）：
|
||||
- POST /api/bioflow/jobs 创建任务（201；FR-BIO-004）
|
||||
- GET /api/bioflow/jobs 列表（分页/过滤；FR-BIO-024）
|
||||
- GET /api/bioflow/jobs/{id} 详情（含 stages）
|
||||
- DELETE /api/bioflow/jobs/{id} 删除（running 409）
|
||||
- POST /api/bioflow/jobs/{id}/files 上传（multipart 流式，大文件超时单独调大）
|
||||
或服务器路径登记（JSON body）
|
||||
- GET /api/bioflow/jobs/{id}/files 文件列表
|
||||
- GET /api/bioflow/jobs/{id}/files/{fid}/content 文件内容
|
||||
- POST /api/bioflow/jobs/{id}/stages/{stage}/confirm 参数确认执行
|
||||
- POST /api/bioflow/jobs/{id}/stages/{stage}/rerun 失败重跑（带参数→waiting_confirm）
|
||||
- POST /api/bioflow/jobs/{id}/stages/{stage}/skip 失败跳过
|
||||
- POST /api/bioflow/jobs/{id}/pause / resume 暂停/恢复
|
||||
- GET /api/bioflow/jobs/{id}/stages/{stage}/manifest manifest + 日志尾 80 行
|
||||
- GET /api/bioflow/jobs/{id}/events SSE 事件流（快照首帧 + 断线回放）
|
||||
- POST /api/bioflow/report/{job_id}/revise 报告图表修订（FR-BIO-019，最小接口）
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, Request, UploadFile
|
||||
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
|
||||
|
||||
from ..config import get_settings
|
||||
from ..context import Context, context, load_record
|
||||
from ..db import Database, get_db
|
||||
from ..schemas import (
|
||||
ArchiveRunRequest,
|
||||
JobCreateRequest,
|
||||
PathRegisterRequest,
|
||||
StageConfirmRequest,
|
||||
StageRerunRequest,
|
||||
ReportReviseRequest,
|
||||
)
|
||||
from ..services.bioflow import store
|
||||
from ..services.bioflow.kernel import tail_lines
|
||||
from ..services.bioflow.manifests import ManifestError, locate_manifest, parse_manifest
|
||||
from ..services.bioflow.runner import (
|
||||
JOB_RUNNING,
|
||||
STAGE_FAILED,
|
||||
STAGE_RUNNING,
|
||||
RunnerError,
|
||||
)
|
||||
from .bioflow import job_or_404
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/bioflow", tags=["bioflow-jobs"])
|
||||
|
||||
KEEPALIVE_INTERVAL = 15.0
|
||||
KEEPALIVE_COMMENT = b": keepalive\n\n"
|
||||
SNAPSHOT_TYPE = "snapshot"
|
||||
|
||||
|
||||
def _err(code: str, message: str, status: int) -> JSONResponse:
|
||||
return JSONResponse(status_code=status, content={
|
||||
"ok": False,
|
||||
"error": {"code": code, "message": message, "degraded": False},
|
||||
})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# jobs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/jobs", status_code=201)
|
||||
async def create_job(body: JobCreateRequest) -> dict[str, Any]:
|
||||
"""创建任务（FR-BIO-004；未知阶段 422；复用归档 TC-BIO-API-029）。"""
|
||||
ctx: Context = context
|
||||
params: dict[str, dict[str, Any]] = dict(body.params or {})
|
||||
archive_id = body.archive_id
|
||||
if body.archive_id:
|
||||
archive = await store.get_archive(ctx.db, body.archive_id)
|
||||
if archive is None:
|
||||
return _err("ARCHIVE_NOT_FOUND", f"存档不存在: {body.archive_id}", 404)
|
||||
params = {**dict(archive.get("params") or {}), **params}
|
||||
await store.bump_archive_reuse(ctx.db, body.archive_id)
|
||||
try:
|
||||
record = await ctx.runner.create_job(
|
||||
body.session_id, body.title, body.stages, params,
|
||||
auto_continue=body.auto_continue, archive_id=archive_id,
|
||||
)
|
||||
except RunnerError as exc:
|
||||
return _err(exc.code, exc.message, 422)
|
||||
ctx.records[record.id] = record
|
||||
return {"ok": True, "data": await _job_detail(record)}
|
||||
|
||||
|
||||
async def _job_detail(record: Any) -> dict[str, Any]:
|
||||
stages = []
|
||||
for stage in record.stage_chain:
|
||||
st = record._stage_states.get(stage, {})
|
||||
stages.append({
|
||||
"stage": stage,
|
||||
"status": st.get("status", "waiting"),
|
||||
"params": st.get("params") or {},
|
||||
"attempt": int(st.get("attempt") or 0),
|
||||
"manifest_path": st.get("manifest_path"),
|
||||
"log_tail": st.get("log_tail") or "",
|
||||
"error": st.get("error"),
|
||||
"started_at": st.get("started_at"),
|
||||
"finished_at": st.get("finished_at"),
|
||||
})
|
||||
return {
|
||||
"id": record.id,
|
||||
"session_id": record.session_id,
|
||||
"title": record.title,
|
||||
"status": record.status,
|
||||
"stages": stages,
|
||||
"params": record.params,
|
||||
"auto_continue": record.auto_continue,
|
||||
"archive_id": record.archive_id,
|
||||
"progress": record.progress(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/jobs")
|
||||
async def list_jobs(
|
||||
session_id: str | None = Query(default=None),
|
||||
status: str | None = Query(default=None),
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=20, ge=1, le=100),
|
||||
) -> dict[str, Any]:
|
||||
ctx: Context = context
|
||||
items, total = await store.list_jobs(
|
||||
ctx.db, session_id=session_id, status=status, page=page, page_size=page_size,
|
||||
)
|
||||
for item in items:
|
||||
stages = await store.list_stages(ctx.db, item["id"])
|
||||
done = sum(1 for s in stages if s["status"] in ("done", "skipped"))
|
||||
total_stages = len(stages)
|
||||
item["progress"] = int(done * 100 / total_stages) if total_stages else 100
|
||||
item["stages_count"] = total_stages
|
||||
return {"ok": True, "data": {"items": items, "total": total, "page": page, "page_size": page_size}}
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}")
|
||||
async def get_job(job_id: str) -> dict[str, Any]:
|
||||
record = await load_record(job_id)
|
||||
if record is None:
|
||||
return _err("NOT_FOUND", "任务不存在", 404)
|
||||
return {"ok": True, "data": await _job_detail(record)}
|
||||
|
||||
|
||||
@router.delete("/jobs/{job_id}", status_code=204)
|
||||
async def delete_job(job_id: str) -> JSONResponse:
|
||||
record = await load_record(job_id)
|
||||
if record is None:
|
||||
return _err("NOT_FOUND", "任务不存在", 404)
|
||||
if record.status in (JOB_RUNNING,):
|
||||
return _err("JOB_RUNNING", "任务执行中不可删除", 409)
|
||||
ctx: Context = context
|
||||
await store.delete_job(ctx.db, job_id) # bio_stages 级联删除
|
||||
ctx.records.pop(job_id, None)
|
||||
return JSONResponse(status_code=204, content=None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# files（FR-BIO-006 上传 / FR-BIO-007 路径登记）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _safe_name(name: str) -> str:
|
||||
"""文件名净化（保留中文；去路径分隔符）。"""
|
||||
return Path(name or "upload").name or "upload"
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/files")
|
||||
async def upload_file(
|
||||
job_id: str,
|
||||
request: Request,
|
||||
file: UploadFile | None = File(default=None),
|
||||
) -> dict[str, Any]:
|
||||
"""上传（multipart 流式）或服务器路径登记（JSON body）。
|
||||
|
||||
大文件超时单独调大：本端点自身逐块流式写盘（不整读内存，TC-BIO-API-041）。
|
||||
"""
|
||||
record = await job_or_404(job_id)
|
||||
ctx: Context = context
|
||||
settings = get_settings()
|
||||
input_dir = record.input_dir
|
||||
input_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
content_type = request.headers.get("content-type", "")
|
||||
if "multipart/form-data" not in content_type:
|
||||
# JSON 路径登记（FR-BIO-007）
|
||||
try:
|
||||
body = await request.json()
|
||||
except Exception: # noqa: BLE001
|
||||
return _err("VALIDATION", "请求体须为 multipart 或 JSON 路径登记", 400)
|
||||
try:
|
||||
payload = PathRegisterRequest.model_validate(body)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return _err("VALIDATION", f"路径登记参数不合法: {exc}", 422)
|
||||
return await _register_path(record, payload.server_path, settings)
|
||||
|
||||
if file is None or not file.filename:
|
||||
return _err("VALIDATION", "缺少上传文件", 422)
|
||||
ext = Path(file.filename).suffix.lower()
|
||||
if ext and ext not in settings.upload_ext_set:
|
||||
return _err("VALIDATION", f"扩展名不允许: {ext}", 422)
|
||||
dest = input_dir / _safe_name(file.filename)
|
||||
written = 0
|
||||
max_bytes = int(settings.max_upload_gb * 1024 ** 3)
|
||||
try:
|
||||
with dest.open("wb") as fh:
|
||||
while True:
|
||||
chunk = await file.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
written += len(chunk)
|
||||
if written > max_bytes:
|
||||
fh.close()
|
||||
dest.unlink(missing_ok=True)
|
||||
return _err("UPLOAD_TOO_LARGE",
|
||||
f"超过上传上限 {settings.max_upload_gb}GB（已收 {written} 字节）", 413)
|
||||
fh.write(chunk)
|
||||
except Exception as exc: # noqa: BLE001 - 写盘失败不登记（TC-BIO-API-050）
|
||||
logger.exception("files: 上传写盘失败 %s", job_id)
|
||||
dest.unlink(missing_ok=True)
|
||||
return _err("INTERNAL", f"上传写盘失败: {exc}", 500)
|
||||
file_id = f"{job_id}:{dest.name}"
|
||||
return {"ok": True, "data": {
|
||||
"file_id": file_id, "name": dest.name, "size": written,
|
||||
"path": str(dest), "source": "upload",
|
||||
}}
|
||||
|
||||
|
||||
async def _register_path(record: Any, server_path: str, settings: Any) -> dict[str, Any]:
|
||||
"""服务器路径登记（FR-BIO-007）：限 BIOFLOW_DATA_ROOT/任务目录内。"""
|
||||
path = Path(server_path).expanduser().resolve()
|
||||
if not path.exists():
|
||||
return _err("PATH_NOT_EXIST", f"路径不存在: {server_path}", 400)
|
||||
if not path.is_file():
|
||||
return _err("NOT_A_FILE", f"非文件: {server_path}", 400)
|
||||
allowed_roots = [Path(settings.data_root_abs).resolve(), record.job_dir.resolve()]
|
||||
if not any(path.is_relative_to(root) for root in allowed_roots):
|
||||
return _err("PATH_NOT_ALLOWED", f"路径不在白名单根内: {server_path}", 403)
|
||||
return {"ok": True, "data": {
|
||||
"file_id": f"{record.id}:{path.name}", "name": path.name,
|
||||
"size": path.stat().st_size, "path": str(path), "source": "server_path",
|
||||
}}
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}/files")
|
||||
async def list_files(job_id: str) -> dict[str, Any]:
|
||||
record = await job_or_404(job_id)
|
||||
items: list[dict[str, Any]] = []
|
||||
input_dir = record.input_dir
|
||||
if input_dir.is_dir():
|
||||
for p in sorted(input_dir.iterdir()):
|
||||
if p.is_file():
|
||||
items.append({
|
||||
"file_id": f"{job_id}:{p.name}", "name": p.name,
|
||||
"size": p.stat().st_size, "path": str(p),
|
||||
"source": "upload", "registered_at": None,
|
||||
})
|
||||
return {"ok": True, "data": {"items": items, "total": len(items)}}
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}/files/{file_id}/content")
|
||||
async def file_content(job_id: str, file_id: str) -> FileResponse:
|
||||
record = await job_or_404(job_id)
|
||||
name = file_id.split(":", 1)[-1] if ":" in file_id else file_id
|
||||
path = record.input_dir / _safe_name(name)
|
||||
if not path.is_file():
|
||||
return _err("NOT_FOUND", "文件不存在", 404)
|
||||
return FileResponse(path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# stages 控制（confirm/rerun/skip/pause/resume/manifest）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _stage_exists(record: Any, stage: str) -> None:
|
||||
if stage not in record.stage_chain:
|
||||
raise HTTPException(status_code=404, detail={
|
||||
"ok": False, "error": {"code": "NOT_FOUND", "message": f"阶段不存在: {stage}", "degraded": False},
|
||||
})
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/stages/{stage}/confirm")
|
||||
async def confirm_stage(job_id: str, stage: str, body: StageConfirmRequest) -> dict[str, Any]:
|
||||
record = await job_or_404(job_id)
|
||||
_stage_exists(record, stage)
|
||||
ctx: Context = context
|
||||
try:
|
||||
await ctx.runner.confirm_stage(record, stage, dict(body.params or {}))
|
||||
except RunnerError as exc:
|
||||
return _err(exc.code, exc.message, 409)
|
||||
# 确认已受理：执行任务已派生（异步推进中）；当前快照可能仍为 waiting_confirm
|
||||
return {"ok": True, "data": {
|
||||
"job_id": job_id, "stage": stage,
|
||||
"status": "running",
|
||||
"params": record._stage_states.get(stage, {}).get("params") or {},
|
||||
}}
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/stages/{stage}/rerun")
|
||||
async def rerun_stage(job_id: str, stage: str, body: StageRerunRequest) -> dict[str, Any]:
|
||||
record = await job_or_404(job_id)
|
||||
_stage_exists(record, stage)
|
||||
ctx: Context = context
|
||||
try:
|
||||
await ctx.runner.rerun_stage(record, stage, dict(body.params) if body.params else None)
|
||||
except RunnerError as exc:
|
||||
return _err(exc.code, exc.message, 409)
|
||||
return {"ok": True, "data": {
|
||||
"job_id": job_id, "stage": stage,
|
||||
"status": ctx.runner.stage_status(record, stage),
|
||||
"attempt": record._stage_states.get(stage, {}).get("attempt"),
|
||||
}}
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/stages/{stage}/skip")
|
||||
async def skip_stage(job_id: str, stage: str) -> dict[str, Any]:
|
||||
record = await job_or_404(job_id)
|
||||
_stage_exists(record, stage)
|
||||
ctx: Context = context
|
||||
try:
|
||||
await ctx.runner.skip_stage(record, stage)
|
||||
except RunnerError as exc:
|
||||
return _err(exc.code, exc.message, 409)
|
||||
return {"ok": True, "data": {
|
||||
"job_id": job_id, "stage": stage,
|
||||
"status": ctx.runner.stage_status(record, stage),
|
||||
}}
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/pause")
|
||||
async def pause_job(job_id: str) -> dict[str, Any]:
|
||||
record = await job_or_404(job_id)
|
||||
ctx: Context = context
|
||||
try:
|
||||
await ctx.runner.pause_job(record)
|
||||
except RunnerError as exc:
|
||||
return _err(exc.code, exc.message, 409)
|
||||
return {"ok": True, "data": {"job_id": job_id, "status": record.status}}
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/resume")
|
||||
async def resume_job(job_id: str) -> dict[str, Any]:
|
||||
record = await job_or_404(job_id)
|
||||
ctx: Context = context
|
||||
try:
|
||||
await ctx.runner.resume_job(record)
|
||||
except RunnerError as exc:
|
||||
return _err(exc.code, exc.message, 409)
|
||||
return {"ok": True, "data": {"job_id": job_id, "status": record.status}}
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}/stages/{stage}/manifest")
|
||||
async def stage_manifest(job_id: str, stage: str) -> dict[str, Any]:
|
||||
"""manifest + 日志尾 80 行（FR-BIO-011；TC-BIO-API-071~076）。"""
|
||||
record = await job_or_404(job_id)
|
||||
_stage_exists(record, stage)
|
||||
manifest: dict[str, Any] | None = None
|
||||
manifest_error: str | None = None
|
||||
try:
|
||||
parsed = parse_manifest(record.workspace_dir, stage, None)
|
||||
manifest = parsed.raw
|
||||
except ManifestError as exc:
|
||||
manifest_error = str(exc)
|
||||
log_tail = ""
|
||||
log_path = record.logs_dir / f"{stage}.log"
|
||||
if log_path.exists():
|
||||
try:
|
||||
log_tail = tail_lines(log_path.read_text(encoding="utf-8", errors="replace"),
|
||||
get_settings().log_tail_lines)
|
||||
except OSError:
|
||||
log_tail = ""
|
||||
if manifest is None and not manifest_error:
|
||||
manifest_error = "manifest 未产出"
|
||||
return {"ok": True, "data": {
|
||||
"job_id": job_id, "stage": stage,
|
||||
"manifest": manifest, "manifest_error": manifest_error,
|
||||
"log_tail": log_tail,
|
||||
"status": record._stage_states.get(stage, {}).get("status"),
|
||||
}}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# events SSE（FR-BIO-014 断线重连回放）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}/events")
|
||||
async def job_events(
|
||||
job_id: str,
|
||||
stage: str | None = Query(default=None),
|
||||
last_seq: int = Query(default=-1, ge=-1),
|
||||
) -> StreamingResponse:
|
||||
"""任务事件 SSE：快照首帧 + 历史回放（>last_seq）+ 实时订阅 + 心跳。
|
||||
|
||||
TC-BIO-API-081：任务终态（done/failed/archived）且回放完成后服务端正常关闭连接。
|
||||
"""
|
||||
record = await job_or_404(job_id)
|
||||
ctx: Context = context
|
||||
|
||||
async def gen():
|
||||
# 1. 快照首帧（TC-BIO-SSE-022）
|
||||
snap = ctx.events.snapshot(job_id)
|
||||
if snap is not None:
|
||||
yield f"event: snapshot\ndata: {json.dumps(snap, ensure_ascii=False)}\n\n".encode()
|
||||
# 2. 历史回放（>last_seq；TC-BIO-SSE-024）
|
||||
for ev in ctx.events.replay(job_id, after_seq=last_seq):
|
||||
if stage and ev.get("data", {}).get("stage") != stage:
|
||||
continue
|
||||
yield _sse_bytes(ev)
|
||||
# 3. 任务终态 → 回放后正常关闭（TC-BIO-API-081）
|
||||
if record.status in ("done", "failed", "archived"):
|
||||
return
|
||||
# 4. 实时订阅
|
||||
sub = await ctx.events.subscribe(job_filter=job_id, last_seq=last_seq)
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
ev = await asyncio.wait_for(sub.queue.get(), timeout=KEEPALIVE_INTERVAL)
|
||||
except asyncio.TimeoutError:
|
||||
yield KEEPALIVE_COMMENT
|
||||
continue
|
||||
if int(ev.get("seq") or 0) <= last_seq:
|
||||
continue
|
||||
if stage and ev.get("data", {}).get("stage") != stage:
|
||||
continue
|
||||
yield _sse_bytes(ev)
|
||||
finally:
|
||||
await ctx.events.unsubscribe(sub)
|
||||
|
||||
headers = {
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
return StreamingResponse(gen(), media_type="text/event-stream", headers=headers)
|
||||
|
||||
|
||||
def _sse_bytes(event: dict[str, Any]) -> bytes:
|
||||
"""SSE 帧编码（data 行只携带载荷，与 bioflow.py 契约一致）。"""
|
||||
data = json.dumps(event.get("data") or {}, ensure_ascii=False)
|
||||
return f"event: {event['type']}\ndata: {data}\n\n".encode("utf-8")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# report revise（FR-BIO-019，最小接口：绘图阶段重跑预留；纯文字走 docgen 由 A3 协调）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/report/{job_id}/revise")
|
||||
async def report_revise(job_id: str, body: ReportReviseRequest) -> dict[str, Any]:
|
||||
"""图表修订（FR-BIO-019）：解析图表目标 → 重跑绘图阶段（预留）。
|
||||
|
||||
P1 最小实现：无绘图目标 → 422（纯文字排版指令由前端走 docgen 编辑链路）。
|
||||
有目标但无对应阶段 → NOT_FOUND。绘图阶段重跑链路（rerun）复用现有端点。
|
||||
"""
|
||||
record = await job_or_404(job_id)
|
||||
instruction = (body.instruction or "").strip()
|
||||
target = (body.target or "").strip()
|
||||
if not target or not re.search(r"图|表|颜色|色", instruction + target):
|
||||
return _err("VALIDATION", "无明确图表修订目标（纯文字排版指令请走文档编辑）", 422)
|
||||
# 候选绘图阶段（基因家族链含绘图能力的阶段）
|
||||
plot_stages = [s for s in ("synteny", "karyotype", "motif", "combo", "gene_family_report")
|
||||
if s in record.stage_chain]
|
||||
rerun_stage = None
|
||||
for s in plot_stages:
|
||||
if s in target or target in s:
|
||||
rerun_stage = s
|
||||
break
|
||||
if rerun_stage is None:
|
||||
return _err("NOT_FOUND", "未找到可重跑的绘图阶段（P2 编排层细化）", 404)
|
||||
return {"ok": True, "data": {
|
||||
"job_id": job_id, "rerun_stage": rerun_stage,
|
||||
"target": target, "instruction": instruction,
|
||||
"params": body.params or {},
|
||||
}}
|
||||
@@ -0,0 +1,112 @@
|
||||
"""schemas.py —— 生信服务请求/响应 pydantic 模型（明文白名单文件，同 rag-new 规则）。
|
||||
|
||||
服务层（app/services/bioflow/）禁止 pydantic（Cython 编译纪律），
|
||||
全部 pydantic 模型只放本文件。路由层把模型 .model_dump() 成 dict 再交给服务层。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
# ---- chat ----
|
||||
|
||||
MAX_MESSAGE_CHARS = 100_000 # 超长消息拒绝（TC-BIO-API-015）
|
||||
|
||||
|
||||
class ChatStreamRequest(BaseModel):
|
||||
session_id: str = Field(min_length=1, max_length=128)
|
||||
message: str = Field(min_length=1, max_length=MAX_MESSAGE_CHARS)
|
||||
has_attachments: bool = False
|
||||
has_active_artifact: bool = False
|
||||
has_bioflow_context: bool = False
|
||||
last_seq: int | None = Field(default=None, ge=0)
|
||||
|
||||
|
||||
class ChatRequest(BaseModel):
|
||||
session_id: str = Field(min_length=1, max_length=128)
|
||||
message: str = Field(min_length=1, max_length=MAX_MESSAGE_CHARS)
|
||||
|
||||
|
||||
class FlowAskRequest(BaseModel):
|
||||
"""生信流内流程咨询（知识库问答）。"""
|
||||
|
||||
session_id: str = Field(min_length=1, max_length=128)
|
||||
question: str = Field(min_length=1, max_length=MAX_MESSAGE_CHARS)
|
||||
|
||||
|
||||
# ---- jobs ----
|
||||
|
||||
STAGE_STATUSES = (
|
||||
"waiting", "await_files", "waiting_confirm", "queued", "running",
|
||||
"done", "failed", "skipped", "paused",
|
||||
)
|
||||
StageStatus = Literal[
|
||||
"waiting", "await_files", "waiting_confirm", "queued", "running",
|
||||
"done", "failed", "skipped", "paused",
|
||||
]
|
||||
|
||||
JOB_STATUSES = (
|
||||
"created", "await_inputs", "running", "paused", "done", "failed", "archiving", "archived",
|
||||
)
|
||||
JobStatus = Literal[
|
||||
"created", "await_inputs", "running", "paused", "done", "failed", "archiving", "archived",
|
||||
]
|
||||
|
||||
|
||||
class JobCreateRequest(BaseModel):
|
||||
session_id: str = Field(min_length=1, max_length=128)
|
||||
title: str = Field(default="生信任务", max_length=500)
|
||||
stages: list[str] = Field(min_length=1)
|
||||
params: dict[str, dict[str, Any]] = Field(default_factory=dict)
|
||||
auto_continue: bool = False
|
||||
archive_id: str | None = None
|
||||
|
||||
@field_validator("stages")
|
||||
@classmethod
|
||||
def _stages_nonempty_names(cls, v: list[str]) -> list[str]:
|
||||
if any(not s or not s.strip() for s in v):
|
||||
raise ValueError("stages 含空阶段名")
|
||||
return [s.strip() for s in v]
|
||||
|
||||
|
||||
class ArchiveRunRequest(BaseModel):
|
||||
session_id: str = Field(min_length=1, max_length=128)
|
||||
name: str | None = Field(default=None, max_length=200)
|
||||
target: str | None = Field(default=None, max_length=2000)
|
||||
params: dict[str, dict[str, Any]] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class StageConfirmRequest(BaseModel):
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class StageRerunRequest(BaseModel):
|
||||
params: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class PauseResumeRequest(BaseModel):
|
||||
reason: str | None = Field(default=None, max_length=500)
|
||||
|
||||
|
||||
class PathRegisterRequest(BaseModel):
|
||||
server_path: str = Field(min_length=1, max_length=4096)
|
||||
|
||||
|
||||
# ---- archives ----
|
||||
|
||||
|
||||
class ArchiveCreateRequest(BaseModel):
|
||||
job_id: str = Field(min_length=1)
|
||||
name: str = Field(min_length=1, max_length=200)
|
||||
description: str = Field(default="", max_length=2000)
|
||||
include_intermediate: bool = False
|
||||
|
||||
|
||||
# ---- report revise ----
|
||||
|
||||
|
||||
class ReportReviseRequest(BaseModel):
|
||||
target: str | None = Field(default=None, max_length=500)
|
||||
instruction: str = Field(min_length=1, max_length=2000)
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
@@ -0,0 +1 @@
|
||||
"""app/services/__init__.py —— 生信服务 services 包。"""
|
||||
@@ -0,0 +1 @@
|
||||
"""app/services/bioflow/__init__.py —— 生信服务层包（dataclass/dict/stdlib 纪律）。"""
|
||||
@@ -0,0 +1,125 @@
|
||||
"""app/services/bioflow/archive.py —— 存档/复用（过程记录/中间文件/结果文件/报告打包落盘）。
|
||||
|
||||
FR-BIO-020/021/022/023：
|
||||
- 任务完成后可存档：过程记录（events.jsonl）、中间文件（workspace/<stage>/analysis，
|
||||
可选 include_intermediate）、结果文件（final/）、报告（报告链产物 HTML）打包到
|
||||
BIOFLOW_ARCHIVES_DIR/{archive_id}/。
|
||||
- bio_archives 行登记（名称/磁盘路径/大小/阶段链+参数/报告制品 id）。
|
||||
- 名称匹配复用：精确优先，模糊给候选（find_archive_by_name/find_archive_candidates
|
||||
在 store；本模块提供 archive_from_job 打包逻辑与名称规范化）。
|
||||
|
||||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
INTERMEDIATE_DIR_NAMES = ("analysis",)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArchiveResult:
|
||||
id: str
|
||||
name: str
|
||||
disk_path: str
|
||||
size_bytes: int
|
||||
report_artifact: str | None
|
||||
|
||||
|
||||
def archive_dir_for(archives_root: str | Path, archive_id: str) -> Path:
|
||||
return Path(archives_root) / archive_id
|
||||
|
||||
|
||||
async def archive_from_job(
|
||||
job: dict[str, Any],
|
||||
job_dir: str | Path,
|
||||
archives_root: str | Path,
|
||||
*,
|
||||
name: str,
|
||||
description: str = "",
|
||||
include_intermediate: bool = False,
|
||||
report_artifact: str | None = None,
|
||||
) -> ArchiveResult:
|
||||
"""任务目录 → 档案目录打包（纯 shutil，不依赖 docker/网络）。"""
|
||||
job_dir = Path(job_dir)
|
||||
archives_root = Path(archives_root)
|
||||
archive_id = uuid.uuid4().hex[:16]
|
||||
dest = archive_dir_for(archives_root, archive_id)
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 结果文件（workspace/final/）与过程记录（events.jsonl、logs/）
|
||||
final_dir = job_dir / "workspace" / "final"
|
||||
if final_dir.is_dir():
|
||||
_copytree_into(final_dir, dest / "final")
|
||||
logs_dir = job_dir / "logs"
|
||||
if logs_dir.is_dir():
|
||||
_copytree_into(logs_dir, dest / "logs")
|
||||
events_file = job_dir / "events.jsonl"
|
||||
if events_file.exists():
|
||||
shutil.copy2(events_file, dest / "events.jsonl")
|
||||
else:
|
||||
# 事件持久日志在 events_dir/{job_id}.jsonl（与任务目录分离）；归档一并打包
|
||||
job_id = job.get("id") or ""
|
||||
if job_id:
|
||||
ext_events = job_dir.parent / "events" / f"{job_id}.jsonl"
|
||||
if ext_events.exists():
|
||||
shutil.copy2(ext_events, dest / "events.jsonl")
|
||||
input_dir = job_dir / "input"
|
||||
if input_dir.is_dir():
|
||||
_copytree_into(input_dir, dest / "input")
|
||||
|
||||
# 中间文件（可选，FR-BIO-020 include_intermediate 开关）
|
||||
if include_intermediate:
|
||||
workspace = job_dir / "workspace"
|
||||
if workspace.is_dir():
|
||||
for stage_dir in workspace.iterdir():
|
||||
if not stage_dir.is_dir() or stage_dir.name in ("final", "logs"):
|
||||
continue
|
||||
analysis = stage_dir / "analysis"
|
||||
if analysis.is_dir():
|
||||
_copytree_into(analysis, dest / "intermediate" / stage_dir.name / "analysis")
|
||||
|
||||
# 报告：任务目录内报告 HTML（final 内 *.html / report.html）随 final 已复制；
|
||||
# report_artifact 为 doc_artifacts 登记 id（由 reports 模块/BFF 登记后回传）。
|
||||
size_bytes = _dir_size(dest)
|
||||
return ArchiveResult(
|
||||
id=archive_id,
|
||||
name=name,
|
||||
disk_path=str(dest),
|
||||
size_bytes=size_bytes,
|
||||
report_artifact=report_artifact,
|
||||
)
|
||||
|
||||
|
||||
def _copytree_into(src: Path, dest: Path) -> None:
|
||||
"""复制目录内容到 dest（dest 可能已存在，逐项合并）。"""
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for item in src.iterdir():
|
||||
target = dest / item.name
|
||||
if item.is_dir():
|
||||
if target.exists():
|
||||
_copytree_into(item, target)
|
||||
else:
|
||||
shutil.copytree(item, target)
|
||||
else:
|
||||
shutil.copy2(item, target)
|
||||
|
||||
|
||||
def _dir_size(path: Path) -> int:
|
||||
return sum(p.stat().st_size for p in path.rglob("*") if p.is_file())
|
||||
|
||||
|
||||
def remove_archive_dir(archives_root: str | Path, archive_id: str) -> None:
|
||||
shutil.rmtree(archive_dir_for(archives_root, archive_id), ignore_errors=True)
|
||||
|
||||
|
||||
def normalize_archive_name(name: str) -> str:
|
||||
"""归档名称规范化（trim；空→占位）。"""
|
||||
return (name or "").strip() or "未命名存档"
|
||||
@@ -0,0 +1,588 @@
|
||||
"""app/services/bioflow/catalog.py —— 阶段目录（stage_image_map.json + STAGE_INFO 移植）。
|
||||
|
||||
- 阶段列表/参数 schema/镜像映射的权威来源：A1 的 stage_image_map.json（不运行时反射）。
|
||||
- 中文名/预计耗时：从内核 gfa_core/mas/stage_catalog.py STAGE_INFO 移植的静态快照
|
||||
（同一仓库内文件，加载失败降级为空，不阻断）。
|
||||
- 前置依赖：从内核 STAGE_DEPENDENCIES 移植（供 planner 前置补全）。
|
||||
|
||||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---- 从内核 stage_catalog.py 移植的静态快照（2026-09-06，A1 交付树）----
|
||||
|
||||
# 阶段中文名 + 预计耗时（分钟）
|
||||
STAGE_INFO_SNAPSHOT: dict[str, dict[str, Any]] = {
|
||||
"wgs_qc": {"name": "WGS质控", "estimated_time": 30},
|
||||
"wgs_index": {"name": "WGS参考索引", "estimated_time": 15},
|
||||
"wgs_align": {"name": "WGS序列比对", "estimated_time": 120},
|
||||
"wgs_markdup": {"name": "WGS去重复", "estimated_time": 90},
|
||||
"wgs_bam_qc": {"name": "WGS BAM质控", "estimated_time": 30},
|
||||
"wgs_haplotype_calling": {"name": "WGS单样本变异检测", "estimated_time": 120},
|
||||
"wgs_gvcf_combine": {"name": "WGS GVCF合并", "estimated_time": 60},
|
||||
"wgs_joint_genotyping": {"name": "WGS联合分型", "estimated_time": 90},
|
||||
"wgs_variant_select": {"name": "WGS变异类型选择", "estimated_time": 30},
|
||||
"wgs_variant_filter": {"name": "WGS变异硬过滤", "estimated_time": 30},
|
||||
"wgs_impute": {"name": "WGS基因型填充", "estimated_time": 60},
|
||||
"wgs_maf_missing_filter": {"name": "WGS MAF与缺失率过滤", "estimated_time": 30},
|
||||
"pop_pca": {"name": "群体PCA分析", "estimated_time": 30},
|
||||
"pop_phylogeny": {"name": "群体系统树分析", "estimated_time": 45},
|
||||
"pop_structure": {"name": "群体结构分析", "estimated_time": 90},
|
||||
"pop_ld_decay": {"name": "群体LD衰减分析", "estimated_time": 60},
|
||||
"pop_snp_density": {"name": "SNP密度分析", "estimated_time": 20},
|
||||
"gwas_fmt": {"name": "GWAS VCF排序与HapMap转换", "estimated_time": 20},
|
||||
"gwas_kinship": {"name": "GWAS TASSEL亲缘关系矩阵", "estimated_time": 20},
|
||||
"gwas_trait": {"name": "GWAS表型数据准备", "estimated_time": 5},
|
||||
"gwas_cov": {"name": "GWAS群体结构协变量准备", "estimated_time": 5},
|
||||
"gwas_mlm": {"name": "GWAS TASSEL MLM/CMLM", "estimated_time": 60},
|
||||
"gwas_plot": {"name": "GWAS MLM结果绘图", "estimated_time": 10},
|
||||
"gwas_emmax": {"name": "GWAS EMMAX关联分析", "estimated_time": 60},
|
||||
"gwas_emplot": {"name": "GWAS EMMAX结果绘图", "estimated_time": 10},
|
||||
"gwas_glm": {"name": "GWAS TASSEL GLM", "estimated_time": 60},
|
||||
"identify": {"name": "基因家族鉴定", "estimated_time": 5},
|
||||
"identify_refine": {"name": "修订鉴定成员", "estimated_time": 5},
|
||||
"subfamily_confirm": {"name": "确认亚家族分类", "estimated_time": 3},
|
||||
"phylogeny": {"name": "系统发育分析", "estimated_time": 30},
|
||||
"reference_classification": {"name": "参考亚家族分类", "estimated_time": 5},
|
||||
"motif": {"name": "Motif分析", "estimated_time": 20},
|
||||
"domain": {"name": "结构域分析", "estimated_time": 15},
|
||||
"structure": {"name": "基因结构分析", "estimated_time": 10},
|
||||
"promoter": {"name": "启动子分析", "estimated_time": 20},
|
||||
"tfbs": {"name": "TF结合位点预测", "estimated_time": 30},
|
||||
"synteny": {"name": "种内共线性分析", "estimated_time": 30},
|
||||
"karyotype": {"name": "核型分析", "estimated_time": 10},
|
||||
"interspecies_synteny": {"name": "物种间共线性", "estimated_time": 40},
|
||||
"expression": {"name": "表达分析", "estimated_time": 15},
|
||||
"kaks": {"name": "Ka/Ks分析", "estimated_time": 20},
|
||||
"ppi": {"name": "PPI分析", "estimated_time": 10},
|
||||
"pep": {"name": "蛋白特征分析", "estimated_time": 10},
|
||||
"ortholog": {"name": "直系同源基因鉴定", "estimated_time": 15},
|
||||
"msa": {"name": "多序列比对", "estimated_time": 10},
|
||||
"enrichment": {"name": "富集分析", "estimated_time": 15},
|
||||
"combo": {"name": "组合图", "estimated_time": 10},
|
||||
"gene_family_report": {"name": "基因家族结果报告", "estimated_time": 2},
|
||||
"genome_prepare": {"name": "基因组准备", "estimated_time": 10},
|
||||
"ncbi_fetch_sequences": {"name": "NCBI序列下载", "estimated_time": 5},
|
||||
"sequence_extract_by_id": {"name": "按ID提取序列", "estimated_time": 5},
|
||||
"msa_region_plot": {"name": "MSA区间绘图", "estimated_time": 3},
|
||||
"synteny_plot_revise": {"name": "种内共线性Circos重绘", "estimated_time": 3},
|
||||
"promoter_plot_revise": {"name": "启动子自定义绘图", "estimated_time": 3},
|
||||
"prepare_organize": {"name": "数据文件整理", "estimated_time": 5},
|
||||
"prepare_normalize_gff": {"name": "GFF ID规范化", "estimated_time": 5},
|
||||
"prepare_longest_transcript": {"name": "提取最长转录本", "estimated_time": 10},
|
||||
"prepare_extract_pep": {"name": "从GFF提取PEP", "estimated_time": 10},
|
||||
"prepare_extract_cds": {"name": "从GFF提取CDS", "estimated_time": 10},
|
||||
"prepare_filter_fasta": {"name": "按ID提取FASTA", "estimated_time": 5},
|
||||
"prepare_simplify_ncbi": {"name": "NCBI ID简化", "estimated_time": 5},
|
||||
"rnaseq_qc": {"name": "RNA-seq质控", "estimated_time": 30},
|
||||
"rnaseq_align": {"name": "RNA-seq比对", "estimated_time": 60},
|
||||
"rnaseq_quant": {"name": "RNA-seq定量", "estimated_time": 30},
|
||||
"rnaseq_sample_qc": {"name": "RNA-seq样本质量评估", "estimated_time": 10},
|
||||
"rnaseq_de": {"name": "差异表达分析（自动生成FPKM表达图）", "estimated_time": 20},
|
||||
"rnaseq_wgcna": {"name": "RNA-seq WGCNA分析", "estimated_time": 30},
|
||||
"annotation": {"name": "背景注释", "estimated_time": 180},
|
||||
"rnaseq_go_enrich": {"name": "RNA-seq GO富集分析", "estimated_time": 15},
|
||||
"rnaseq_kegg_enrich": {"name": "RNA-seq KEGG富集分析", "estimated_time": 15},
|
||||
"rnaseq_enrich": {"name": "RNA-seq富集分析", "estimated_time": 15},
|
||||
"rnaseq_fgsea": {"name": "RNA-seq fgsea分析", "estimated_time": 15},
|
||||
"rnaseq_candidate_family": {"name": "RNA-seq候选家族筛选", "estimated_time": 20},
|
||||
"rnaseq_report": {"name": "RNA-seq结果报告", "estimated_time": 2},
|
||||
}
|
||||
|
||||
# 前置依赖（STAGE_DEPENDENCIES 快照；planner 前置补全依据）
|
||||
STAGE_DEPENDENCIES_SNAPSHOT: dict[str, list[str]] = {
|
||||
"wgs_qc": [], "wgs_index": [], "wgs_align": ["wgs_qc", "wgs_index"],
|
||||
"wgs_markdup": ["wgs_align"], "wgs_bam_qc": ["wgs_markdup", "wgs_index"],
|
||||
"wgs_haplotype_calling": ["wgs_markdup", "wgs_bam_qc", "wgs_index"],
|
||||
"wgs_gvcf_combine": ["wgs_haplotype_calling", "wgs_index"],
|
||||
"wgs_joint_genotyping": ["wgs_gvcf_combine", "wgs_index"],
|
||||
"wgs_variant_select": ["wgs_joint_genotyping"],
|
||||
"wgs_variant_filter": ["wgs_variant_select"], "wgs_impute": ["wgs_variant_filter"],
|
||||
"wgs_maf_missing_filter": ["wgs_impute"],
|
||||
"pop_pca": [], "pop_phylogeny": [], "pop_structure": [], "pop_ld_decay": [],
|
||||
"pop_snp_density": [], "gwas_fmt": [], "gwas_kinship": ["gwas_fmt"],
|
||||
"gwas_trait": [], "gwas_cov": [], "gwas_mlm": ["gwas_fmt", "gwas_kinship", "gwas_trait", "gwas_cov"],
|
||||
"gwas_plot": ["gwas_mlm"], "gwas_emmax": ["gwas_fmt", "gwas_trait"],
|
||||
"gwas_emplot": ["gwas_emmax"], "gwas_glm": ["gwas_fmt", "gwas_trait", "gwas_cov"],
|
||||
"identify": [], "identify_refine": [], "subfamily_confirm": ["reference_classification"],
|
||||
"genome_prepare": [], "ncbi_fetch_sequences": [], "sequence_extract_by_id": [],
|
||||
"msa_region_plot": [], "tree_plot_revise": [], "synteny_plot_revise": [],
|
||||
"promoter_plot_revise": [], "prepare_organize": [], "prepare_normalize_gff": [],
|
||||
"prepare_longest_transcript": [], "prepare_extract_pep": [], "prepare_extract_cds": [],
|
||||
"prepare_filter_fasta": [], "prepare_simplify_ncbi": [], "rnaseq_qc": [],
|
||||
"phylogeny": ["identify"],
|
||||
"reference_classification": ["identify", "phylogeny", "motif", "structure", "domain", "combo"],
|
||||
"synteny": ["identify"], "pep": ["identify"], "msa": ["identify"],
|
||||
"motif": ["identify", "phylogeny"], "domain": ["identify", "phylogeny"],
|
||||
"structure": ["identify", "phylogeny"], "promoter": ["identify", "phylogeny"],
|
||||
"tfbs": ["genome_prepare", "identify"], "expression": ["identify", "phylogeny"],
|
||||
"enrichment": ["identify", "phylogeny"], "ortholog": ["identify"],
|
||||
"interspecies_synteny": ["identify"], "karyotype": ["identify", "synteny"],
|
||||
"ppi": ["identify"], "kaks": ["identify"],
|
||||
"combo": ["identify", "phylogeny", "motif", "structure", "domain"],
|
||||
"rnaseq_align": ["rnaseq_qc"], "rnaseq_quant": ["rnaseq_align"],
|
||||
"rnaseq_sample_qc": ["rnaseq_quant"], "rnaseq_de": ["rnaseq_sample_qc"],
|
||||
"rnaseq_wgcna": ["rnaseq_sample_qc"], "annotation": [],
|
||||
"rnaseq_enrich": ["rnaseq_de"], "rnaseq_fgsea": ["rnaseq_de"],
|
||||
"rnaseq_go_enrich": ["rnaseq_de"], "rnaseq_kegg_enrich": ["rnaseq_de"],
|
||||
"rnaseq_candidate_family": ["rnaseq_de", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea"],
|
||||
"rnaseq_report": [],
|
||||
}
|
||||
|
||||
GENE_FAMILY_CHAIN = [
|
||||
"genome_prepare", "identify", "identify_refine", "phylogeny",
|
||||
"reference_classification", "subfamily_confirm", "motif", "domain", "structure",
|
||||
"combo", "promoter", "tfbs", "synteny", "interspecies_synteny", "karyotype",
|
||||
"pep", "ortholog", "kaks", "ppi", "gene_family_report", "expression",
|
||||
]
|
||||
|
||||
CHAIN_ALIASES = {"gene_family": GENE_FAMILY_CHAIN, "gf": GENE_FAMILY_CHAIN,
|
||||
"wgs": None, "rnaseq": None}
|
||||
|
||||
|
||||
@dataclass
|
||||
class StageDef:
|
||||
"""阶段目录条目（catalog 端点返回形态）。"""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
image: str
|
||||
module: str
|
||||
executable: bool
|
||||
estimated_time: int
|
||||
params: list[dict[str, Any]] = field(default_factory=list)
|
||||
inputs: list[dict[str, Any]] = field(default_factory=list)
|
||||
outputs: list[dict[str, Any]] = field(default_factory=list)
|
||||
note: str = ""
|
||||
dependencies: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"description": self.description,
|
||||
"image": self.image,
|
||||
"module": self.module,
|
||||
"executable": self.executable,
|
||||
"estimated_time": self.estimated_time,
|
||||
"params": self.params,
|
||||
"inputs": self.inputs,
|
||||
"outputs": self.outputs,
|
||||
"note": self.note,
|
||||
"dependencies": self.dependencies,
|
||||
}
|
||||
|
||||
|
||||
# ---- 基因家族链参数 schema（手写；参数 help 文案自入口 argparse 提取整理）----
|
||||
|
||||
_GENOME_PREPARE_PARAMS = [
|
||||
{"name": "species", "type": "string", "default": "", "source": "user",
|
||||
"help": "目标物种代码（如 Ft），基因组数据文件名中的物种标识"},
|
||||
{"name": "suffix", "type": "string", "default": "", "source": "user",
|
||||
"help": "文件后缀过滤（如 .fa.gz），留空自动匹配"},
|
||||
]
|
||||
_IDENTIFY_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀，例如 Ft.SPL（输出文件命名空间）"},
|
||||
{"name": "required_pfams", "type": "string", "default": "", "source": "contract",
|
||||
"help": "逗号分隔的 Pfam accession，如 PF03110（家族识别条件）"},
|
||||
{"name": "dom_i_evalue_max", "type": "float", "default": 1e-3, "source": "contract",
|
||||
"help": "domain i-Evalue threshold（结构域独立 E 值阈值）"},
|
||||
{"name": "hmm_cov_min", "type": "float", "default": 0.3, "source": "contract",
|
||||
"help": "HMM coverage threshold（HMM 覆盖度阈值）"},
|
||||
{"name": "blast_evalue", "type": "float", "default": 1e-7, "source": "contract",
|
||||
"help": "Blast E-value threshold（BLAST E 值阈值）"},
|
||||
{"name": "blast_identity_min", "type": "float", "default": 30.0, "source": "contract",
|
||||
"help": "minimum BLAST percent identity（参考蛋白命中最低一致度）"},
|
||||
{"name": "hmm_combine_mode", "type": "choice", "choices": ["intersection", "union"],
|
||||
"default": "union", "source": "contract",
|
||||
"help": "多 HMM 文件整合方式（交集/并集），默认并集"},
|
||||
{"name": "hmm_blast_merge_mode", "type": "choice", "choices": ["intersection", "union"],
|
||||
"default": "", "source": "contract", "help": "HMM 和 BLAST 结果整合方式"},
|
||||
{"name": "cut_ga", "type": "boolean", "default": False, "source": "contract",
|
||||
"help": "use Pfam GA thresholds（使用 Pfam GA 阈值）"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数（--cpus 注入）"},
|
||||
{"name": "memory", "type": "choice", "choices": ["16g", "32g", "64g"], "default": "32g",
|
||||
"source": "runtime", "help": "容器内存上限（--memory 注入）"},
|
||||
]
|
||||
_PHYLOGENY_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀（同 identify）"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
{"name": "memory", "type": "choice", "choices": ["16g", "32g", "64g"], "default": "32g",
|
||||
"source": "runtime", "help": "容器内存上限"},
|
||||
]
|
||||
_MOTIF_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
{"name": "memory", "type": "choice", "choices": ["16g", "32g", "64g"], "default": "32g",
|
||||
"source": "runtime", "help": "容器内存上限"},
|
||||
]
|
||||
_DOMAIN_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_STRUCTURE_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_COMBO_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_PROMOTER_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "upstream", "type": "integer", "default": 2000, "source": "contract",
|
||||
"help": "启动子上游区间长度（bp）"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_TFBS_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_SYNTENY_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_INTERSPECIES_SYNTENY_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "ref_species", "type": "string", "default": "", "source": "user",
|
||||
"help": "参考物种代码（跨物种共线性）"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_KARYOTYPE_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_PEP_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_ORTHOLOG_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "ref_species", "type": "string", "default": "", "source": "user",
|
||||
"help": "参考物种代码（直系同源鉴定）"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_KAKS_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_PPI_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "species_code", "type": "string", "default": "", "source": "user",
|
||||
"help": "物种代码（STRING 蛋白互作）"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_REFERENCE_CLASSIFICATION_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_SUBFAMILY_CONFIRM_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_EXPRESSION_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_REPORT_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
]
|
||||
_RNASEQ_QC_PARAMS = [
|
||||
{"name": "samples", "type": "string", "default": "", "source": "user",
|
||||
"help": "样本列表（逗号分隔）"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "8",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_RNASEQ_ALIGN_PARAMS = [
|
||||
{"name": "samples", "type": "string", "default": "", "source": "user",
|
||||
"help": "样本列表（逗号分隔）"},
|
||||
{"name": "ref_genome", "type": "string", "default": "", "source": "user",
|
||||
"help": "参考基因组路径/索引名"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "8",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_RNASEQ_QUANT_PARAMS = [
|
||||
{"name": "samples", "type": "string", "default": "", "source": "user",
|
||||
"help": "样本列表（逗号分隔）"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "6",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_RNASEQ_SAMPLE_QC_PARAMS = [
|
||||
{"name": "samples", "type": "string", "default": "", "source": "user",
|
||||
"help": "样本列表（逗号分隔）"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_RNASEQ_DE_PARAMS = [
|
||||
{"name": "design", "type": "string", "default": "", "source": "user",
|
||||
"help": "实验设计矩阵文件（对比组声明）"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_RNASEQ_WGCNA_PARAMS = [
|
||||
{"name": "design", "type": "string", "default": "", "source": "user",
|
||||
"help": "实验设计矩阵文件"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_RNASEQ_ENRICH_PARAMS = [
|
||||
{"name": "gene_list", "type": "string", "default": "", "source": "contract",
|
||||
"help": "差异基因列表（上游 DE 结果）"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_RNASEQ_CANDIDATE_FAMILY_PARAMS = [
|
||||
{"name": "family_pfams", "type": "string", "default": "", "source": "user",
|
||||
"help": "候选家族 Pfam accession"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||||
"source": "runtime", "help": "容器 CPU 数"},
|
||||
]
|
||||
_RNASEQ_REPORT_PARAMS = [
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||||
"help": "分析前缀"},
|
||||
]
|
||||
|
||||
# 通用输入/输出描述（契约文档摘要；契约文件为权威，这里给目录层最小描述）
|
||||
_IDENTIFY_INPUTS = [
|
||||
{"name": "protein_fasta", "type": "fasta", "required": True,
|
||||
"description": "物种全蛋白质 FASTA（input_dir 内查找）"},
|
||||
{"name": "hmm_file", "type": "hmm", "required": False,
|
||||
"description": "用户上传 HMM 优先；否则按 required_pfams 从 Pfam-A.hmm 获取（BBA_PFAM_DIR）"},
|
||||
]
|
||||
_IDENTIFY_OUTPUTS = [
|
||||
{"name": "final_members", "type": "table", "description": "final/{prefix}.identified_members.id"},
|
||||
{"name": "final_proteins", "type": "fasta", "description": "final/{prefix}.identified_members.pep.fa"},
|
||||
{"name": "identify_report", "type": "markdown", "description": "final/{prefix}.identify_report.md"},
|
||||
{"name": "manifest", "type": "manifest", "description": "final/{prefix}.identify.manifest.json"},
|
||||
]
|
||||
_GENOME_PREPARE_INPUTS = [
|
||||
{"name": "genome_fasta", "type": "fasta", "required": False,
|
||||
"description": "{species}.genome.fasta.gz"},
|
||||
{"name": "pep_fasta", "type": "fasta", "required": False,
|
||||
"description": "{species}.pep.fasta.gz"},
|
||||
{"name": "cds_fasta", "type": "fasta", "required": False,
|
||||
"description": "{species}.cds.fasta.gz"},
|
||||
{"name": "gff3", "type": "gff3", "required": False, "description": "{species}.gff3.gz"},
|
||||
]
|
||||
_GENOME_PREPARE_OUTPUTS = [
|
||||
{"name": "normalized_pep", "type": "fasta", "description": "{species}.pep.fasta"},
|
||||
{"name": "normalized_cds", "type": "fasta", "description": "{species}.cds.fasta"},
|
||||
{"name": "normalized_gff", "type": "gff3", "description": "{species}_final.gff3"},
|
||||
{"name": "manifest", "type": "manifest", "description": "{prefix}.genome_prepare.manifest.json"},
|
||||
]
|
||||
|
||||
# 下游阶段通用描述（目录层最小；契约文档权威）
|
||||
_DOWNSTREAM_INPUTS = [
|
||||
{"name": "final_members", "type": "table", "required": True,
|
||||
"description": "上游 identify 的 final_members.id（跨阶段输入）"},
|
||||
]
|
||||
_DOWNSTREAM_OUTPUTS = [
|
||||
{"name": "manifest", "type": "manifest", "description": "final/{prefix}.{stage}.manifest.json"},
|
||||
{"name": "report", "type": "markdown", "description": "final/{prefix}.{stage}_report.md"},
|
||||
]
|
||||
|
||||
# stage → (params, inputs, outputs) 手写表
|
||||
_STAGE_PARAM_SCHEMAS: dict[str, tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]] = {
|
||||
"genome_prepare": (_GENOME_PREPARE_PARAMS, _GENOME_PREPARE_INPUTS, _GENOME_PREPARE_OUTPUTS),
|
||||
"identify": (_IDENTIFY_PARAMS, _IDENTIFY_INPUTS, _IDENTIFY_OUTPUTS),
|
||||
"identify_refine": ([
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", "source": "runtime", "help": "容器 CPU 数"},
|
||||
], _IDENTIFY_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"phylogeny": (_PHYLOGENY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"reference_classification": (_REFERENCE_CLASSIFICATION_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"subfamily_confirm": (_SUBFAMILY_CONFIRM_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"motif": (_MOTIF_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"domain": (_DOMAIN_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"structure": (_STRUCTURE_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"combo": (_COMBO_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"promoter": (_PROMOTER_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"tfbs": (_TFBS_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"synteny": (_SYNTENY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"interspecies_synteny": (_INTERSPECIES_SYNTENY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"karyotype": (_KARYOTYPE_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"pep": (_PEP_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"ortholog": (_ORTHOLOG_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"kaks": (_KAKS_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"ppi": (_PPI_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"expression": (_EXPRESSION_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"gene_family_report": (_REPORT_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"msa": ([
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", "source": "runtime", "help": "容器 CPU 数"},
|
||||
], _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"enrichment": ([
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"},
|
||||
], _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_qc": (_RNASEQ_QC_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_align": (_RNASEQ_ALIGN_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_quant": (_RNASEQ_QUANT_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_sample_qc": (_RNASEQ_SAMPLE_QC_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_de": (_RNASEQ_DE_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_wgcna": (_RNASEQ_WGCNA_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_enrich": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_go_enrich": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_kegg_enrich": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_fgsea": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_candidate_family": (_RNASEQ_CANDIDATE_FAMILY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"rnaseq_report": (_RNASEQ_REPORT_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
"annotation": ([
|
||||
{"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"},
|
||||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "8", "source": "runtime", "help": "容器 CPU 数"},
|
||||
], _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||||
}
|
||||
|
||||
|
||||
class StageCatalog:
|
||||
"""阶段目录（数据来自 stage_image_map.json + 手写快照，不运行时反射）。"""
|
||||
|
||||
def __init__(self, image_map: Any) -> None:
|
||||
self._map = image_map # ImageMap 实例（kernel 模块）
|
||||
|
||||
# ---- 查询 ----
|
||||
|
||||
def stage_names(self) -> list[str]:
|
||||
return self._map.all_stage_names()
|
||||
|
||||
def family_chain(self) -> list[str]:
|
||||
return list(self._map.family_chain) or GENE_FAMILY_CHAIN
|
||||
|
||||
def has(self, stage: str) -> bool:
|
||||
return stage in self._map.stages
|
||||
|
||||
def validate_chain(self, stages: list[str]) -> list[str]:
|
||||
"""链硬校验：返回未知阶段名列表（空=全合法）。"""
|
||||
return [s for s in stages if not self.has(s)]
|
||||
|
||||
def get(self, stage: str) -> StageDef | None:
|
||||
"""返回阶段目录条目（未知阶段 None）。"""
|
||||
if not self.has(stage):
|
||||
return None
|
||||
entry = self._map.image_for(stage) or {}
|
||||
params, inputs, outputs = _STAGE_PARAM_SCHEMAS.get(
|
||||
stage, ([], _DOWNSTREAM_INPUTS if stage not in ("identify", "genome_prepare") else [], [])
|
||||
)
|
||||
if stage not in _STAGE_PARAM_SCHEMAS:
|
||||
params, inputs, outputs = (
|
||||
[{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"],
|
||||
"default": "4", "source": "runtime", "help": "容器 CPU 数"}],
|
||||
_DOWNSTREAM_INPUTS if stage not in ("identify", "genome_prepare") else [],
|
||||
_DOWNSTREAM_OUTPUTS,
|
||||
)
|
||||
info = STAGE_INFO_SNAPSHOT.get(stage, {})
|
||||
return StageDef(
|
||||
name=stage,
|
||||
description=str(info.get("name") or stage),
|
||||
image=str(entry.get("image") or ""),
|
||||
module=str(entry.get("module") or ""),
|
||||
executable=bool(entry.get("executable")),
|
||||
estimated_time=int(info.get("estimated_time") or 0),
|
||||
params=params,
|
||||
inputs=inputs,
|
||||
outputs=outputs,
|
||||
note=str(entry.get("note") or ""),
|
||||
dependencies=STAGE_DEPENDENCIES_SNAPSHOT.get(stage, []),
|
||||
)
|
||||
|
||||
def resolve_chain(self, chain: str | None) -> list[str] | None:
|
||||
"""chain 别名解析（gene_family 链 21 阶段）；None 表示无该链。"""
|
||||
if not chain:
|
||||
return None
|
||||
alias = CHAIN_ALIASES.get(chain)
|
||||
if alias is None:
|
||||
return None
|
||||
return list(alias)
|
||||
|
||||
def auto_fill_prerequisites(self, stages: list[str]) -> tuple[list[str], list[str]]:
|
||||
"""前置补全（FR-BIO-003）：缺失的前置阶段自动补全。
|
||||
|
||||
返回 (补全后的链, 被补全的阶段名列表)。
|
||||
链内阶段按依赖拓扑顺序排列（依赖在前）。
|
||||
"""
|
||||
missing: list[str] = []
|
||||
added: list[str] = []
|
||||
chain_set = set(stages)
|
||||
# 反复补全直到闭合（依赖的依赖也补）
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for s in list(stages):
|
||||
for dep in STAGE_DEPENDENCIES_SNAPSHOT.get(s, []):
|
||||
if dep not in chain_set and self.has(dep):
|
||||
chain_set.add(dep)
|
||||
missing.append(dep)
|
||||
added.append(dep)
|
||||
changed = True
|
||||
ordered = topo_order([s for s in GENE_FAMILY_CHAIN if s in chain_set]
|
||||
or list(chain_set))
|
||||
return ordered, added
|
||||
|
||||
def dependencies(self, stage: str) -> list[str]:
|
||||
return STAGE_DEPENDENCIES_SNAPSHOT.get(stage, [])
|
||||
|
||||
|
||||
def topo_order(stages: list[str]) -> list[str]:
|
||||
"""按依赖拓扑排序（依赖在前，保持输入相对顺序稳定）。"""
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def visit(s: str) -> None:
|
||||
if s in seen:
|
||||
return
|
||||
seen.add(s)
|
||||
for dep in STAGE_DEPENDENCIES_SNAPSHOT.get(s, []):
|
||||
visit(dep)
|
||||
result.append(s)
|
||||
|
||||
for s in stages:
|
||||
visit(s)
|
||||
return result
|
||||
|
||||
|
||||
def build_catalog(image_map: Any) -> StageCatalog:
|
||||
return StageCatalog(image_map)
|
||||
@@ -0,0 +1,202 @@
|
||||
"""app/services/bioflow/events.py —— 进程内事件总线（pub/sub 供 SSE）+ 持久事件日志。
|
||||
|
||||
- 发布：runner/路由在任务状态变化时 publish(type, data)（事件类型清单 = 实施计划 §六
|
||||
12 类 + 内部 job_recovered）。
|
||||
- 订阅：async queue（供 SSE 端点）；内存环形缓冲（断线重连回放）。
|
||||
- 持久：events/{job_id}.jsonl 追加落盘（服务重启恢复回放；FR-BIO-031）。
|
||||
- 快照：snapshot(job_id) 构造断线重连首帧（TC-BIO-SSE-022/024）。
|
||||
|
||||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
EVENT_TYPES = (
|
||||
"job_created", "plan_ready", "stage_started", "stage_log", "stage_done",
|
||||
"stage_failed", "stage_skipped", "stage_await_confirm", "stage_await_files",
|
||||
"job_done", "job_paused", "job_archived", "job_recovered",
|
||||
)
|
||||
|
||||
# 断线重连时需要回放的关键状态事件（含历史事件全量）
|
||||
REPLAYABLE_TYPES = set(EVENT_TYPES)
|
||||
|
||||
MEMORY_BUFFER_LIMIT = 500
|
||||
SNAPSHOT_TYPE = "snapshot"
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class Subscriber:
|
||||
"""订阅者（eq=False：身份哈希，可入 set）。"""
|
||||
|
||||
queue: asyncio.Queue = field(default_factory=asyncio.Queue)
|
||||
last_seq: int = -1
|
||||
job_filter: str | None = None
|
||||
|
||||
|
||||
class EventBus:
|
||||
"""进程内事件总线：publish → 订阅队列广播 + 内存环形缓冲 + jsonl 持久日志。"""
|
||||
|
||||
def __init__(self, events_dir: str | Path) -> None:
|
||||
self.events_dir = Path(events_dir)
|
||||
self.events_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._subscribers: set[Subscriber] = set()
|
||||
self._sub_lock = asyncio.Lock()
|
||||
self._buffer: deque[dict[str, Any]] = deque(maxlen=MEMORY_BUFFER_LIMIT)
|
||||
self._seq = 0
|
||||
self._file_lock = threading.Lock()
|
||||
|
||||
# ---- 内部 ----
|
||||
|
||||
def _events_file(self, job_id: str) -> Path:
|
||||
return self.events_dir / f"{job_id}.jsonl"
|
||||
|
||||
def _append_file(self, event: dict[str, Any]) -> None:
|
||||
try:
|
||||
job_id = str((event.get("data") or {}).get("job_id") or "")
|
||||
with self._file_lock:
|
||||
with self._events_file(job_id).open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(event, ensure_ascii=False) + "\n")
|
||||
except OSError:
|
||||
logger.warning("bioflow events: 持久化失败 %s", event.get("type"))
|
||||
|
||||
# ---- 发布 ----
|
||||
|
||||
def publish(self, event_type: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""发布事件（同步）：返回完整事件（含 seq），并唤醒所有订阅者。"""
|
||||
if event_type not in EVENT_TYPES:
|
||||
raise ValueError(f"未知事件类型: {event_type}")
|
||||
self._seq += 1
|
||||
event: dict[str, Any] = {
|
||||
"type": event_type,
|
||||
"seq": self._seq,
|
||||
"data": dict(data),
|
||||
}
|
||||
self._buffer.append(event)
|
||||
self._append_file(event)
|
||||
# 广播给订阅者（跨线程安全：queue.put_nowait 本身线程安全）
|
||||
for sub in list(self._subscribers):
|
||||
if sub.job_filter is not None and sub.job_filter != str(data.get("job_id") or ""):
|
||||
continue
|
||||
try:
|
||||
sub.queue.put_nowait(event)
|
||||
except asyncio.QueueFull: # pragma: no cover - 防御
|
||||
logger.warning("bioflow events: 订阅队列满，丢弃事件 %s", event_type)
|
||||
return event
|
||||
|
||||
def publish_async(self, event_type: str, data: dict[str, Any]) -> None:
|
||||
"""异步发布（供路由协程调用，publish 本身为同步安全实现）。"""
|
||||
self.publish(event_type, data)
|
||||
|
||||
# ---- 订阅（SSE 端点）----
|
||||
|
||||
async def subscribe(self, job_filter: str | None = None, last_seq: int = -1) -> Subscriber:
|
||||
sub = Subscriber(last_seq=last_seq, job_filter=job_filter)
|
||||
async with self._sub_lock:
|
||||
self._subscribers.add(sub)
|
||||
return sub
|
||||
|
||||
async def unsubscribe(self, sub: Subscriber) -> None:
|
||||
async with self._sub_lock:
|
||||
self._subscribers.discard(sub)
|
||||
|
||||
# ---- 断线重连回放 ----
|
||||
|
||||
def snapshot(self, job_id: str) -> dict[str, Any] | None:
|
||||
"""构造任务快照（首帧 data.type=snapshot；TC-BIO-SSE-022）。
|
||||
|
||||
从持久日志读该任务全部事件，拼出 job + stages 当前状态。
|
||||
事件不存在（未知任务）返回 None。
|
||||
"""
|
||||
path = self._events_file(job_id)
|
||||
if not path.exists():
|
||||
return None
|
||||
events: list[dict[str, Any]] = []
|
||||
try:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
if ev.get("type") in REPLAYABLE_TYPES:
|
||||
events.append(ev)
|
||||
except ValueError:
|
||||
continue
|
||||
except OSError:
|
||||
return None
|
||||
if not events:
|
||||
return None
|
||||
# 由事件序列推导 job/stages 状态
|
||||
job_status = "created"
|
||||
stage_states: dict[str, dict[str, Any]] = {}
|
||||
for ev in events:
|
||||
t, d = ev["type"], ev.get("data") or {}
|
||||
if t == "job_created":
|
||||
job_status = "created"
|
||||
for s in d.get("stages") or []:
|
||||
stage_states.setdefault(str(s), {"stage": str(s), "status": "waiting"})
|
||||
elif t == "stage_started":
|
||||
st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))})
|
||||
st["status"] = "running"
|
||||
elif t == "stage_done":
|
||||
st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))})
|
||||
st["status"] = "done"
|
||||
elif t == "stage_failed":
|
||||
st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))})
|
||||
st["status"] = "failed"
|
||||
elif t == "stage_skipped":
|
||||
st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))})
|
||||
st["status"] = "skipped"
|
||||
elif t == "stage_await_confirm":
|
||||
st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))})
|
||||
st["status"] = "waiting_confirm"
|
||||
elif t == "stage_await_files":
|
||||
st = stage_states.setdefault(str(d.get("stage")), {"stage": str(d.get("stage"))})
|
||||
st["status"] = "await_files"
|
||||
elif t == "job_done":
|
||||
job_status = "done"
|
||||
elif t == "job_paused":
|
||||
job_status = "paused"
|
||||
elif t == "job_archived":
|
||||
job_status = "archived"
|
||||
return {
|
||||
"type": SNAPSHOT_TYPE,
|
||||
"job_id": job_id,
|
||||
"job": {"status": job_status},
|
||||
"stages": [stage_states[k] for k in stage_states],
|
||||
"last_seq": events[-1]["seq"] if events else 0,
|
||||
}
|
||||
|
||||
def replay(self, job_id: str, after_seq: int = -1) -> list[dict[str, Any]]:
|
||||
"""回放任务历史事件（seq > after_seq；断线重连补齐；TC-BIO-SSE-023/024）。"""
|
||||
path = self._events_file(job_id)
|
||||
if not path.exists():
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
try:
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
if ev.get("type") in REPLAYABLE_TYPES and int(ev.get("seq") or 0) > after_seq:
|
||||
out.append(ev)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
except OSError:
|
||||
return []
|
||||
return out
|
||||
|
||||
def memory_replay(self, after_seq: int = -1) -> list[dict[str, Any]]:
|
||||
"""内存环形缓冲回放（进程重启后内存为空，此时走持久回放）。"""
|
||||
return [ev for ev in self._buffer if int(ev.get("seq") or 0) > after_seq]
|
||||
@@ -0,0 +1,218 @@
|
||||
"""app/services/bioflow/intent.py —— 生信意图分流（词典唯一事实源 docs/team/cases_bioflow_intent.md v1.0）。
|
||||
|
||||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。
|
||||
route(text, has_attachments, has_active_artifact, has_bioflow_context)
|
||||
-> "qa" | "docgen" | "edit" | "export" | "bioflow" | "bioflow_qa"
|
||||
判定顺序与词典一致（优先级从上到下，命中即返回；漏判安全）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
# ---- 词典（与 cases_bioflow_intent.md §3 逐条一致，小写不敏感比较）----
|
||||
|
||||
EXPORT_WORD = ["导出", "转成", "转为", "转换成", "下载成", "输出为", "存为", "转pdf", "转word"]
|
||||
FORMAT_WORD = ["pdf", "docx", "doc", "word"]
|
||||
|
||||
EDIT_WORD = [
|
||||
"改", "修改", "调整", "删", "删除", "加一段", "补充", "润色",
|
||||
"缩写", "扩写", "重写", "改写", "优化", "加个", "增加一段",
|
||||
]
|
||||
|
||||
CHART_WORD = [
|
||||
"图表", "图颜色", "颜色换成", "图改成", "柱状图", "热图", "火山图", "曼哈顿图",
|
||||
"韦恩图", "气泡图", "散点图", "折线图", "改成红色", "换成蓝色", "颜色",
|
||||
]
|
||||
|
||||
# 图表对象词（FR-BIO-019 触发：活动文档 + 图表对象词 → 生信图表调整）
|
||||
CHART_OBJECT_WORD = [
|
||||
"图表", "图颜色", "颜色换成", "图改成", "柱状图", "热图", "火山图", "曼哈顿图",
|
||||
"韦恩图", "气泡图", "散点图", "折线图",
|
||||
]
|
||||
|
||||
# 家族名语境词（词典用例表：家族名后须跟语境词或位于句尾才判定为家族命中，
|
||||
# 如「SPL 会议」不命中、「玉米 bZIP」句尾命中）
|
||||
FAMILY_CONTEXT = ("家族", "因子", "基因家族", "转录因子", "family", "成员", "基因")
|
||||
|
||||
# docgen 写文档意图词典（沿用 docgen intent.py 全表，cases_intent.md 冻结）
|
||||
WRITE_VERB = ["写", "撰写", "拟", "起草", "编制", "生成", "编写", "拟定", "草拟", "制定", "整理"]
|
||||
|
||||
DOC_TYPE = [
|
||||
"论文", "报告", "通知", "通告", "公告", "请示", "批复", "函", "申请书",
|
||||
"申请", "申报书", "标书", "合同", "协议", "方案", "计划", "总结", "汇报",
|
||||
"讲话稿", "致辞", "证明", "纪要", "简报", "材料", "说明书", "指南", "文档",
|
||||
"申报材料", "工作总结", "调研报告", "可行性报告", "项目书", "策划书",
|
||||
"演讲稿", "新闻稿", "文案", "简历", "制度", "规定", "办法", "细则", "章程",
|
||||
"声明", "邀请函", "感谢信", "慰问信", "介绍信", "推荐信", "倡议书",
|
||||
"述职报告", "开题报告", "结题报告", "实验报告", "分析报告", "会议纪要",
|
||||
]
|
||||
|
||||
DOC_TYPE_EXCLUDE = ["介绍", "内容", "代码", "诗", "作业", "问题", "建议", "想法"]
|
||||
|
||||
# 生信词典补充「文档类型」表达（基因家族分析报告等，与 DOC_TYPE 并集）
|
||||
_BIO_DOC_TYPE_EXTRA = ["分析报告", "研究论文"]
|
||||
|
||||
FLOW_QA_WORD = [
|
||||
"需要什么输入", "有什么输入", "输入是什么", "需要什么参数", "参数是什么",
|
||||
"什么参数", "怎么设置", "如何设置", "什么软件", "用什么软件", "前置条件",
|
||||
"介绍", "是什么", "怎么跑", "怎么做", "需要准备什么输入", "需要准备什么参数",
|
||||
]
|
||||
|
||||
ANALYZE_VERB = [
|
||||
"跑", "跑一下", "分析", "鉴定", "识别", "研究", "做个", "做一个", "做一下", "做",
|
||||
"执行", "运行", "run", "identify",
|
||||
]
|
||||
|
||||
# 分析对象词（按词典长词优先；「做」动词 + 对象组合）
|
||||
ANALYSIS_OBJECT = [
|
||||
"rna-seq", "rnaseq", "转录组", "差异表达", "wgs", "全基因组测序", "全基因组",
|
||||
"变异检测", "gwas", "全基因组关联", "关联分析", "群体遗传", "遗传多样性",
|
||||
"进化", "系统发育", "选择性清除", "群体结构", "群体", "拷贝数变异", "基因预测",
|
||||
"基因注释", "结构注释", "功能注释", "宏基因组",
|
||||
]
|
||||
|
||||
# 英文家族名 → 中文语境等价词（英文句「run rice SPL family identification」
|
||||
# 需 family/identification 等语境）
|
||||
_FAMILY_EN_CONTEXT = ("family", "gene", "factor", "identification", "analysis", "member")
|
||||
|
||||
FAMILY_NAME = [
|
||||
"spl", "sbp", "mads", "mads-box", "madsbox", "wrky", "nac", "myb", "bzip", "bhlh",
|
||||
"ap2", "erf", "gras", "tcp", "hd-zip", "hdzip", "arf", "aux", "iaa", "dof", "gata",
|
||||
"hsf", "lbd", "lob", "trihelix", "e2f", "cpp", "nf-y", "nfy", "c2h2", "zf-hd",
|
||||
"zfhd", "far1", "frs", "hb", "wox", "pin", "mate", "abc", "pp2c", "snrk", "cdpk",
|
||||
"mapk", "rlck", "nbs", "lrr", "nlr", "tps", "cyp", "gst", "ugt", "转录因子家族",
|
||||
"基因家族",
|
||||
]
|
||||
|
||||
ANALYSIS_OBJECT = [
|
||||
"rna-seq", "rnaseq", "rna-seq", "转录组", "差异表达", "wgs", "全基因组", "变异检测",
|
||||
"全基因组测序", "gwas", "关联分析", "全基因组关联", "群体", "群体遗传", "遗传多样性",
|
||||
"进化", "系统发育", "选择性清除", "群体结构", "拷贝数变异", "基因预测", "基因注释",
|
||||
"结构注释", "功能注释", "宏基因组",
|
||||
]
|
||||
|
||||
REUSE_PHRASE = ["存档任务的方法", "存档的方法"]
|
||||
|
||||
|
||||
def _contains_any_longest_first(text_lower: str, words: list[str]) -> bool:
|
||||
"""词表按长度降序匹配（长词优先，避免「改」命中「改成红色」类图表词）。"""
|
||||
for w in sorted(words, key=len, reverse=True):
|
||||
if w in text_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _contains_any(text_lower: str, words: list[str]) -> bool:
|
||||
return _contains_any_longest_first(text_lower, words)
|
||||
|
||||
|
||||
def _has_doc_type(text_lower: str) -> bool:
|
||||
"""含 DOC_TYPE 任一，且不含 DOC_TYPE_EXCLUDE 任一（排除优先，同 docgen）。"""
|
||||
if _contains_any(text_lower, DOC_TYPE_EXCLUDE):
|
||||
return False
|
||||
return _contains_any(text_lower, DOC_TYPE) or _contains_any(text_lower, _BIO_DOC_TYPE_EXTRA)
|
||||
|
||||
|
||||
def extract_reuse_name(text: str) -> str | None:
|
||||
"""提取存档复用句式的「方法名」（REUSE_PHRASE 之前最近的 3-20 字符片段）。
|
||||
|
||||
FR-BIO-023：「用 XXX 存档任务的方法帮我跑 xxxx」→ 方法名 = XXX。
|
||||
返回 None 表示未命中复用句式。
|
||||
"""
|
||||
text_lower = text.lower()
|
||||
pos = -1
|
||||
phrase = ""
|
||||
for p in REUSE_PHRASE:
|
||||
idx = text_lower.rfind(p)
|
||||
if idx > pos:
|
||||
pos, phrase = idx, p
|
||||
if pos < 0:
|
||||
return None
|
||||
before = text[:pos].strip()
|
||||
# 取短语前最近的 3-20 字符片段（剔除「用/根据/参考/按」等引导词）
|
||||
for lead in ("根据", "参考", "用", "按", "按照", "《", "》", "的"):
|
||||
before = before.replace(lead, " ")
|
||||
parts = [s for s in before.replace("《", " ").replace("》", " ").split() if s.strip()]
|
||||
if not parts:
|
||||
return None
|
||||
name = parts[-1]
|
||||
if 3 <= len(name) <= 20:
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def route(
|
||||
text: str,
|
||||
has_attachments: bool = False,
|
||||
has_active_artifact: bool = False,
|
||||
has_bioflow_context: bool = False,
|
||||
) -> str:
|
||||
"""生信意图分流主入口（判定顺序同词典 §2，命中即返回；漏判安全）。"""
|
||||
t = text.strip()
|
||||
if not t:
|
||||
return "qa"
|
||||
tl = t[:2000].lower() # 边界约定：>2000 字符按截断前 2000 判定
|
||||
|
||||
# 1. export（需 has_active_artifact）
|
||||
if has_active_artifact and _contains_any(tl, EXPORT_WORD) and _contains_any(tl, FORMAT_WORD):
|
||||
return "export"
|
||||
|
||||
# 2. edit（需 has_active_artifact；含图表对象词跳过 → 生信 FR-BIO-019）
|
||||
if (
|
||||
has_active_artifact
|
||||
and _contains_any(tl, EDIT_WORD)
|
||||
and not _contains_any(tl, CHART_OBJECT_WORD)
|
||||
):
|
||||
return "edit"
|
||||
|
||||
# 2b. 图表调整走生信（FR-BIO-019）：活动文档 + 图表对象词 → bioflow
|
||||
if has_active_artifact and _contains_any(tl, CHART_OBJECT_WORD):
|
||||
return "bioflow"
|
||||
|
||||
# 3. docgen（写文档意图；DOC_TYPE_EXCLUDE 排除优先）
|
||||
if _contains_any(tl, WRITE_VERB) and _has_doc_type(tl):
|
||||
return "docgen"
|
||||
|
||||
# 4. docgen（附件信号）
|
||||
if has_attachments and _contains_any(tl, WRITE_VERB):
|
||||
return "docgen"
|
||||
|
||||
# 5. bioflow_qa（生信流内流程咨询；FLOW_QA 命中即拦，不建任务）
|
||||
if has_bioflow_context and _contains_any(tl, FLOW_QA_WORD):
|
||||
return "bioflow_qa"
|
||||
|
||||
# 5b. 流程咨询句式（无上下文）：词典用例表 #18「跑 SPL 家族鉴定需要准备什么输入？」
|
||||
# 期望 qa——FLOW_QA 词命中即拦（漏判安全），不进 bioflow
|
||||
if _contains_any(tl, FLOW_QA_WORD):
|
||||
return "qa"
|
||||
|
||||
has_analyze = _contains_any(tl, ANALYZE_VERB)
|
||||
has_object = _family_hit(tl) or _contains_any(tl, ANALYSIS_OBJECT)
|
||||
|
||||
# 6. bioflow（A 分析动词 且 B 家族名/分析对象；C 复用句式同判）
|
||||
if has_analyze and has_object:
|
||||
return "bioflow"
|
||||
|
||||
# 7. 漏判安全：不命中绝不进生信流
|
||||
return "qa"
|
||||
|
||||
|
||||
def _family_hit(text_lower: str) -> bool:
|
||||
"""家族名命中：词典词 + 语境词（中文）或英文语境（family/gene 等）。
|
||||
|
||||
家族名后须跟语境词或位于句尾（词典用例表：SPL 会议不命中、玉米 bZIP 句尾命中）。
|
||||
"""
|
||||
for name in sorted(FAMILY_NAME, key=len, reverse=True):
|
||||
idx = text_lower.find(name)
|
||||
if idx < 0:
|
||||
continue
|
||||
tail = text_lower[idx + len(name):]
|
||||
tail_stripped = tail.lstrip()
|
||||
# 中文语境：家族名后紧跟语境词
|
||||
if tail_stripped.startswith(FAMILY_CONTEXT):
|
||||
return True
|
||||
# 英文语境（family/identification 等在句内任意位置即算）
|
||||
if _contains_any(text_lower, _FAMILY_EN_CONTEXT) and "family" in text_lower:
|
||||
return True
|
||||
# 句尾（家族名后仅空白/标点）
|
||||
if not tail_stripped.strip("，。！？；：,.!?;:（）() "):
|
||||
return True
|
||||
return False
|
||||
@@ -0,0 +1,288 @@
|
||||
"""app/services/bioflow/kernel.py —— Docker 执行器（阶段→镜像映射表驱动）。
|
||||
|
||||
职责：
|
||||
- 读 A1 交付的 stage_image_map.json（路径来自配置 BIOFLOW_IMAGE_MAP_PATH，
|
||||
config 按 __file__ 推导 repo 根；零硬编码）。
|
||||
- 按阶段→镜像查表拼装 docker run 命令（list 形式，无 shell；命令拼装可 mock 断言）。
|
||||
- 容器内统一 `python -B -m cli.run_{stage}`（模块名来自映射表 module 字段）。
|
||||
- --user 宿主 uid（BIOFLOW_HOST_UID 环境变量，默认服务进程 os.getuid()）。
|
||||
- -v 任务目录 + BIOFLOW_DATA 只读挂载 + -e BBA_* + --cpus/--memory + --rm。
|
||||
- 超时 kill；日志全量落任务 logs/，返回尾部截断文本。
|
||||
|
||||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。执行经 async subprocess
|
||||
（真跑 docker 用默认实现；测试注入 fake 执行器 mock docker）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CONTAINER_CMD_PREFIX = ("python", "-B") # 容器内命令形态（TC-BIO-RUN-037）
|
||||
|
||||
|
||||
class ImageMapError(Exception):
|
||||
"""镜像映射表缺失/非法（服务可用但所有 docker 执行前置拦截）。"""
|
||||
|
||||
|
||||
class UnknownStageError(Exception):
|
||||
"""阶段不在映射表（catalog 校验后仍未知 → 执行前置拦截）。"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageMap:
|
||||
"""stage_image_map.json 内存形态。"""
|
||||
|
||||
images: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
stages: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
family_chain: list[str] = field(default_factory=list)
|
||||
|
||||
def image_for(self, stage: str) -> dict[str, Any] | None:
|
||||
return self.stages.get(stage)
|
||||
|
||||
def docker_tag(self, image: str) -> str:
|
||||
return str(self.images.get(image, {}).get("docker_tag") or image)
|
||||
|
||||
def all_stage_names(self) -> list[str]:
|
||||
return sorted(self.stages.keys())
|
||||
|
||||
|
||||
@dataclass
|
||||
class KernelConfig:
|
||||
"""docker 执行配置（由 app.config 注入；测试可构造最小实例）。"""
|
||||
|
||||
host_uid: int = 0
|
||||
docker_tag_suffix: str = "dev"
|
||||
data_root: str = ""
|
||||
container_data_dir: str = "/bioflow-data"
|
||||
container_input_dir: str = "/job/input"
|
||||
container_workspace_dir: str = "/job/workspace"
|
||||
default_cpus: str = "4"
|
||||
default_memory: str = "32g"
|
||||
stage_timeout_sec: float = 36000.0
|
||||
log_tail_lines: int = 80
|
||||
bba_env: dict[str, str] = field(default_factory=dict)
|
||||
cpu_allow: str = "4,8,16,32"
|
||||
memory_allow: str = "16g,32g,64g"
|
||||
|
||||
|
||||
class Kernel:
|
||||
"""Docker 执行内核：映射表加载 + 命令拼装 + 阶段执行。"""
|
||||
|
||||
def __init__(self, image_map: ImageMap, config: KernelConfig) -> None:
|
||||
self.map = image_map
|
||||
self.cfg = config
|
||||
|
||||
# ---- 映射表加载（类方法，供应用启动）----
|
||||
|
||||
@classmethod
|
||||
def load(cls, image_map_path: str | Path, config: KernelConfig) -> "Kernel":
|
||||
path = Path(image_map_path)
|
||||
if not path.exists():
|
||||
raise ImageMapError(f"stage_image_map.json 缺失: {path}")
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (ValueError, OSError) as exc:
|
||||
raise ImageMapError(f"stage_image_map.json 解析失败: {exc}") from exc
|
||||
image_map = ImageMap(
|
||||
images=raw.get("images") or {},
|
||||
stages=raw.get("stages") or {},
|
||||
family_chain=raw.get("family_chain") or [],
|
||||
)
|
||||
return cls(image_map, config)
|
||||
|
||||
# ---- 查表 ----
|
||||
|
||||
def image_for(self, stage: str) -> dict[str, Any] | None:
|
||||
return self.map.image_for(stage)
|
||||
|
||||
def has_stage(self, stage: str) -> bool:
|
||||
return stage in self.map.stages
|
||||
|
||||
# ---- 命令拼装（纯函数式，可 mock 断言）----
|
||||
|
||||
def build_command(
|
||||
self,
|
||||
stage: str,
|
||||
job_input_dir: str | Path,
|
||||
job_workspace_dir: str | Path,
|
||||
params: dict[str, Any] | None = None,
|
||||
*,
|
||||
cpu: str | None = None,
|
||||
memory: str | None = None,
|
||||
module: str | None = None,
|
||||
) -> list[str]:
|
||||
"""拼装 docker run 命令（list，无 shell；TC-BIO-KERN-001 基元）。
|
||||
|
||||
顺序：docker,run,--rm,--user,uid,-v(×3),--cpus,--memory,-e(×N),
|
||||
tag:镜像,python,-B,-m,cli.run_X,[参数…]
|
||||
"""
|
||||
params = params or {}
|
||||
entry = self.map.stages.get(stage)
|
||||
if entry is None:
|
||||
raise UnknownStageError(f"阶段不在映射表: {stage}")
|
||||
image = str(entry.get("image") or "")
|
||||
if not image:
|
||||
raise UnknownStageError(f"阶段 {stage} 无镜像映射")
|
||||
mod = module or entry.get("module")
|
||||
if not mod:
|
||||
raise UnknownStageError(f"阶段 {stage} 无执行模块")
|
||||
|
||||
cmd: list[str] = ["docker", "run", "--rm", "--user", str(self.cfg.host_uid)]
|
||||
cmd += ["-v", f"{job_input_dir}:{self.cfg.container_input_dir}"]
|
||||
cmd += ["-v", f"{job_workspace_dir}:{self.cfg.container_workspace_dir}"]
|
||||
if self.cfg.data_root:
|
||||
cmd += ["-v", f"{self.cfg.data_root}:{self.cfg.container_data_dir}:ro"]
|
||||
eff_cpu = cpu or str(params.get("cpu") or "")
|
||||
eff_memory = memory or str(params.get("memory") or "")
|
||||
cmd += ["--cpus", self._safe_cpu(eff_cpu or self.cfg.default_cpus)]
|
||||
cmd += ["--memory", self._safe_memory(eff_memory or self.cfg.default_memory)]
|
||||
for key in sorted(self.cfg.bba_env):
|
||||
cmd += ["-e", f"{key}={self.cfg.bba_env[key]}"]
|
||||
# 用户自定义 env 追加（params 中 env 键）
|
||||
extra_env = params.get("env") if isinstance(params.get("env"), dict) else {}
|
||||
for key in sorted(extra_env):
|
||||
cmd += ["-e", f"{key}={extra_env[key]}"]
|
||||
tag = self.map.docker_tag(image)
|
||||
cmd.append(f"{tag}:{self.cfg.docker_tag_suffix}")
|
||||
cmd.extend(CONTAINER_CMD_PREFIX)
|
||||
cmd.extend(["-m", mod])
|
||||
# 通用容器内参数：input-dir/workspace/cpu（各入口 argparse 兼容）
|
||||
cmd += ["--input-dir", self.cfg.container_input_dir]
|
||||
cmd += ["--workspace", self.cfg.container_workspace_dir]
|
||||
for key in sorted(params):
|
||||
value = params[key]
|
||||
if key in ("env", "cpu", "memory"):
|
||||
continue # cpu/memory 走 --cpus/--memory；env 走 -e
|
||||
if value is None:
|
||||
continue
|
||||
if isinstance(value, bool):
|
||||
if value:
|
||||
cmd.append(f"--{key.replace('_', '-')}")
|
||||
elif isinstance(value, (list, tuple)):
|
||||
cmd.append(f"--{key.replace('_', '-')}")
|
||||
cmd.append(",".join(str(v) for v in value))
|
||||
else:
|
||||
cmd.append(f"--{key.replace('_', '-')}")
|
||||
cmd.append(str(value))
|
||||
return cmd
|
||||
|
||||
def _safe_cpu(self, cpu: str) -> str:
|
||||
if str(cpu).strip() in {c.strip() for c in self.cfg.cpu_allow.split(",")}:
|
||||
return str(cpu).strip()
|
||||
logger.warning("bioflow kernel: cpu=%s 不在白名单 %s，回落默认 %s",
|
||||
cpu, self.cfg.cpu_allow, self.cfg.default_cpus)
|
||||
return self.cfg.default_cpus
|
||||
|
||||
def _safe_memory(self, memory: str) -> str:
|
||||
if str(memory).strip() in {m.strip() for m in self.cfg.memory_allow.split(",")}:
|
||||
return str(memory).strip()
|
||||
logger.warning("bioflow kernel: memory=%s 不在白名单 %s，回落默认 %s",
|
||||
memory, self.cfg.memory_allow, self.cfg.default_memory)
|
||||
return self.cfg.default_memory
|
||||
|
||||
@staticmethod
|
||||
def command_display(cmd: list[str]) -> str:
|
||||
"""命令可读形态（日志用；list 执行保证无 shell 注入）。"""
|
||||
return " ".join(shlex.quote(str(c)) for c in cmd)
|
||||
|
||||
# ---- 执行 ----
|
||||
|
||||
async def run_stage(
|
||||
self,
|
||||
stage: str,
|
||||
job_input_dir: str | Path,
|
||||
job_workspace_dir: str | Path,
|
||||
params: dict[str, Any] | None = None,
|
||||
*,
|
||||
log_path: str | Path | None = None,
|
||||
cpu: str | None = None,
|
||||
memory: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""执行一个阶段 docker：返回 {exit_code, timed_out, log_tail, cmd}。
|
||||
|
||||
- 超时：terminate → 3s → kill（TC-BIO-KERN-009）
|
||||
- 日志全量写 log_path（任务 logs/），返回尾部 log_tail_lines 行
|
||||
- 退出码非 0 / 超时 → 失败语义由 runner 判定（本函数只报事实）
|
||||
"""
|
||||
params = params or {}
|
||||
cpu = cpu or str(params.get("cpu") or "")
|
||||
memory = memory or str(params.get("memory") or "")
|
||||
cmd = self.build_command(stage, job_input_dir, job_workspace_dir, params,
|
||||
cpu=cpu or None, memory=memory or None)
|
||||
log_path = Path(log_path) if log_path else Path(job_workspace_dir) / "logs" / f"{stage}.log"
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
timed_out = False
|
||||
try:
|
||||
stdout, _ = await asyncio.wait_for(
|
||||
proc.communicate(), timeout=self.cfg.stage_timeout_sec
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
timed_out = True
|
||||
try:
|
||||
proc.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=3.0)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
stdout = b""
|
||||
exit_code = proc.returncode
|
||||
text = (stdout or b"").decode("utf-8", errors="replace")
|
||||
log_path.write_text(text, encoding="utf-8")
|
||||
return {
|
||||
"exit_code": exit_code,
|
||||
"timed_out": timed_out,
|
||||
"log_tail": tail_lines(text, self.cfg.log_tail_lines),
|
||||
"cmd": cmd,
|
||||
}
|
||||
|
||||
|
||||
def tail_lines(text: str, n: int) -> str:
|
||||
"""返回文本最后 n 行（日志截断；TC-BIO-RUN-013）。"""
|
||||
lines = text.splitlines()
|
||||
if len(lines) <= n:
|
||||
return "\n".join(lines)
|
||||
return "\n".join(lines[-n:])
|
||||
|
||||
|
||||
def _default_kernel_config() -> KernelConfig:
|
||||
"""从 app.config 推导（默认 uid = 服务进程 os.getuid()）。"""
|
||||
from ...config import get_settings
|
||||
|
||||
s = get_settings()
|
||||
return KernelConfig(
|
||||
host_uid=s.host_uid,
|
||||
docker_tag_suffix=s.docker_image_tag_suffix,
|
||||
data_root=s.data_root_abs,
|
||||
container_data_dir=s.container_data_dir,
|
||||
container_input_dir=s.container_input_dir,
|
||||
container_workspace_dir=s.container_workspace_dir,
|
||||
default_cpus=s.default_cpus,
|
||||
default_memory=s.default_memory,
|
||||
stage_timeout_sec=s.stage_timeout_sec,
|
||||
log_tail_lines=s.log_tail_lines,
|
||||
bba_env=s.bba_env,
|
||||
)
|
||||
|
||||
|
||||
def load_kernel() -> Kernel:
|
||||
"""应用启动：加载映射表 + 配置推导（映射表损坏 → ImageMapError 上抛）。"""
|
||||
from ...config import get_settings
|
||||
|
||||
return Kernel.load(get_settings().image_map_abs, _default_kernel_config())
|
||||
@@ -0,0 +1,154 @@
|
||||
"""app/services/bioflow/llm.py —— 生信规划 LLM 调用（沿用 docgen deepseek 直连模式）。
|
||||
|
||||
DOCGEN_LLM_* 键（DOCGEN_LLM_BASE_URL/API_KEY/MODEL/TIMEOUT）+ BIOFLOW_LLM_PLAN_TIMEOUT
|
||||
规划调用超时。complete_json 带一次 JSON 修复重试；重试指数退避（同 docgen）。
|
||||
密钥只存在于本进程，绝不进入任何 API 响应/日志。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BACKOFF_SECONDS = (1.5, 3.0, 6.0)
|
||||
MAX_ATTEMPTS = 4
|
||||
|
||||
|
||||
class LLMError(Exception):
|
||||
"""LLM 调用失败（重试尽/响应异常）。"""
|
||||
|
||||
|
||||
class LLMTimeout(LLMError):
|
||||
"""规划 LLM 超时（chat 流降级 PLAN_LLM_TIMEOUT）。"""
|
||||
|
||||
|
||||
class LLMUnavailable(LLMError):
|
||||
"""LLM 完全不可用（连接失败/未配置 key；降级 PLAN_UNAVAILABLE）。"""
|
||||
|
||||
|
||||
def llm_config() -> dict[str, Any]:
|
||||
return {
|
||||
"base_url": (os.environ.get("DOCGEN_LLM_BASE_URL") or "https://api.deepseek.com/v1").rstrip("/"),
|
||||
"api_key": os.environ.get("DOCGEN_LLM_API_KEY", ""),
|
||||
"model": os.environ.get("DOCGEN_LLM_MODEL") or "deepseek-chat",
|
||||
"timeout": float(os.environ.get("DOCGEN_LLM_TIMEOUT") or "300"),
|
||||
}
|
||||
|
||||
|
||||
def _extract_text(payload: dict[str, Any]) -> str:
|
||||
try:
|
||||
choices = payload.get("choices") or []
|
||||
if choices:
|
||||
content = choices[0].get("message", {}).get("content")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
return "".join(str(c.get("text", "")) if isinstance(c, dict) else str(c) for c in content)
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
|
||||
async def _chat_once(
|
||||
client: httpx.AsyncClient,
|
||||
cfg: dict[str, Any],
|
||||
system: str,
|
||||
user: str,
|
||||
*,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
json_mode: bool,
|
||||
timeout: float,
|
||||
) -> str:
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if cfg["api_key"]:
|
||||
headers["Authorization"] = f"Bearer {cfg['api_key']}"
|
||||
body: dict[str, Any] = {
|
||||
"model": cfg["model"],
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if json_mode:
|
||||
body["response_format"] = {"type": "json_object"}
|
||||
resp = await client.post(
|
||||
f"{cfg['base_url']}/chat/completions",
|
||||
headers=headers,
|
||||
json=body,
|
||||
timeout=httpx.Timeout(connect=10.0, read=timeout, write=30.0, pool=10.0),
|
||||
)
|
||||
if resp.status_code == 401 or resp.status_code == 403:
|
||||
raise LLMUnavailable("LLM 鉴权失败")
|
||||
if resp.status_code >= 500:
|
||||
raise LLMError(f"LLM 5xx: {resp.status_code}")
|
||||
if resp.status_code >= 400:
|
||||
raise LLMError(f"LLM 4xx: {resp.status_code}")
|
||||
return _extract_text(resp.json())
|
||||
|
||||
|
||||
def _repair_json(text: str) -> Any:
|
||||
"""一次 JSON 修复重试：剥 code fence、截取首尾大括号。"""
|
||||
cleaned = text.strip()
|
||||
fence = re.match(r"^```(?:json)?\s*(.*?)\s*```$", cleaned, re.S)
|
||||
if fence:
|
||||
cleaned = fence.group(1).strip()
|
||||
try:
|
||||
return json.loads(cleaned)
|
||||
except ValueError:
|
||||
pass
|
||||
start, end = cleaned.find("{"), cleaned.rfind("}")
|
||||
if start >= 0 and end > start:
|
||||
try:
|
||||
return json.loads(cleaned[start:end + 1])
|
||||
except ValueError:
|
||||
pass
|
||||
raise ValueError("LLM 输出非 JSON")
|
||||
|
||||
|
||||
async def complete_json(
|
||||
system: str,
|
||||
user: str,
|
||||
*,
|
||||
temperature: float = 0.3,
|
||||
max_tokens: int = 4000,
|
||||
timeout: float | None = None,
|
||||
) -> Any:
|
||||
"""JSON 输出 LLM 调用：指数退避 + 一次 JSON 修复。
|
||||
|
||||
timeout=None 用配置默认（规划用 BIOFLOW_LLM_PLAN_TIMEOUT 更短）。
|
||||
"""
|
||||
cfg = llm_config()
|
||||
if not cfg["api_key"]:
|
||||
raise LLMUnavailable("LLM 未配置（DOCGEN_LLM_API_KEY 为空）")
|
||||
call_timeout = float(timeout if timeout is not None else cfg["timeout"])
|
||||
last_exc: Exception | None = None
|
||||
async with httpx.AsyncClient() as client:
|
||||
for attempt in range(MAX_ATTEMPTS):
|
||||
try:
|
||||
text = await _chat_once(
|
||||
client, cfg, system, user,
|
||||
temperature=temperature, max_tokens=max_tokens,
|
||||
json_mode=True, timeout=call_timeout,
|
||||
)
|
||||
return _repair_json(text)
|
||||
except asyncio.TimeoutError as exc:
|
||||
last_exc = exc
|
||||
raise LLMTimeout("规划 LLM 超时") from exc
|
||||
except (httpx.ConnectError, httpx.ConnectTimeout) as exc:
|
||||
last_exc = exc
|
||||
raise LLMUnavailable(f"LLM 不可用: {exc}") from exc
|
||||
except (LLMError, ValueError) as exc:
|
||||
last_exc = exc
|
||||
if attempt < MAX_ATTEMPTS - 1:
|
||||
await asyncio.sleep(BACKOFF_SECONDS[min(attempt, len(BACKOFF_SECONDS) - 1)])
|
||||
raise LLMError(f"LLM 调用失败（重试尽）: {last_exc}")
|
||||
@@ -0,0 +1,157 @@
|
||||
"""app/services/bioflow/manifests.py —— manifest 解析与校验（schema 随内核树）。
|
||||
|
||||
- 按 gfa_core/schemas/core/manifest_v1.schema.json 必需字段解析
|
||||
（manifest_version/job_id/stage/status/artifacts/parameters/metrics…）。
|
||||
- 定位规则（同内核 cli 行为）：final/{prefix}.{stage}.manifest.json 优先，
|
||||
final/ 下任意 *.manifest.json 兜底。
|
||||
- 跨阶段输入定位：上游 manifest 的 artifacts 路径（相对 workspace_root 解析
|
||||
为容器内路径，供下游 docker -v/参数拼接）。
|
||||
|
||||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REQUIRED_MANIFEST_KEYS = (
|
||||
"manifest_version", "job_id", "stage", "status", "artifacts",
|
||||
)
|
||||
|
||||
# 内核命名约定（workspace_contract_v1.md）：final/{prefix}.{stage}.manifest.json
|
||||
MANIFEST_GLOB = "*.manifest.json"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Manifest:
|
||||
"""解析后的 manifest（原始 dict 保留，供透传）。"""
|
||||
|
||||
stage: str
|
||||
status: str
|
||||
path: str
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def artifacts(self) -> dict[str, Any]:
|
||||
return self.raw.get("artifacts") or {}
|
||||
|
||||
@property
|
||||
def parameters(self) -> dict[str, Any]:
|
||||
return self.raw.get("parameters") or {}
|
||||
|
||||
@property
|
||||
def metrics(self) -> dict[str, Any]:
|
||||
return self.raw.get("metrics") or {}
|
||||
|
||||
@property
|
||||
def prefix(self) -> str:
|
||||
return str(self.raw.get("prefix") or "")
|
||||
|
||||
|
||||
class ManifestError(Exception):
|
||||
"""manifest 缺失/非法 JSON/必需字段缺失。"""
|
||||
|
||||
|
||||
def locate_manifest(workspace: str | Path, stage: str, prefix: str | None = None) -> Path | None:
|
||||
"""在 workspace final/ 下定位阶段 manifest（prefix + 阶段名双匹配）。
|
||||
|
||||
不存在返回 None（runner 据此判 reason=no_manifest）。
|
||||
注意：无兜底——其他阶段的 manifest 绝不误配给当前阶段
|
||||
（重启恢复按 manifest 判成败时曾因兜底把 identify 的 manifest 判给
|
||||
phylogeny，导致 running 阶段被误判 done 不重跑）。
|
||||
"""
|
||||
final_dir = Path(workspace) / "final"
|
||||
if not final_dir.is_dir():
|
||||
return None
|
||||
candidates = list(final_dir.glob(MANIFEST_GLOB))
|
||||
if not candidates:
|
||||
return None
|
||||
if prefix:
|
||||
for cand in candidates:
|
||||
if cand.name.startswith(prefix) and f".{stage}." in cand.name:
|
||||
return cand
|
||||
for cand in candidates:
|
||||
if f".{stage}." in cand.name:
|
||||
return cand
|
||||
return None
|
||||
|
||||
|
||||
def parse_manifest(workspace: str | Path, stage: str, prefix: str | None = None) -> Manifest:
|
||||
"""定位并解析 manifest；失败抛 ManifestError（含具体原因）。"""
|
||||
path = locate_manifest(workspace, stage, prefix)
|
||||
if path is None:
|
||||
raise ManifestError(f"manifest 缺失: workspace={workspace} stage={stage}")
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
except ValueError as exc:
|
||||
raise ManifestError(f"manifest 非法 JSON: {path}") from exc
|
||||
except OSError as exc:
|
||||
raise ManifestError(f"manifest 读取失败: {path}") from exc
|
||||
if not isinstance(raw, dict):
|
||||
raise ManifestError(f"manifest 非对象: {path}")
|
||||
missing = [k for k in REQUIRED_MANIFEST_KEYS if k not in raw]
|
||||
if missing:
|
||||
raise ManifestError(f"manifest 缺必需字段 {missing}: {path}")
|
||||
status = str(raw.get("status") or "")
|
||||
if status not in ("success", "failed", "partial_success"):
|
||||
raise ManifestError(f"manifest status 非法: {status!r}")
|
||||
return Manifest(stage=stage, status=status, path=str(path), raw=raw)
|
||||
|
||||
|
||||
def locate_artifacts(workspace: str | Path, stage: str, prefix: str | None = None) -> dict[str, Any]:
|
||||
"""解析上游 manifest 并返回 artifacts（跨阶段输入定位）。
|
||||
|
||||
解析失败返回 {}（调用方决定 fallback：上游失败则下游不应执行）。
|
||||
"""
|
||||
try:
|
||||
m = parse_manifest(workspace, stage, prefix)
|
||||
return m.artifacts
|
||||
except ManifestError:
|
||||
return {}
|
||||
|
||||
|
||||
def artifact_paths(artifacts: dict[str, Any]) -> list[str]:
|
||||
"""artifacts → 文件路径列表（full_path 优先，path/relpath_or_hint 兜底）。"""
|
||||
out: list[str] = []
|
||||
for value in artifacts.values():
|
||||
if isinstance(value, str):
|
||||
if value:
|
||||
out.append(value)
|
||||
continue
|
||||
if isinstance(value, dict):
|
||||
for key in ("full_path", "path", "relpath_or_hint"):
|
||||
v = value.get(key)
|
||||
if v:
|
||||
out.append(str(v))
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def input_resolution(
|
||||
upstream_workspace: str | Path,
|
||||
upstream_stage: str,
|
||||
prefix: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""下游阶段的跨阶段输入解析结果。
|
||||
|
||||
返回 {resolved: bool, inputs: [路径…], manifest_path: str|None, error: str|None}
|
||||
"""
|
||||
try:
|
||||
m = parse_manifest(upstream_workspace, upstream_stage, prefix)
|
||||
except ManifestError as exc:
|
||||
return {"resolved": False, "inputs": [], "manifest_path": None, "error": str(exc)}
|
||||
paths = artifact_paths(m.artifacts)
|
||||
existing = [p for p in paths if Path(p).exists()]
|
||||
if not existing:
|
||||
return {
|
||||
"resolved": False,
|
||||
"inputs": [],
|
||||
"manifest_path": m.path,
|
||||
"error": f"上游 {upstream_stage} artifacts 文件均不存在",
|
||||
}
|
||||
return {"resolved": True, "inputs": existing, "manifest_path": m.path, "error": None}
|
||||
@@ -0,0 +1,54 @@
|
||||
"""app/services/bioflow/params.py —— 参数预填（BBA_* 历史/契约文档 → 表单默认值）。
|
||||
|
||||
- 契约文档（gfa_core/schemas/stages/*_contract_v1.md）解析参数默认值
|
||||
（parameters 小节 key=value 形式）。
|
||||
- BBA_* 环境变量历史（进程内 config.bba_env 派生）。
|
||||
- 存档参数预填（FR-BIO-023：表单预填存档值——由 runner._form_defaults 用
|
||||
record.params 完成，本模块提供读取存档参数的工具）。
|
||||
|
||||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 契约文档 parameters 小节匹配（`- key` 或 `key:` 列表项）
|
||||
_PARAM_RE = re.compile(r"^\s*[-*]\s+([a-z_][a-z0-9_]*)\b", re.M)
|
||||
|
||||
|
||||
def read_contract_defaults(contract_path: str | Path) -> dict[str, Any]:
|
||||
"""契约文档 → 参数默认值快照（解析失败返回 {}，不阻断）。"""
|
||||
path = Path(contract_path)
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return {}
|
||||
defaults: dict[str, Any] = {}
|
||||
for key in _PARAM_RE.findall(text):
|
||||
defaults.setdefault(key, "")
|
||||
return defaults
|
||||
|
||||
|
||||
def contract_path_for(repo_root: str | Path, stage: str) -> Path:
|
||||
"""阶段契约文档路径（仓库内 gfa_core/schemas/stages/{stage}_contract_v1.md）。"""
|
||||
return Path(repo_root) / "bioflow_kernel" / "gfa_core" / "schemas" / "stages" / f"{stage}_contract_v1.md"
|
||||
|
||||
|
||||
def bba_env_defaults(settings_bba: dict[str, str]) -> dict[str, str]:
|
||||
"""BBA_* 环境 → 表单默认值标注（BBA_PFAM_DIR 等）。"""
|
||||
return {key: value for key, value in settings_bba.items()}
|
||||
|
||||
|
||||
def archive_param_prefill(archive: dict[str, Any], stage: str) -> dict[str, Any]:
|
||||
"""存档参数预填（FR-BIO-023）：取存档 params[stage]；无则空。"""
|
||||
params = archive.get("params") or {}
|
||||
if not isinstance(params, dict):
|
||||
return {}
|
||||
return dict(params.get(stage) or {})
|
||||
@@ -0,0 +1,214 @@
|
||||
"""app/services/bioflow/planner.py —— 路径规划（知识库检索 + LLM 候选 → catalog 硬校验）。
|
||||
|
||||
FR-BIO-002/003：
|
||||
- 生信意图消息 → 规划：vstore/flow.py FlowStore 本地实例混合检索
|
||||
（向量+词面双通道）+ LLM 生成 2-3 条候选路径。
|
||||
- catalog 硬校验：未知阶段直接拒绝（UNKNOWN_STAGE）；缺失前置自动补全并明示
|
||||
（auto_filled）。LLM 输出反复非法 → 最多 plan_max_rounds 轮。
|
||||
- vstore 不可用 → 降级 llm_only（LLM 直接候选；source 标记；无 5xx 中断）。
|
||||
- LLM 超时 → PLAN_LLM_TIMEOUT 降级提示；LLM 完全不可用 → PLAN_UNAVAILABLE。
|
||||
- 复用句式（「用 XXX 存档任务的方法…」）：按名称匹配存档直接装载阶段链+参数
|
||||
（不调 LLM）。
|
||||
|
||||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .catalog import StageCatalog
|
||||
from .intent import extract_reuse_name
|
||||
from .llm import LLMError, LLMTimeout, LLMUnavailable, complete_json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PLAN_SYSTEM_PROMPT = (
|
||||
"你是 PlantAI 生信流程规划器。根据用户的分析需求，从给定的阶段目录中"
|
||||
"选出 2-3 条候选分析路径。每条候选："
|
||||
"{\"purpose\":\"一句话用途说明\",\"stages\":[\"阶段名1\",\"阶段名2\",...]}。"
|
||||
"阶段名必须严格取自阶段目录（不得自造）。"
|
||||
"基因家族分析常用链示例：genome_prepare→identify→phylogeny→motif→domain。"
|
||||
"只输出 JSON：{\"candidates\":[{...},{...}]}。"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlanCandidate:
|
||||
purpose: str
|
||||
stages: list[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlanResult:
|
||||
"""规划结果（chat 流 plan_ready / 降级事件的 data 载荷）。"""
|
||||
|
||||
candidates: list[PlanCandidate] = field(default_factory=list)
|
||||
source: str = "llm_only" # llm_only | llm_rag | archive | builtin
|
||||
kb_hits: list[dict[str, Any]] = field(default_factory=list)
|
||||
auto_filled: list[str] = field(default_factory=list)
|
||||
reuse_name: str | None = None
|
||||
reuse_archive_id: str | None = None
|
||||
message: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlannerError(Exception):
|
||||
"""规划失败（带错误码，chat 流按码降级/报错）。"""
|
||||
|
||||
code: str
|
||||
message: str
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class Planner:
|
||||
"""路径规划器（vstore 检索 + LLM 候选 + catalog 硬校验）。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
catalog: StageCatalog,
|
||||
flow_store: Any = None, # vstore.flow.FlowStore 本地实例（可注入）
|
||||
*,
|
||||
llm_plan_timeout: float = 60.0,
|
||||
plan_candidates: int = 3,
|
||||
plan_max_rounds: int = 2,
|
||||
archive_finder: Any = None, # callable(name)->archive|None（复用匹配）
|
||||
) -> None:
|
||||
self.catalog = catalog
|
||||
self.flow_store = flow_store
|
||||
self.llm_plan_timeout = llm_plan_timeout
|
||||
self.plan_candidates = plan_candidates
|
||||
self.plan_max_rounds = plan_max_rounds
|
||||
self.archive_finder = archive_finder
|
||||
|
||||
# ---- 知识库检索（FR-BIO-026；vstore 不可用降级空）----
|
||||
|
||||
def kb_search(self, query: str, top_k: int = 6) -> list[dict[str, Any]]:
|
||||
"""FlowStore 混合检索；失败降级 []（source 标记 llm_only）。"""
|
||||
if self.flow_store is None:
|
||||
return []
|
||||
try:
|
||||
result = self.flow_store.search(query, top_k=top_k)
|
||||
return result.get("items") or []
|
||||
except Exception as exc: # noqa: BLE001 - 检索降级，不阻断规划
|
||||
logger.warning("planner: 知识库检索失败（降级 llm_only）: %s", exc)
|
||||
return []
|
||||
|
||||
def kb_context(self, query: str, top_k: int = 6) -> str:
|
||||
"""检索块 → LLM 上下文（标题路径 + 正文摘录；预算裁剪）。"""
|
||||
hits = self.kb_search(query, top_k=top_k)
|
||||
if not hits:
|
||||
return ""
|
||||
parts: list[str] = []
|
||||
budget = 4000
|
||||
for hit in hits:
|
||||
title = hit.get("title_path") or hit.get("title") or hit.get("stage_name") or ""
|
||||
text = str(hit.get("description") or "")[:200]
|
||||
snippet = f"- {title}: {text}"
|
||||
if sum(len(p) for p in parts) + len(snippet) > budget:
|
||||
break
|
||||
parts.append(snippet)
|
||||
return "流程知识库参考（供你选择阶段）：\n" + "\n".join(parts)
|
||||
|
||||
# ---- 规划主入口 ----
|
||||
|
||||
async def plan(self, message: str) -> PlanResult:
|
||||
"""消息 → 候选路径（复用句式优先，其次 LLM+KB，均不可用→内置兜底）。"""
|
||||
# 1. 存档复用句式（FR-BIO-023）：按名称匹配存档直接装载
|
||||
reuse_name = extract_reuse_name(message)
|
||||
if reuse_name and self.archive_finder is not None:
|
||||
archive = await self._find_archive(reuse_name)
|
||||
if archive is not None:
|
||||
stages = [s for s in (archive.get("stage_chain") or []) if self.catalog.has(s)]
|
||||
if stages:
|
||||
return PlanResult(
|
||||
candidates=[PlanCandidate(
|
||||
purpose=f"复用存档「{archive.get('name')}」的方法",
|
||||
stages=stages,
|
||||
)],
|
||||
source="archive",
|
||||
reuse_name=reuse_name,
|
||||
reuse_archive_id=str(archive.get("id") or ""),
|
||||
)
|
||||
|
||||
# 2. 知识库检索（失败自动降级）
|
||||
stage_names = self.catalog.stage_names()
|
||||
kb_context = self.kb_context(message)
|
||||
|
||||
# 3. LLM 候选生成 + catalog 硬校验（最多 plan_max_rounds）
|
||||
try:
|
||||
raw = await complete_json(
|
||||
PLAN_SYSTEM_PROMPT,
|
||||
(
|
||||
f"阶段目录（全部合法阶段名）：{json.dumps(stage_names, ensure_ascii=False)}\n"
|
||||
f"{kb_context}"
|
||||
f"用户需求：{message}\n"
|
||||
"请输出 2-3 条候选路径 JSON。"
|
||||
),
|
||||
temperature=0.3,
|
||||
max_tokens=2000,
|
||||
timeout=self.llm_plan_timeout,
|
||||
)
|
||||
except LLMTimeout as exc:
|
||||
raise PlannerError("PLAN_LLM_TIMEOUT", "规划服务超时，已为您提供内置默认路径",
|
||||
{"fallback": True}) from exc
|
||||
except (LLMUnavailable, LLMError) as exc:
|
||||
raise PlannerError("PLAN_UNAVAILABLE", "规划服务暂不可用，已为您提供内置默认路径",
|
||||
{"fallback": True}) from exc
|
||||
|
||||
candidates = await self._validate_llm_candidates(raw)
|
||||
return PlanResult(
|
||||
candidates=candidates,
|
||||
source="llm_rag" if kb_context else "llm_only",
|
||||
)
|
||||
|
||||
async def _find_archive(self, name: str) -> dict[str, Any] | None:
|
||||
try:
|
||||
result = self.archive_finder(name)
|
||||
if asyncio.iscoroutine(result):
|
||||
result = await result
|
||||
return result if isinstance(result, dict) else None
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
async def _validate_llm_candidates(self, raw: Any) -> list[PlanCandidate]:
|
||||
"""LLM 输出 → catalog 硬校验（未知阶段拒绝；前置补全明示）。
|
||||
|
||||
非法候选（无有效 stages 或全部未知阶段）直接丢弃；
|
||||
多次重试后仍无有效候选 → 内置兜底链。
|
||||
"""
|
||||
candidates: list[PlanCandidate] = []
|
||||
raw_candidates = (raw.get("candidates") if isinstance(raw, dict) else None) or []
|
||||
if not isinstance(raw_candidates, list):
|
||||
raise PlannerError("PLAN_FALLBACK", "规划结果异常", {"fallback": True})
|
||||
|
||||
for cand in raw_candidates[: self.plan_candidates]:
|
||||
if not isinstance(cand, dict):
|
||||
continue
|
||||
stages = [str(s) for s in (cand.get("stages") or []) if s]
|
||||
unknown = self.catalog.validate_chain(stages)
|
||||
if unknown:
|
||||
# 未知阶段硬拒绝（FR-BIO-003）：整条候选丢弃并记录
|
||||
logger.warning("planner: 候选含未知阶段被拒 %s: %s", unknown, stages)
|
||||
continue
|
||||
ordered, auto_filled = self.catalog.auto_fill_prerequisites(stages)
|
||||
if not ordered:
|
||||
continue
|
||||
candidates.append(PlanCandidate(
|
||||
purpose=str(cand.get("purpose") or "").strip() or "生信分析流程",
|
||||
stages=ordered,
|
||||
))
|
||||
if auto_filled:
|
||||
# 明示补全（首条候选带出即可；PlanResult.auto_filled 在 router 组装）
|
||||
pass
|
||||
|
||||
if not candidates:
|
||||
# 内置兜底（plan_max_rounds 已由 router 重试；此处为最终兜底）
|
||||
candidates = [PlanCandidate(
|
||||
purpose="基因家族鉴定标准流程（内置默认）",
|
||||
stages=[s for s in ("identify",) if self.catalog.has(s)],
|
||||
)]
|
||||
return candidates
|
||||
@@ -0,0 +1,99 @@
|
||||
"""app/services/bioflow/reports.py —— 报告登记（内核报告产物 → docgen 报告登记端点协调）。
|
||||
|
||||
FR-BIO-015/016：任务最后阶段运行报告链产出图文 HTML（图 base64 内嵌）。
|
||||
- locate_report：任务 final/ 下找报告 HTML/MD。
|
||||
- register_report：内部 HTTP 调 BFF 反代报告登记端点（/api/bioflow/reports/register，
|
||||
BFF 落 doc_artifacts，字段 job_id/标题/md/图清单——与 A3 report 模板协调）；
|
||||
BFF 不可用（直连形态 FR-BIO-027）→ 降级：仅返回本地登记载荷不抛错。
|
||||
- 图清单：HTML 中 fig://id 引用统计（base64 内嵌断言 TC-BIO-API-124 由报告链保证）。
|
||||
|
||||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。httpx 只在此处（网络边界）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REPORT_GLOB_HTML = "*.html"
|
||||
REPORT_GLOB_MD = "*_report.md"
|
||||
FIG_REF_RE = re.compile(r"fig://([a-zA-Z0-9_\-]+)")
|
||||
|
||||
|
||||
def locate_report(workspace_dir: str | Path) -> Path | None:
|
||||
"""任务 workspace final/ 下定位报告文件（HTML 优先，MD 兜底）。"""
|
||||
final_dir = Path(workspace_dir) / "final"
|
||||
if not final_dir.is_dir():
|
||||
return None
|
||||
htmls = sorted(final_dir.glob(REPORT_GLOB_HTML))
|
||||
if htmls:
|
||||
return htmls[0]
|
||||
mds = sorted(final_dir.glob(REPORT_GLOB_MD))
|
||||
return mds[0] if mds else None
|
||||
|
||||
|
||||
def extract_figure_refs(report_path: str | Path) -> list[str]:
|
||||
"""报告 HTML 中 fig://id 图引用清单（顺序去重）。"""
|
||||
path = Path(report_path)
|
||||
if not path.exists():
|
||||
return []
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return []
|
||||
seen: list[str] = []
|
||||
for m in FIG_REF_RE.finditer(text):
|
||||
ref = m.group(1)
|
||||
if ref not in seen:
|
||||
seen.append(ref)
|
||||
return seen
|
||||
|
||||
|
||||
def report_payload(
|
||||
job: dict[str, Any],
|
||||
report_path: str | Path,
|
||||
*,
|
||||
title: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""登记载荷（BFF 落 doc_artifacts 的字段约定：job_id/标题/md/图清单）。"""
|
||||
figures = extract_figure_refs(report_path)
|
||||
return {
|
||||
"job_id": job.get("id") or "",
|
||||
"session_id": job.get("session_id") or "",
|
||||
"title": title or job.get("title") or "生信分析报告",
|
||||
"md_path": str(report_path),
|
||||
"figures": figures,
|
||||
}
|
||||
|
||||
|
||||
async def register_report(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
register_url: str | None = None,
|
||||
timeout: float = 30.0,
|
||||
) -> dict[str, Any]:
|
||||
"""内部 HTTP 登记到 BFF 报告登记端点。
|
||||
|
||||
BFF 不可达（只部署生信形态 FR-BIO-027）→ 返回 {ok:True, degraded:True}，
|
||||
报告仍可经服务直连下载 HTML。登记失败不抛错（登记为尽力而为）。
|
||||
"""
|
||||
if not register_url:
|
||||
return {"ok": False, "degraded": True, "reason": "no_register_url"}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=5.0, read=timeout)) as client:
|
||||
resp = await client.post(register_url, json=payload)
|
||||
if resp.status_code >= 400:
|
||||
logger.warning("reports: 登记端点返回 %s（降级为本地仅登记）", resp.status_code)
|
||||
return {"ok": False, "degraded": True, "reason": f"http_{resp.status_code}"}
|
||||
try:
|
||||
return resp.json()
|
||||
except ValueError:
|
||||
return {"ok": True, "degraded": False}
|
||||
except httpx.HTTPError as exc:
|
||||
logger.warning("reports: 登记端点不可达（降级为本地仅登记）: %s", exc)
|
||||
return {"ok": False, "degraded": True, "reason": "unreachable"}
|
||||
@@ -0,0 +1,722 @@
|
||||
"""app/services/bioflow/runner.py —— 任务状态机（逐阶段推进）+ 并发信号量 + 重启恢复。
|
||||
|
||||
状态机（job）：created → await_inputs → running → paused / done / failed / archiving / archived
|
||||
状态机（stage）：waiting → await_files → waiting_confirm → queued → running
|
||||
→ done / failed / skipped；failed →（rerun）waiting_confirm/running 或（skip）skipped
|
||||
|
||||
职责：
|
||||
- run_job：逐阶段检查输入→等待确认（手动模式）→docker run→读 manifest 判成败→推进/暂停。
|
||||
- 事件总线：每个状态变化 publish（12 类事件全清单）。
|
||||
- 并发信号量（BIOFLOW_CONCURRENCY 默认 3，FR-BIO-031）：docker 执行前获取，
|
||||
无槽则排队（stage=queued）；任务失败/完成/暂停释放槽。
|
||||
- 服务重启恢复：扫描 DB，running/running 阶段任务从断点续跑
|
||||
（已完成阶段不重跑；按 manifest 判成败——FR-BIO-031）。
|
||||
- 执行器注入：默认 Kernel.run_stage（真 docker）；测试注入 fake（mock docker）。
|
||||
|
||||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
from .catalog import StageCatalog
|
||||
from .events import EventBus
|
||||
from .manifests import ManifestError, locate_manifest, parse_manifest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---- 状态常量 ----
|
||||
|
||||
STAGE_WAITING = "waiting"
|
||||
STAGE_AWAIT_FILES = "await_files"
|
||||
STAGE_WAITING_CONFIRM = "waiting_confirm"
|
||||
STAGE_QUEUED = "queued"
|
||||
STAGE_RUNNING = "running"
|
||||
STAGE_DONE = "done"
|
||||
STAGE_FAILED = "failed"
|
||||
STAGE_SKIPPED = "skipped"
|
||||
STAGE_PAUSED = "paused"
|
||||
|
||||
STAGE_TERMINAL = {STAGE_DONE, STAGE_FAILED, STAGE_SKIPPED}
|
||||
STAGE_RUNNABLE = {STAGE_WAITING, STAGE_AWAIT_FILES, STAGE_WAITING_CONFIRM, STAGE_QUEUED}
|
||||
|
||||
JOB_CREATED = "created"
|
||||
JOB_AWAIT_INPUTS = "await_inputs"
|
||||
JOB_RUNNING = "running"
|
||||
JOB_PAUSED = "paused"
|
||||
JOB_DONE = "done"
|
||||
JOB_FAILED = "failed"
|
||||
JOB_ARCHIVING = "archiving"
|
||||
JOB_ARCHIVED = "archived"
|
||||
|
||||
JOB_TERMINAL = {JOB_DONE, JOB_FAILED, JOB_ARCHIVED}
|
||||
|
||||
# 首阶段默认输入文件名（FR-BIO-005 文件引导提示用）
|
||||
DEFAULT_INPUT_HINTS = ("pep.fa", "pep.fasta", ".fasta", ".fa")
|
||||
|
||||
# manifest 判成败兜底：退出码 0 但 manifest status=failed → 失败（TC-BIO-RUN-016）
|
||||
# manifest 缺必需字段但退出码 0 → 按退出码兜底（TC-BIO-KERN-017）
|
||||
|
||||
|
||||
class RunnerError(Exception):
|
||||
"""runner 操作错误（状态机非法迁移等；路由层转 409 信封）。"""
|
||||
|
||||
def __init__(self, code: str, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
# ---- 阶段输入检查 ----
|
||||
|
||||
def _stage_input_satisfied(stage: str, job_input_dir: Path) -> tuple[bool, list[str]]:
|
||||
"""检查阶段输入文件是否就绪（目录内存在生信数据文件）。
|
||||
|
||||
- 首阶段（identify/genome_prepare/seq 提取类）：要求 input/ 下有 fasta/gz 类文件；
|
||||
genome_prepare 放宽（允许 gff3/cds）。
|
||||
- 后续阶段：跨阶段输入（上游 final/ 产物），有上游 done 即满足（无显式缺名）。
|
||||
返回 (satisfied, missing_hints)。实现为保守启发：目录内任一数据文件即视为满足。
|
||||
"""
|
||||
if not job_input_dir.is_dir():
|
||||
return False, list(DEFAULT_INPUT_HINTS)
|
||||
data_exts = (".fa", ".fasta", ".faa", ".fna", ".pep", ".cds", ".gz", ".gff", ".gff3", ".gtf", ".txt", ".tsv")
|
||||
files = [p for p in job_input_dir.rglob("*") if p.is_file()]
|
||||
data_files = [p for p in files if p.suffix.lower() in data_exts or ".fasta" in p.name.lower()]
|
||||
if data_files:
|
||||
return True, []
|
||||
return False, list(DEFAULT_INPUT_HINTS)
|
||||
|
||||
|
||||
# ---- runner ----
|
||||
|
||||
ExecutorFn = Callable[..., Awaitable[dict[str, Any]]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobRecord:
|
||||
"""任务内存记录（DB 行 + 派生路径）。"""
|
||||
|
||||
id: str
|
||||
session_id: str = ""
|
||||
title: str = ""
|
||||
status: str = JOB_CREATED
|
||||
stage_chain: list[str] = field(default_factory=list)
|
||||
params: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
auto_continue: bool = False
|
||||
archive_id: str | None = None
|
||||
pause_requested: bool = False
|
||||
_stage_states: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def job_dir(self) -> Path:
|
||||
from ...config import get_settings
|
||||
return Path(get_settings().jobs_dir_abs) / self.id
|
||||
|
||||
@property
|
||||
def input_dir(self) -> Path:
|
||||
return self.job_dir / "input"
|
||||
|
||||
@property
|
||||
def workspace_dir(self) -> Path:
|
||||
return self.job_dir / "workspace"
|
||||
|
||||
@property
|
||||
def logs_dir(self) -> Path:
|
||||
return self.job_dir / "logs"
|
||||
|
||||
def progress(self) -> int:
|
||||
total = len(self.stage_chain)
|
||||
if not total:
|
||||
return 100
|
||||
done = sum(
|
||||
1 for s in self.stage_chain
|
||||
if self._stage_states.get(s, {}).get("status") in (STAGE_DONE, STAGE_SKIPPED)
|
||||
)
|
||||
return int(done * 100 / total)
|
||||
|
||||
|
||||
class Runner:
|
||||
"""任务状态机执行器（每任务一个实例；共享 db/events/kernel/catalog/semaphore）。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db: Any,
|
||||
events: EventBus,
|
||||
executor: ExecutorFn,
|
||||
catalog: StageCatalog,
|
||||
semaphore: asyncio.Semaphore,
|
||||
*,
|
||||
force_input_check: bool = True,
|
||||
store: Any = None, # 存储层（测试可注入；生产用 app/services/bioflow/store）
|
||||
task_factory: Any = None, # asyncio.create_task 替身（测试注入）
|
||||
auto_mode_default: bool = False,
|
||||
) -> None:
|
||||
self.db = db
|
||||
self.events = events
|
||||
self.executor = executor
|
||||
self.catalog = catalog
|
||||
self.semaphore = semaphore
|
||||
self.force_input_check = force_input_check
|
||||
self.store = store
|
||||
self._task_factory = task_factory or asyncio.create_task
|
||||
self.auto_mode_default = auto_mode_default
|
||||
self._tasks: set[asyncio.Task] = set()
|
||||
|
||||
# ---- 任务生命周期 ----
|
||||
|
||||
async def create_job(
|
||||
self,
|
||||
session_id: str,
|
||||
title: str,
|
||||
stages: list[str],
|
||||
params: dict[str, dict[str, Any]] | None = None,
|
||||
*,
|
||||
auto_continue: bool = False,
|
||||
archive_id: str | None = None,
|
||||
auto_start: bool = True,
|
||||
) -> JobRecord:
|
||||
"""创建任务（job_created 事件 + 三表落行 + 可选自动启动）。"""
|
||||
unknown = self.catalog.validate_chain(stages)
|
||||
if unknown:
|
||||
raise RunnerError("UNKNOWN_STAGE", f"未知阶段: {', '.join(unknown)}")
|
||||
ordered, _added = self.catalog.auto_fill_prerequisites(stages)
|
||||
job_id = uuid.uuid4().hex[:16]
|
||||
record = JobRecord(
|
||||
id=job_id, session_id=session_id, title=title or "生信任务",
|
||||
status=JOB_CREATED, stage_chain=ordered,
|
||||
params=params or {}, auto_continue=auto_continue or self.auto_mode_default,
|
||||
archive_id=archive_id,
|
||||
)
|
||||
record.job_dir.mkdir(parents=True, exist_ok=True)
|
||||
record.input_dir.mkdir(parents=True, exist_ok=True)
|
||||
record.workspace_dir.mkdir(parents=True, exist_ok=True)
|
||||
record.logs_dir.mkdir(parents=True, exist_ok=True)
|
||||
for stage in ordered:
|
||||
record._stage_states[stage] = {"status": STAGE_WAITING, "params": params.get(stage) or {} if params else {}}
|
||||
await self._persist_job(record)
|
||||
for stage in ordered:
|
||||
await self._persist_stage(record, stage)
|
||||
self.events.publish("job_created", {
|
||||
"job_id": record.id,
|
||||
"session_id": record.session_id,
|
||||
"title": record.title,
|
||||
"stages": [{"stage": s, "status": STAGE_WAITING} for s in ordered],
|
||||
"auto_continue": record.auto_continue,
|
||||
})
|
||||
if auto_start:
|
||||
self.spawn(self.run_job(record))
|
||||
return record
|
||||
|
||||
def spawn(self, coro: Awaitable[Any]) -> asyncio.Task:
|
||||
task = self._task_factory(coro)
|
||||
self._tasks.add(task)
|
||||
task.add_done_callback(self._tasks.discard)
|
||||
return task
|
||||
|
||||
# ---- DB 持久化 ----
|
||||
|
||||
async def _persist_job(self, record: JobRecord) -> None:
|
||||
if self.store is not None:
|
||||
await self.store.upsert_job(record)
|
||||
return
|
||||
from .store import upsert_job
|
||||
await upsert_job(self.db, record)
|
||||
|
||||
async def _persist_stage(self, record: JobRecord, stage: str) -> None:
|
||||
if self.store is not None:
|
||||
await self.store.upsert_stage(record, stage)
|
||||
return
|
||||
from .store import upsert_stage
|
||||
await upsert_stage(self.db, record, stage)
|
||||
|
||||
# ---- 阶段状态读写 ----
|
||||
|
||||
def stage_status(self, record: JobRecord, stage: str) -> str:
|
||||
return str(record._stage_states.get(stage, {}).get("status") or STAGE_WAITING)
|
||||
|
||||
def _set_stage(self, record: JobRecord, stage: str, **fields: Any) -> None:
|
||||
st = record._stage_states.setdefault(stage, {"status": STAGE_WAITING})
|
||||
st.update(fields)
|
||||
|
||||
# ---- 主循环 ----
|
||||
|
||||
async def run_job(self, record: JobRecord) -> None:
|
||||
"""逐阶段推进（成功/失败/暂停/等待确认）。"""
|
||||
try:
|
||||
record.status = JOB_RUNNING
|
||||
await self._persist_job(record)
|
||||
for idx, stage in enumerate(record.stage_chain):
|
||||
if record.pause_requested:
|
||||
await self._pause_job(record, stage)
|
||||
return
|
||||
if record.status == JOB_PAUSED:
|
||||
return
|
||||
outcome = await self._run_stage(record, stage, idx)
|
||||
if outcome == "paused":
|
||||
return
|
||||
if outcome == "failed":
|
||||
record.status = JOB_FAILED
|
||||
await self._persist_job(record)
|
||||
return
|
||||
if outcome == "await":
|
||||
# 等待确认/文件：任务进入 await_inputs（可删除；确认/补齐后续跑）
|
||||
record.status = JOB_AWAIT_INPUTS
|
||||
await self._persist_job(record)
|
||||
return
|
||||
# 全链完成
|
||||
if record.pause_requested:
|
||||
await self._pause_job(record, None)
|
||||
return
|
||||
record.status = JOB_DONE
|
||||
await self._persist_job(record)
|
||||
self.events.publish("job_done", {
|
||||
"job_id": record.id,
|
||||
"title": record.title,
|
||||
"progress": 100,
|
||||
"elapsed_sec": None,
|
||||
})
|
||||
except asyncio.CancelledError:
|
||||
# 任务级取消：不落脏状态（阶段取消由 _run_stage 单独处理）
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - 防御：任何异常不挂服务
|
||||
logger.exception("runner: 任务执行异常 %s", record.id)
|
||||
record.status = JOB_FAILED
|
||||
await self._persist_job(record)
|
||||
|
||||
async def _run_stage(self, record: JobRecord, stage: str, idx: int) -> str:
|
||||
"""执行单个阶段（状态机核心）。返回 "done"|"await"|"failed"|"paused"|"skipped"。"""
|
||||
status = self.stage_status(record, stage)
|
||||
if status == STAGE_DONE:
|
||||
return "done"
|
||||
if status == STAGE_SKIPPED:
|
||||
return "skipped"
|
||||
|
||||
# 0. confirm 已确认旁路：状态停留 waiting_confirm 但参数已被确认 → 直接执行
|
||||
st = record._stage_states.setdefault(stage, {})
|
||||
if status == STAGE_WAITING_CONFIRM and st.get("_confirmed"):
|
||||
pass # 跳过下方确认门
|
||||
else:
|
||||
# 1. 输入检查（FR-BIO-005：缺失→await_files；force_input_check=False 时跳过）
|
||||
if self.force_input_check:
|
||||
satisfied, missing = _stage_input_satisfied(stage, record.input_dir)
|
||||
if not satisfied:
|
||||
self._set_stage(record, stage, status=STAGE_AWAIT_FILES, error="")
|
||||
await self._persist_stage(record, stage)
|
||||
self.events.publish("stage_await_files", {
|
||||
"job_id": record.id, "stage": stage,
|
||||
"required_files": missing,
|
||||
"message": "请将数据文件拖入窗口上传至服务器，或在聊天窗口中指定数据文件的具体路径",
|
||||
})
|
||||
return "await"
|
||||
|
||||
# 2. 手动模式等待确认（FR-BIO-008）
|
||||
if not record.auto_continue and status != STAGE_QUEUED:
|
||||
self._set_stage(record, stage, status=STAGE_WAITING_CONFIRM)
|
||||
await self._persist_stage(record, stage)
|
||||
defaults = self._form_defaults(stage, record)
|
||||
self.events.publish("stage_await_confirm", {
|
||||
"job_id": record.id, "stage": stage,
|
||||
"form_defaults": defaults,
|
||||
})
|
||||
return "await"
|
||||
|
||||
# 3. 并发信号量（FR-BIO-031；槽满时内部置 queued 并阻塞排队）
|
||||
await self._acquire(record, stage)
|
||||
|
||||
# 排队期间收到暂停：释放槽，停在阶段边界（TC-BIO-API-122）
|
||||
if record.pause_requested:
|
||||
self.semaphore.release()
|
||||
self._set_stage(record, stage, status=STAGE_WAITING_CONFIRM if not record.auto_continue else STAGE_WAITING)
|
||||
await self._persist_stage(record, stage)
|
||||
await self._pause_job(record, stage)
|
||||
return "paused"
|
||||
|
||||
# 4. docker 执行
|
||||
try:
|
||||
params = record._stage_states.get(stage, {}).get("params") or {}
|
||||
attempt = int(record._stage_states.get(stage, {}).get("attempt") or 0) + 1
|
||||
self._set_stage(record, stage, status=STAGE_RUNNING, attempt=attempt,
|
||||
started_at=_now_iso(), error="")
|
||||
await self._persist_stage(record, stage)
|
||||
self.events.publish("stage_started", {
|
||||
"job_id": record.id, "stage": stage, "attempt": attempt,
|
||||
})
|
||||
result = await self.executor(stage, record.input_dir, record.workspace_dir,
|
||||
params, log_path=record.logs_dir / f"{stage}.log")
|
||||
log_tail = str(result.get("log_tail") or "")
|
||||
self._set_stage(record, stage, log_tail=log_tail)
|
||||
await self._persist_stage(record, stage)
|
||||
self.events.publish("stage_log", {
|
||||
"job_id": record.id, "stage": stage, "log": log_tail,
|
||||
})
|
||||
outcome = await self._judge_stage(record, stage, result)
|
||||
return outcome
|
||||
finally:
|
||||
self.semaphore.release()
|
||||
|
||||
async def _acquire(self, record: JobRecord, stage: str) -> bool:
|
||||
"""获取并发槽（FR-BIO-031）。
|
||||
|
||||
- 槽空闲：立即获取，返回 True。
|
||||
- 槽满：阶段置 queued（事件 stage_log 说明排队），随后阻塞等待
|
||||
asyncio.Semaphore 内建 FIFO 等待队列（先入队者先执行）。
|
||||
竞态：locked() 检查与 acquire() 之间可能被其他任务抢占，最坏情况是
|
||||
排队阶段未标记 queued 直接执行——状态正确性不受影响。
|
||||
"""
|
||||
if self.semaphore.locked():
|
||||
self._set_stage(record, stage, status=STAGE_QUEUED)
|
||||
await self._persist_stage(record, stage)
|
||||
self.events.publish("stage_log", {
|
||||
"job_id": record.id, "stage": stage,
|
||||
"log": "并发槽已满，阶段排队等待中…（queue）",
|
||||
})
|
||||
await self.semaphore.acquire()
|
||||
return True
|
||||
|
||||
async def _judge_stage(self, record: JobRecord, stage: str, result: dict[str, Any]) -> str:
|
||||
"""执行结果 + manifest 判成败（TC-BIO-RUN-012/014/015/016）。"""
|
||||
exit_code = result.get("exit_code")
|
||||
timed_out = bool(result.get("timed_out"))
|
||||
if timed_out:
|
||||
await self._fail_stage(record, stage, exit_code=exit_code, reason="timeout")
|
||||
return "failed"
|
||||
if exit_code not in (0, None):
|
||||
await self._fail_stage(record, stage, exit_code=exit_code, reason="exit_code")
|
||||
return "failed"
|
||||
|
||||
# 退出码 0：读 manifest 判成败
|
||||
prefix = self._prefix_for(record, stage)
|
||||
manifest_path = locate_manifest(record.workspace_dir, stage, prefix)
|
||||
if manifest_path is None:
|
||||
await self._fail_stage(record, stage, exit_code=exit_code, reason="no_manifest")
|
||||
return "failed"
|
||||
try:
|
||||
manifest = parse_manifest(record.workspace_dir, stage, prefix)
|
||||
except ManifestError as exc:
|
||||
# manifest 解析失败：退出码 0 且 JSON 损坏 → 失败（KERN-016）
|
||||
await self._fail_stage(record, stage, exit_code=exit_code, reason=f"manifest_error: {exc}")
|
||||
return "failed"
|
||||
if manifest.status == "failed":
|
||||
await self._fail_stage(record, stage, exit_code=exit_code, reason="manifest_failed")
|
||||
return "failed"
|
||||
|
||||
# 成功
|
||||
self._set_stage(record, stage, status=STAGE_DONE, manifest_path=manifest.path,
|
||||
finished_at=_now_iso())
|
||||
await self._persist_stage(record, stage)
|
||||
self.events.publish("stage_done", {
|
||||
"job_id": record.id, "stage": stage, "exit_code": exit_code,
|
||||
"manifest_ref": manifest.path,
|
||||
"metrics": manifest.metrics,
|
||||
})
|
||||
return "done"
|
||||
|
||||
async def _fail_stage(self, record: JobRecord, stage: str, *,
|
||||
exit_code: int | None, reason: str) -> None:
|
||||
self._set_stage(record, stage, status=STAGE_FAILED, finished_at=_now_iso(),
|
||||
error=reason)
|
||||
await self._persist_stage(record, stage)
|
||||
self.events.publish("stage_failed", {
|
||||
"job_id": record.id, "stage": stage,
|
||||
"exit_code": exit_code if exit_code is not None else -1,
|
||||
"reason": reason,
|
||||
})
|
||||
|
||||
def _prefix_for(self, record: JobRecord, stage: str) -> str | None:
|
||||
params = record._stage_states.get(stage, {}).get("params") or {}
|
||||
return str(params.get("prefix") or "") or None
|
||||
|
||||
def _form_defaults(self, stage: str, record: JobRecord) -> list[dict[str, Any]]:
|
||||
"""参数确认表单默认值（目录 schema + 任务参数/存档参数预填 + 来源标注）。"""
|
||||
entry = self.catalog.get(stage)
|
||||
schema = entry.params if entry else []
|
||||
prefill = record.params.get(stage) or {}
|
||||
st = record._stage_states.get(stage, {})
|
||||
existing = st.get("params") or {}
|
||||
merged = {p.get("name"): p for p in schema}
|
||||
defaults: list[dict[str, Any]] = []
|
||||
for p in schema:
|
||||
name = p.get("name")
|
||||
value = existing.get(name, prefill.get(name, p.get("default")))
|
||||
defaults.append({
|
||||
"name": name,
|
||||
"type": p.get("type"),
|
||||
"default": value,
|
||||
"source": p.get("source", "contract"),
|
||||
"help": p.get("help", ""),
|
||||
"choices": p.get("choices"),
|
||||
})
|
||||
return defaults
|
||||
|
||||
# ---- 控制操作（路由层调用）----
|
||||
|
||||
async def confirm_stage(self, record: JobRecord, stage: str, params: dict[str, Any]) -> None:
|
||||
"""阶段确认（FR-BIO-009）：校验参数→存表→执行（并发排队）。"""
|
||||
status = self.stage_status(record, stage)
|
||||
if status not in (STAGE_WAITING_CONFIRM, STAGE_QUEUED, STAGE_WAITING):
|
||||
raise RunnerError("STAGE_NOT_CONFIRMABLE", f"阶段 {stage} 当前状态 {status} 不可确认")
|
||||
allowed = self._allowed_param_names(stage)
|
||||
unknown = [k for k in params if k not in allowed]
|
||||
if unknown:
|
||||
raise RunnerError("UNKNOWN_PARAMS", f"未知参数: {', '.join(unknown)}")
|
||||
st = record._stage_states.setdefault(stage, {"status": STAGE_WAITING})
|
||||
st["params"] = {**st.get("params", {}), **params}
|
||||
st["_confirmed"] = True # 确认旁路标记（_run_stage 跳过确认门直接执行）
|
||||
await self._persist_stage(record, stage)
|
||||
record.pause_requested = False
|
||||
if record.status in (JOB_PAUSED, JOB_CREATED, JOB_FAILED, JOB_AWAIT_INPUTS):
|
||||
record.status = JOB_RUNNING
|
||||
await self._persist_job(record)
|
||||
self.spawn(self._run_confirmed_stage(record, stage))
|
||||
|
||||
async def _run_confirmed_stage(self, record: JobRecord, stage: str) -> None:
|
||||
"""确认后执行单个阶段并推进后续链（done→继续；failed→停）。"""
|
||||
try:
|
||||
outcome = await self._run_stage(record, stage, 0)
|
||||
if outcome == "failed":
|
||||
record.status = JOB_FAILED
|
||||
await self._persist_job(record)
|
||||
return
|
||||
if outcome == "await":
|
||||
return
|
||||
# 后续阶段继续推进
|
||||
idx = record.stage_chain.index(stage) if stage in record.stage_chain else 0
|
||||
for nxt in record.stage_chain[idx + 1:]:
|
||||
if record.pause_requested or record.status == JOB_PAUSED:
|
||||
return
|
||||
res = await self._run_stage(record, nxt, idx)
|
||||
if res in ("failed", "await", "paused"):
|
||||
if res == "failed":
|
||||
record.status = JOB_FAILED
|
||||
await self._persist_job(record)
|
||||
return
|
||||
if not record.pause_requested:
|
||||
record.status = JOB_DONE
|
||||
await self._persist_job(record)
|
||||
self.events.publish("job_done", {"job_id": record.id, "progress": 100})
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("runner: confirm 推进异常 %s", record.id)
|
||||
|
||||
def _allowed_param_names(self, stage: str) -> set[str]:
|
||||
entry = self.catalog.get(stage)
|
||||
if entry is None:
|
||||
return set()
|
||||
return {p.get("name") for p in entry.params if p.get("name")} | {"cpu", "memory", "env"}
|
||||
|
||||
async def rerun_stage(self, record: JobRecord, stage: str, params: dict[str, Any] | None) -> None:
|
||||
"""失败阶段重跑（FR-BIO-012）。带参数 → waiting_confirm 再确认；否则直接跑。"""
|
||||
status = self.stage_status(record, stage)
|
||||
if status != STAGE_FAILED:
|
||||
raise RunnerError("STAGE_NOT_FAILED", f"阶段 {stage} 当前状态 {status} 不可重跑")
|
||||
if params is None:
|
||||
params = {}
|
||||
st = record._stage_states.setdefault(stage, {})
|
||||
st["params"] = {**st.get("params", {}), **params}
|
||||
if params:
|
||||
st["status"] = STAGE_WAITING_CONFIRM
|
||||
await self._persist_stage(record, stage)
|
||||
defaults = self._form_defaults(stage, record)
|
||||
self.events.publish("stage_await_confirm", {
|
||||
"job_id": record.id, "stage": stage, "form_defaults": defaults,
|
||||
"rerun_with_params": True,
|
||||
})
|
||||
record.status = JOB_RUNNING
|
||||
await self._persist_job(record)
|
||||
return
|
||||
st["status"] = STAGE_WAITING
|
||||
await self._persist_stage(record, stage)
|
||||
record.status = JOB_RUNNING
|
||||
record.pause_requested = False
|
||||
await self._persist_job(record)
|
||||
self.spawn(self._run_confirmed_stage(record, stage))
|
||||
|
||||
async def skip_stage(self, record: JobRecord, stage: str) -> None:
|
||||
"""失败阶段跳过（FR-BIO-012）：置 skipped + 事件 + 后续阶段推进。"""
|
||||
status = self.stage_status(record, stage)
|
||||
if status != STAGE_FAILED:
|
||||
raise RunnerError("STAGE_NOT_FAILED", f"阶段 {stage} 当前状态 {status} 不可跳过")
|
||||
self._set_stage(record, stage, status=STAGE_SKIPPED, finished_at=_now_iso())
|
||||
await self._persist_stage(record, stage)
|
||||
self.events.publish("stage_skipped", {"job_id": record.id, "stage": stage})
|
||||
record.status = JOB_RUNNING
|
||||
record.pause_requested = False
|
||||
await self._persist_job(record)
|
||||
self.spawn(self._run_after_stage(record, stage))
|
||||
|
||||
async def _run_after_stage(self, record: JobRecord, stage: str) -> None:
|
||||
try:
|
||||
idx = record.stage_chain.index(stage) if stage in record.stage_chain else 0
|
||||
for nxt in record.stage_chain[idx + 1:]:
|
||||
if record.pause_requested or record.status == JOB_PAUSED:
|
||||
return
|
||||
res = await self._run_stage(record, nxt, idx)
|
||||
if res in ("failed", "await", "paused"):
|
||||
if res == "failed":
|
||||
record.status = JOB_FAILED
|
||||
await self._persist_job(record)
|
||||
return
|
||||
if not record.pause_requested and record.status != JOB_PAUSED:
|
||||
record.status = JOB_DONE
|
||||
await self._persist_job(record)
|
||||
self.events.publish("job_done", {"job_id": record.id, "progress": 100})
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("runner: skip 后推进异常 %s", record.id)
|
||||
|
||||
async def pause_job(self, record: JobRecord) -> None:
|
||||
"""暂停（FR-BIO-010）：等待当前阶段完成后停在阶段边界。"""
|
||||
if record.status not in (JOB_RUNNING, JOB_AWAIT_INPUTS):
|
||||
raise RunnerError("JOB_NOT_RUNNING", f"任务 {record.id} 当前状态 {record.status} 不可暂停")
|
||||
record.pause_requested = True
|
||||
await self._persist_job(record)
|
||||
# 无 running 阶段 → 立即暂停
|
||||
running = [s for s in record.stage_chain
|
||||
if self.stage_status(record, s) == STAGE_RUNNING]
|
||||
if not running:
|
||||
await self._pause_job(record, None)
|
||||
|
||||
async def _pause_job(self, record: JobRecord, current_stage: str | None) -> None:
|
||||
record.status = JOB_PAUSED
|
||||
record.pause_requested = False
|
||||
await self._persist_job(record)
|
||||
if current_stage:
|
||||
st = record._stage_states.get(current_stage)
|
||||
if st and st.get("status") not in (STAGE_DONE, STAGE_FAILED, STAGE_SKIPPED, STAGE_WAITING_CONFIRM):
|
||||
st["status"] = STAGE_PAUSED
|
||||
await self._persist_stage(record, current_stage)
|
||||
self.events.publish("job_paused", {
|
||||
"job_id": record.id,
|
||||
"current_stage": current_stage,
|
||||
})
|
||||
|
||||
async def resume_job(self, record: JobRecord) -> None:
|
||||
"""恢复（FR-BIO-010）：从下一阶段继续，不重跑已完成；确认点不跳过。"""
|
||||
if record.status != JOB_PAUSED:
|
||||
raise RunnerError("JOB_NOT_PAUSED", f"任务 {record.id} 当前状态 {record.status} 不可恢复")
|
||||
# paused 阶段复位为等待确认（不跳过确认，TC-BIO-API-069）
|
||||
for s, st in record._stage_states.items():
|
||||
if st.get("status") == STAGE_PAUSED:
|
||||
st["status"] = STAGE_WAITING_CONFIRM
|
||||
await self._persist_stage(record, s)
|
||||
record.status = JOB_RUNNING
|
||||
record.pause_requested = False
|
||||
await self._persist_job(record)
|
||||
self.spawn(self._resume_loop(record))
|
||||
|
||||
async def _resume_loop(self, record: JobRecord) -> None:
|
||||
try:
|
||||
for stage in record.stage_chain:
|
||||
if record.pause_requested or record.status == JOB_PAUSED:
|
||||
return
|
||||
status = self.stage_status(record, stage)
|
||||
if status in (STAGE_DONE, STAGE_SKIPPED):
|
||||
continue
|
||||
if status == STAGE_FAILED:
|
||||
record.status = JOB_FAILED
|
||||
await self._persist_job(record)
|
||||
return
|
||||
res = await self._run_stage(record, stage, 0)
|
||||
if res in ("failed", "await", "paused"):
|
||||
if res == "failed":
|
||||
record.status = JOB_FAILED
|
||||
await self._persist_job(record)
|
||||
return
|
||||
if not record.pause_requested and record.status != JOB_PAUSED:
|
||||
record.status = JOB_DONE
|
||||
await self._persist_job(record)
|
||||
self.events.publish("job_done", {"job_id": record.id, "progress": 100})
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("runner: resume 循环异常 %s", record.id)
|
||||
|
||||
# ---- 重启恢复（FR-BIO-031）----
|
||||
|
||||
async def recover(self) -> list[JobRecord]:
|
||||
"""服务重启恢复：扫 DB，running/created 任务从断点续跑。
|
||||
|
||||
- running 阶段：按 manifest 判成败（有 success manifest→done，否则重跑）。
|
||||
- done 阶段不重跑（attempt 不变，TC-BIO-API-111）。
|
||||
- 已归档任务跳过（TC-BIO-RUN-030）。
|
||||
"""
|
||||
from .store import load_running_jobs, load_job_stages
|
||||
rows = await load_running_jobs(self.db)
|
||||
recovered: list[JobRecord] = []
|
||||
for row in rows:
|
||||
stages = await load_job_stages(self.db, row["id"])
|
||||
record = JobRecord(
|
||||
id=row["id"], session_id=row["session_id"] or "",
|
||||
title=row["title"] or "生信任务",
|
||||
status=row["status"],
|
||||
stage_chain=stages_list(row["stage_chain"]),
|
||||
params=params_dict(row["params"]),
|
||||
auto_continue=bool(row["auto_continue"]),
|
||||
archive_id=row["archive_id"],
|
||||
)
|
||||
for s in stages:
|
||||
record._stage_states[s["stage"]] = {
|
||||
"status": s["status"], "params": params_dict(s["params"]),
|
||||
"attempt": int(s["attempt"] or 0),
|
||||
"manifest_path": s["manifest_path"], "log_tail": s["log_tail"] or "",
|
||||
"error": s["error"],
|
||||
}
|
||||
# running 阶段按 manifest 判成败（断点续跑不重跑已成功部分）
|
||||
for stage, st in record._stage_states.items():
|
||||
if st.get("status") == STAGE_RUNNING:
|
||||
prefix = self._prefix_for(record, stage)
|
||||
if locate_manifest(record.workspace_dir, stage, prefix):
|
||||
try:
|
||||
manifest = parse_manifest(record.workspace_dir, stage, prefix)
|
||||
if manifest.status == "success":
|
||||
st["status"] = STAGE_DONE
|
||||
st["manifest_path"] = manifest.path
|
||||
else:
|
||||
st["status"] = STAGE_FAILED
|
||||
st["error"] = "manifest_failed"
|
||||
except ManifestError:
|
||||
st["status"] = STAGE_FAILED
|
||||
st["error"] = "manifest_error"
|
||||
else:
|
||||
st["status"] = STAGE_WAITING
|
||||
await self._persist_stage(record, stage)
|
||||
self.events.publish("job_recovered", {
|
||||
"job_id": record.id, "status": record.status,
|
||||
})
|
||||
self.spawn(self._resume_loop(record))
|
||||
recovered.append(record)
|
||||
return recovered
|
||||
|
||||
|
||||
def stages_list(raw: Any) -> list[str]:
|
||||
if isinstance(raw, list):
|
||||
return [str(s) for s in raw]
|
||||
import json
|
||||
try:
|
||||
value = json.loads(raw or "[]")
|
||||
return [str(s) for s in value] if isinstance(value, list) else []
|
||||
except (ValueError, TypeError):
|
||||
return []
|
||||
|
||||
|
||||
def params_dict(raw: Any) -> dict[str, dict[str, Any]]:
|
||||
import json
|
||||
if isinstance(raw, dict):
|
||||
return raw
|
||||
try:
|
||||
value = json.loads(raw or "{}")
|
||||
return value if isinstance(value, dict) else {}
|
||||
except (ValueError, TypeError):
|
||||
return {}
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
from datetime import datetime, timezone
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
@@ -0,0 +1,238 @@
|
||||
"""app/services/bioflow/store.py —— 存储层（runner 与路由共用的三表 CRUD 辅助）。
|
||||
|
||||
全部通过 app.db.Database（aiosqlite）访问；JSON 列走 dumps_json/loads_json。
|
||||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from ...db import dumps_json, loads_json, now_iso
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---- jobs ----
|
||||
|
||||
async def upsert_job(db: Any, record: Any) -> None:
|
||||
"""任务行 upsert（JobRecord 字段 → bio_jobs）。"""
|
||||
progress = record.progress() if hasattr(record, "progress") else 0
|
||||
await db.execute(
|
||||
"""INSERT INTO bio_jobs (id, session_id, title, status, stage_chain, params,
|
||||
archive_id, auto_continue, progress, error, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
status=excluded.status, stage_chain=excluded.stage_chain, params=excluded.params,
|
||||
archive_id=excluded.archive_id, auto_continue=excluded.auto_continue,
|
||||
progress=excluded.progress, error=excluded.error, updated_at=excluded.updated_at""",
|
||||
(
|
||||
record.id, record.session_id, record.title, record.status,
|
||||
dumps_json(record.stage_chain), dumps_json(record.params),
|
||||
record.archive_id, int(bool(record.auto_continue)), progress,
|
||||
None, now_iso(), now_iso(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def update_job_status(db: Any, job_id: str, status: str, error: str | None = None) -> None:
|
||||
await db.execute(
|
||||
"UPDATE bio_jobs SET status=?, error=?, updated_at=? WHERE id=?",
|
||||
(status, error, now_iso(), job_id),
|
||||
)
|
||||
|
||||
|
||||
async def get_job(db: Any, job_id: str) -> dict[str, Any] | None:
|
||||
row = await db.fetchone("SELECT * FROM bio_jobs WHERE id=?", (job_id,))
|
||||
return _job_row(row) if row is not None else None
|
||||
|
||||
|
||||
async def list_jobs(
|
||||
db: Any,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
status: str | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
where: list[str] = []
|
||||
params: list[Any] = []
|
||||
if session_id:
|
||||
where.append("session_id=?")
|
||||
params.append(session_id)
|
||||
if status:
|
||||
where.append("status=?")
|
||||
params.append(status)
|
||||
clause = (" WHERE " + " AND ".join(where)) if where else ""
|
||||
total_row = await db.fetchone(f"SELECT COUNT(*) AS c FROM bio_jobs{clause}", params)
|
||||
total = int(total_row["c"]) if total_row is not None else 0
|
||||
rows = await db.fetchall(
|
||||
f"SELECT * FROM bio_jobs{clause} ORDER BY rowid DESC LIMIT ? OFFSET ?",
|
||||
[*params, page_size, (page - 1) * page_size],
|
||||
)
|
||||
return [_job_row(r) for r in rows], total
|
||||
|
||||
|
||||
async def delete_job(db: Any, job_id: str) -> None:
|
||||
await db.execute("DELETE FROM bio_jobs WHERE id=?", (job_id,))
|
||||
|
||||
|
||||
async def load_running_jobs(db: Any) -> list[dict[str, Any]]:
|
||||
"""重启恢复：running/await_inputs 任务（in-flight）。"""
|
||||
rows = await db.fetchall(
|
||||
"SELECT * FROM bio_jobs WHERE status IN ('running','await_inputs','created')"
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def _job_row(row: Any) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"session_id": row["session_id"] or "",
|
||||
"title": row["title"] or "",
|
||||
"status": row["status"] or "created",
|
||||
"stage_chain": loads_json(row["stage_chain"], []),
|
||||
"params": loads_json(row["params"], {}),
|
||||
"archive_id": row["archive_id"],
|
||||
"auto_continue": bool(row["auto_continue"]),
|
||||
"progress": int(row["progress"] or 0),
|
||||
"error": row["error"],
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
|
||||
|
||||
# ---- stages ----
|
||||
|
||||
async def upsert_stage(db: Any, record: Any, stage: str) -> None:
|
||||
"""阶段行 upsert（内存状态 → bio_stages）。"""
|
||||
st = record._stage_states.get(stage, {})
|
||||
await db.execute(
|
||||
"""INSERT INTO bio_stages (id, job_id, stage, status, params, manifest_path,
|
||||
log_tail, attempt, started_at, finished_at, error)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(job_id, stage) DO UPDATE SET
|
||||
status=excluded.status, params=excluded.params, manifest_path=excluded.manifest_path,
|
||||
log_tail=excluded.log_tail, attempt=excluded.attempt,
|
||||
started_at=excluded.started_at, finished_at=excluded.finished_at,
|
||||
error=excluded.error""",
|
||||
(
|
||||
f"{record.id}:{stage}", record.id, stage,
|
||||
st.get("status") or "waiting", dumps_json(st.get("params") or {}),
|
||||
st.get("manifest_path"), st.get("log_tail") or "",
|
||||
int(st.get("attempt") or 0), st.get("started_at"), st.get("finished_at"),
|
||||
st.get("error"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def get_stage(db: Any, job_id: str, stage: str) -> dict[str, Any] | None:
|
||||
row = await db.fetchone(
|
||||
"SELECT * FROM bio_stages WHERE job_id=? AND stage=?", (job_id, stage)
|
||||
)
|
||||
return _stage_row(row) if row is not None else None
|
||||
|
||||
|
||||
async def list_stages(db: Any, job_id: str) -> list[dict[str, Any]]:
|
||||
rows = await db.fetchall(
|
||||
"SELECT * FROM bio_stages WHERE job_id=? ORDER BY rowid", (job_id,)
|
||||
)
|
||||
return [_stage_row(r) for r in rows]
|
||||
|
||||
|
||||
async def load_job_stages(db: Any, job_id: str) -> list[dict[str, Any]]:
|
||||
return await list_stages(db, job_id)
|
||||
|
||||
|
||||
def _stage_row(row: Any) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"job_id": row["job_id"],
|
||||
"stage": row["stage"],
|
||||
"status": row["status"] or "waiting",
|
||||
"params": loads_json(row["params"], {}),
|
||||
"manifest_path": row["manifest_path"],
|
||||
"log_tail": row["log_tail"] or "",
|
||||
"attempt": int(row["attempt"] or 0),
|
||||
"started_at": row["started_at"],
|
||||
"finished_at": row["finished_at"],
|
||||
"error": row["error"],
|
||||
}
|
||||
|
||||
|
||||
# ---- archives ----
|
||||
|
||||
async def insert_archive(db: Any, archive: dict[str, Any]) -> None:
|
||||
await db.execute(
|
||||
"""INSERT INTO bio_archives (id, job_id, name, description, stage_chain, params,
|
||||
report_artifact, disk_path, size_bytes, reused_count, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
archive["id"], archive["job_id"], archive["name"], archive["description"],
|
||||
dumps_json(archive["stage_chain"]), dumps_json(archive["params"]),
|
||||
archive.get("report_artifact"), archive.get("disk_path"),
|
||||
int(archive.get("size_bytes") or 0), int(archive.get("reused_count") or 0),
|
||||
now_iso(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def get_archive(db: Any, archive_id: str) -> dict[str, Any] | None:
|
||||
row = await db.fetchone("SELECT * FROM bio_archives WHERE id=?", (archive_id,))
|
||||
return _archive_row(row) if row is not None else None
|
||||
|
||||
|
||||
async def find_archive_by_name(db: Any, name: str) -> dict[str, Any] | None:
|
||||
"""名称匹配复用（FR-BIO-023）：精确优先，模糊给候选。"""
|
||||
row = await db.fetchone("SELECT * FROM bio_archives WHERE name=? ORDER BY created_at DESC", (name,))
|
||||
if row is not None:
|
||||
return _archive_row(row)
|
||||
row = await db.fetchone(
|
||||
"SELECT * FROM bio_archives WHERE name LIKE ? ORDER BY created_at DESC",
|
||||
(f"%{name}%",),
|
||||
)
|
||||
return _archive_row(row) if row is not None else None
|
||||
|
||||
|
||||
async def find_archive_candidates(db: Any, name: str) -> list[dict[str, Any]]:
|
||||
rows = await db.fetchall(
|
||||
"SELECT * FROM bio_archives WHERE name LIKE ? ORDER BY created_at DESC LIMIT 10",
|
||||
(f"%{name}%",),
|
||||
)
|
||||
return [_archive_row(r) for r in rows]
|
||||
|
||||
|
||||
async def list_archives(db: Any, page: int = 1, page_size: int = 20) -> tuple[list[dict[str, Any]], int]:
|
||||
total_row = await db.fetchone("SELECT COUNT(*) AS c FROM bio_archives")
|
||||
total = int(total_row["c"]) if total_row is not None else 0
|
||||
rows = await db.fetchall(
|
||||
"SELECT * FROM bio_archives ORDER BY created_at DESC LIMIT ? OFFSET ?",
|
||||
(page_size, (page - 1) * page_size),
|
||||
)
|
||||
return [_archive_row(r) for r in rows], total
|
||||
|
||||
|
||||
async def delete_archive(db: Any, archive_id: str) -> None:
|
||||
await db.execute("DELETE FROM bio_archives WHERE id=?", (archive_id,))
|
||||
|
||||
|
||||
async def bump_archive_reuse(db: Any, archive_id: str) -> None:
|
||||
await db.execute(
|
||||
"UPDATE bio_archives SET reused_count=reused_count+1 WHERE id=?", (archive_id,)
|
||||
)
|
||||
|
||||
|
||||
def _archive_row(row: Any) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"job_id": row["job_id"],
|
||||
"name": row["name"] or "",
|
||||
"description": row["description"] or "",
|
||||
"stage_chain": loads_json(row["stage_chain"], []),
|
||||
"params": loads_json(row["params"], {}),
|
||||
"report_artifact": row["report_artifact"],
|
||||
"disk_path": row["disk_path"],
|
||||
"size_bytes": int(row["size_bytes"] or 0),
|
||||
"reused_count": int(row["reused_count"] or 0),
|
||||
"created_at": row["created_at"],
|
||||
}
|
||||
Executable
+117
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env bash
|
||||
# build.sh —— BIOFLOW 服务编译硬化(复制自 tooling/compile/compile_sdk.sh, 白名单改造)。
|
||||
# 白名单(明文):app/main.py(uvicorn 入口)、app/config.py(运维可读面)、
|
||||
# app/schemas.py(pydantic 模型——Cython 转 cyfunction 会触发 PydanticUserError,
|
||||
# 同 sdk 侧实测教训)、app/db.py(aiosqlite 协程取消热区——Cython 协程断流断言
|
||||
# abort 是 docgen 既有教训, 明文保留)+ 全部 __init__.py。
|
||||
# 用法:build.sh <source-root> <output-dir> [python]
|
||||
set -euo pipefail
|
||||
umask 027
|
||||
|
||||
SOURCE_ROOT="${1:?Usage: $0 <source-root> <output-dir> [python]}"
|
||||
OUTPUT_DIR="${2:?Usage: $0 <source-root> <output-dir> [python]}"
|
||||
PYTHON="${3:-python3}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
VISIBILITY_TOOL="${ELF_VISIBILITY_TOOL:-$SCRIPT_DIR/../tooling/compile/tools/elf_visibility.py}"
|
||||
|
||||
SOURCE_ROOT="$(realpath "$SOURCE_ROOT")"
|
||||
OUTPUT_DIR="$(realpath -m "$OUTPUT_DIR")"
|
||||
|
||||
[[ -d "$SOURCE_ROOT" ]] || { echo "Bioflow service source missing: $SOURCE_ROOT" >&2; exit 2; }
|
||||
[[ -x "$PYTHON" ]] || { echo "Python is not executable: $PYTHON" >&2; exit 2; }
|
||||
[[ -f "$VISIBILITY_TOOL" ]] || { echo "ELF visibility tool missing: $VISIBILITY_TOOL" >&2; exit 2; }
|
||||
for command_name in gcc strip rsync; do command -v "$command_name" >/dev/null; done
|
||||
|
||||
case "$OUTPUT_DIR" in
|
||||
"/"|"$(realpath -m "$HOME")"|"$SOURCE_ROOT")
|
||||
echo "Unsafe output directory: $OUTPUT_DIR" >&2; exit 2 ;;
|
||||
esac
|
||||
[[ ! -e "$OUTPUT_DIR" ]] || { echo "Output exists; refusing to overwrite: $OUTPUT_DIR" >&2; exit 2; }
|
||||
|
||||
echo "[BIOFLOW BUILD 1/5] Copying source (excluding tests)"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
rsync -a --exclude '__pycache__' --exclude 'test_*.py' --exclude 'data' --exclude '.env' \
|
||||
--exclude 'build.sh' --exclude 'restart.sh' --exclude 'pytest.ini' --exclude 'tests' \
|
||||
"$SOURCE_ROOT/" "$OUTPUT_DIR/"
|
||||
|
||||
# 白名单明文:main.py + config.py + schemas.py + db.py + __init__.py(编译前从发现集排除)
|
||||
# 白名单理由:main.py=uvicorn 入口;config.py=运维可读面;
|
||||
# schemas.py=pydantic v2 模型(validator 方法被 Cython 转 cyfunction 触发
|
||||
# PydanticUserError,实测编译后无法导入——纯数据契约面,无敏感逻辑,明文保留);
|
||||
# db.py=断流崩溃根治(同 sdk 侧 2026-09-03 教训):aiosqlite 协程取消热点,
|
||||
# Cython 协程 __Pyx_Coroutine_SendEx 断言会 abort 整个进程,明文走 CPython 原生协程
|
||||
WHITELIST_PY=(
|
||||
"app/main.py"
|
||||
"app/config.py"
|
||||
"app/schemas.py"
|
||||
"app/db.py"
|
||||
)
|
||||
_is_whitelisted() {
|
||||
local rel="$1"
|
||||
[[ "$(basename "$rel")" == "__init__.py" ]] && return 0
|
||||
for w in "${WHITELIST_PY[@]}"; do
|
||||
case "$rel" in
|
||||
$w) return 0 ;;
|
||||
esac
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "[BIOFLOW BUILD 2/5] Discovering Python modules"
|
||||
PY_FILES=()
|
||||
while IFS= read -r -d '' path; do PY_FILES+=("$path"); done \
|
||||
< <(find "$OUTPUT_DIR" -type f -name '*.py' -print0)
|
||||
FILTERED=()
|
||||
for py_file in "${PY_FILES[@]}"; do
|
||||
rel="${py_file#"$OUTPUT_DIR"/}"
|
||||
if _is_whitelisted "$rel"; then
|
||||
echo " [whitelist] $rel (明文保留)"
|
||||
continue
|
||||
fi
|
||||
FILTERED+=("$py_file")
|
||||
done
|
||||
PY_FILES=("${FILTERED[@]}")
|
||||
TOTAL="${#PY_FILES[@]}"
|
||||
(( TOTAL > 0 )) || { echo "No bioflow service modules selected" >&2; exit 3; }
|
||||
|
||||
INCLUDE_PY="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("INCLUDEPY") or "")')"
|
||||
EXT_SUFFIX="$("$PYTHON" -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX") or ".so")')"
|
||||
|
||||
echo "[BIOFLOW BUILD 3/5] Cython preflight ($TOTAL modules)"
|
||||
COUNT=0; FAILURES=0
|
||||
for py_file in "${PY_FILES[@]}"; do
|
||||
COUNT=$((COUNT+1))
|
||||
directory="$(dirname "$py_file")"
|
||||
module="$(basename "$py_file" .py)"
|
||||
c_file="$directory/$module.c"
|
||||
echo " [$COUNT/$TOTAL] ${py_file#"$OUTPUT_DIR"/}"
|
||||
if ! "$PYTHON" -m cython -3 \
|
||||
--directive always_allow_keywords=true,annotation_typing=False \
|
||||
--no-docstrings "$py_file" -o "$c_file" 2>/dev/null; then
|
||||
FAILURES=$((FAILURES+1))
|
||||
fi
|
||||
done
|
||||
(( FAILURES == 0 )) || { echo "Bioflow service Cython failed: $FAILURES" >&2; exit 3; }
|
||||
|
||||
echo "[BIOFLOW BUILD 4/5] GCC hidden visibility + strip"
|
||||
COUNT=0
|
||||
for py_file in "${PY_FILES[@]}"; do
|
||||
COUNT=$((COUNT+1))
|
||||
directory="$(dirname "$py_file")"
|
||||
module="$(basename "$py_file" .py)"
|
||||
c_file="$directory/$module.c"
|
||||
so_file="$directory/$module$EXT_SUFFIX"
|
||||
gcc -shared -pthread -fPIC -O2 -fvisibility=hidden \
|
||||
-I"$INCLUDE_PY" -o "$so_file" "$c_file"
|
||||
strip --strip-unneeded "$so_file"
|
||||
"$PYTHON" -B "$VISIBILITY_TOOL" --hide-non-pyinit --check "$so_file" >/dev/null
|
||||
rm -f -- "$c_file" "$py_file"
|
||||
done
|
||||
|
||||
echo "[BIOFLOW BUILD 5/5] Final checks"
|
||||
SO_COUNT="$(find "$OUTPUT_DIR" -type f -name '*.so' | wc -l)"
|
||||
[[ "$SO_COUNT" -eq "$TOTAL" ]] || { echo "count mismatch: $TOTAL vs $SO_COUNT" >&2; exit 4; }
|
||||
if find "$OUTPUT_DIR" -type f -name '*.c' -print -quit | grep -q .; then
|
||||
echo "Generated C source remains" >&2; exit 4
|
||||
fi
|
||||
echo "Bioflow service compile+harden complete: modules=$SO_COUNT output=$OUTPUT_DIR"
|
||||
@@ -0,0 +1,6 @@
|
||||
[pytest]
|
||||
python_files = test_*.py
|
||||
markers =
|
||||
bioflow_unit: 生信服务层单元测试（runner 状态机/kernel 拼装/manifest/DB 迁移/意图分流）
|
||||
bioflow_api: 生信服务端点测试（TestClient + mock docker/LLM/vstore）
|
||||
asyncio_mode = auto
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# restart.sh —— BIOFLOW 服务重启脚本（8098；不触碰生产服务 8090/8091）。
|
||||
# 零硬编码：venv/python/仓库路径全部从脚本位置推导或环境变量覆盖。
|
||||
# 用法: restart.sh [python] [port]
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
PYTHON="${1:-$REPO_ROOT/../plantai-rag-new-venv/bin/python}"
|
||||
PORT="${2:-8098}"
|
||||
HOST="${BIOFLOW_BIND_HOST:-0.0.0.0}"
|
||||
|
||||
[[ -x "$PYTHON" ]] || { echo "Python not found: $PYTHON" >&2; exit 2; }
|
||||
|
||||
APP_MODULE="app.main:app"
|
||||
PID=""
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
PID="$(lsof -ti tcp:"$PORT" 2>/dev/null || true)"
|
||||
fi
|
||||
|
||||
if [[ -n "$PID" ]]; then
|
||||
echo "[restart] 停止 8098 旧进程 pid=$PID"
|
||||
kill "$PID" 2>/dev/null || true
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
cd "$SCRIPT_DIR"
|
||||
echo "[restart] 启动生信服务 :$PORT（$PYTHON）"
|
||||
nohup "$PYTHON" -m uvicorn "$APP_MODULE" --host "$HOST" --port "$PORT" \
|
||||
> "$SCRIPT_DIR/bioflow.log" 2>&1 &
|
||||
echo "[restart] 完成；日志 bioflow_service/bioflow.log"
|
||||
@@ -0,0 +1,350 @@
|
||||
"""tests/conftest.py —— 生信服务测试共享夹具。
|
||||
|
||||
分层标记（TEST_PLAN §9-12 / TEST_CASES §6）：
|
||||
- bioflow_unit：服务层单元（runner 状态机/kernel 拼装/manifest/DB 迁移/意图分流）
|
||||
- bioflow_api：端点用例（TestClient + mock docker/LLM/vstore）
|
||||
|
||||
全部 mock docker/LLM/vstore，不依赖生产服务（8090/8091/8094/8098 绝不探测）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
# 测试运行 cwd = bioflow_service/（任务书自测命令）；repo 根在其上一级
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from app.config import Settings # noqa: E402
|
||||
from app.context import Context, build_context # noqa: E402
|
||||
from app.db import Database # noqa: E402
|
||||
from app.services.bioflow.catalog import StageCatalog, build_catalog # noqa: E402
|
||||
from app.services.bioflow.events import EventBus # noqa: E402
|
||||
from app.services.bioflow.kernel import ImageMap, Kernel, KernelConfig, UnknownStageError # noqa: E402
|
||||
from app.services.bioflow.runner import Runner # noqa: E402
|
||||
|
||||
|
||||
def _run(coro):
|
||||
"""同步跑一个协程（新建 event loop；docgen 测试同款可靠模式）。"""
|
||||
return asyncio.new_event_loop().run_until_complete(coro)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 静态映射表 fixture（不依赖真实 stage_image_map.json；测试确定性）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
STAGE_MAP_JSON = {
|
||||
"version": "2.0.0",
|
||||
"images": {
|
||||
"bioflow-gf": {"python": "3.11.14", "source_env": "test4py311", "docker_tag": "bioflow-gf"},
|
||||
"bioflow-rnaseq": {"python": "3.11.15", "source_env": "rnaseq", "docker_tag": "bioflow-rnaseq"},
|
||||
"bioflow-rnaR": {"python": "3.14.3", "source_env": "rnaR", "docker_tag": "bioflow-rnar",
|
||||
"auxiliary": True},
|
||||
},
|
||||
"stages": {
|
||||
"identify": {"image": "bioflow-gf", "module": "cli.run_identify", "executable": True},
|
||||
"genome_prepare": {"image": "bioflow-gf", "module": "cli.run_genome_prepare", "executable": True},
|
||||
"phylogeny": {"image": "bioflow-gf", "module": "cli.run_phylogeny", "executable": True},
|
||||
"motif": {"image": "bioflow-gf", "module": "cli.run_motif", "executable": True},
|
||||
"expression": {
|
||||
"image": "bioflow-rnaR", "module": "cli.run_expression", "executable": True,
|
||||
"note": "跨环境: 主 rnaR; rnaseq_quant 矩阵子步骤拆 bioflow-rnaseq",
|
||||
},
|
||||
"rnaseq_quant": {"image": "bioflow-rnaseq", "module": "cli.run_rnaseq_quant", "executable": True},
|
||||
"synteny_plot_revise": {"image": "bioflow-gf", "module": None, "executable": True,
|
||||
"note": "入口脚本缺失"},
|
||||
},
|
||||
"family_chain": ["genome_prepare", "identify", "phylogeny", "motif", "expression"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def image_map_path(tmp_path):
|
||||
p = tmp_path / "stage_image_map.json"
|
||||
p.write_text(json.dumps(STAGE_MAP_JSON), encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def image_map(image_map_path) -> ImageMap:
|
||||
return Kernel.load(image_map_path, KernelConfig()).map
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def catalog(image_map) -> StageCatalog:
|
||||
return build_catalog(image_map)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def kcfg(tmp_path) -> KernelConfig:
|
||||
return KernelConfig(
|
||||
host_uid=1234,
|
||||
docker_tag_suffix="dev",
|
||||
data_root=str(tmp_path / "data"),
|
||||
bba_env={"BBA_PFAM_DIR": "/bioflow-data/hmm"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def kernel(image_map, kcfg) -> Kernel:
|
||||
return Kernel(image_map, kcfg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DB（临时 SQLite；三表幂等迁移）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db(tmp_path):
|
||||
database = Database(tmp_path / "bioflow" / "bioflow.db")
|
||||
_run(database.connect())
|
||||
yield database
|
||||
_run(database.close())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# runner（mock docker 执行器）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FakeExecutor:
|
||||
"""mock docker：可编程退出码/超时/延迟/日志；记录全部调用。"""
|
||||
|
||||
def __init__(self, exit_code: int = 0, log: str = "", delay: float = 0.0,
|
||||
manifest_status: str | None = None) -> None:
|
||||
self.exit_code = exit_code
|
||||
self.log = log or "stage running log line\n"
|
||||
self.delay = delay
|
||||
self.manifest_status = manifest_status # None=用参数决定；"success"/"failed"/"missing"
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
self.running = 0
|
||||
self.max_running = 0
|
||||
|
||||
def _write_manifest(self, workspace, stage: str, status: str, prefix: str = "Ft.SPL") -> None:
|
||||
final = Path(workspace) / "final"
|
||||
final.mkdir(parents=True, exist_ok=True)
|
||||
manifest = {
|
||||
"manifest_version": "1", "job_id": "t", "stage": stage,
|
||||
"status": status, "prefix": prefix,
|
||||
"artifacts": {"final_members": {"artifact": "final_members",
|
||||
"full_path": str(final / f"{prefix}.identified_members.id"),
|
||||
"type": "table", "size": 10}},
|
||||
"parameters": {}, "metrics": {"final_members": 24},
|
||||
}
|
||||
(final / f"{prefix}.{stage}.manifest.json").write_text(
|
||||
json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
async def __call__(self, stage, input_dir, workspace, params=None,
|
||||
log_path=None, **kwargs):
|
||||
params = params or {}
|
||||
if self.delay:
|
||||
await asyncio.sleep(self.delay)
|
||||
self.running += 1
|
||||
self.max_running = max(self.max_running, self.running)
|
||||
try:
|
||||
log = str(self.log) if isinstance(self.log, str) else str(self.log)
|
||||
status = self.manifest_status
|
||||
if status is None:
|
||||
status = "success" if self.exit_code == 0 else "failed"
|
||||
if status != "missing" and self.exit_code == 0:
|
||||
self._write_manifest(workspace, stage, status)
|
||||
if log_path:
|
||||
Path(log_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(log_path).write_text(log, encoding="utf-8")
|
||||
cmd_hint = {"stage": stage, "cpu": params.get("cpu"), "memory": params.get("memory")}
|
||||
self.calls.append(cmd_hint)
|
||||
from app.services.bioflow.kernel import tail_lines
|
||||
return {"exit_code": self.exit_code, "timed_out": False,
|
||||
"log_tail": tail_lines(log, 80), "cmd": []}
|
||||
finally:
|
||||
self.running -= 1
|
||||
|
||||
|
||||
class FakeTimedOutExecutor(FakeExecutor):
|
||||
async def __call__(self, stage, input_dir, workspace, params=None,
|
||||
log_path=None, **kwargs):
|
||||
self.calls.append({"stage": stage, "timed_out": True})
|
||||
return {"exit_code": 137, "timed_out": True, "log_tail": "timeout killed\n", "cmd": []}
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def fake_executor() -> FakeExecutor:
|
||||
return FakeExecutor()
|
||||
|
||||
|
||||
def make_runner(db, events, executor, catalog, *, concurrency: int = 3,
|
||||
check_inputs: bool = False, **kwargs) -> Runner:
|
||||
"""Runner 工厂：默认 force_input_check=False（状态机用例不卡输入检查），
|
||||
需要输入检查的用例传 check_inputs=True（见 test_await_files_when_missing）。"""
|
||||
return Runner(
|
||||
db, events, executor, catalog, asyncio.Semaphore(concurrency),
|
||||
force_input_check=check_inputs, **kwargs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def events(tmp_path) -> EventBus:
|
||||
return EventBus(tmp_path / "events")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# context（端点用例用）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FakeFlowStore:
|
||||
"""mock vstore FlowStore（P6 接口形态）。"""
|
||||
|
||||
def __init__(self, items=None) -> None:
|
||||
self.items = items or []
|
||||
|
||||
def search(self, query, top_k=6, **kwargs):
|
||||
return {"mode": "hybrid", "items": self.items[:top_k]}
|
||||
|
||||
def get_chunk(self, chunk_key):
|
||||
for it in self.items:
|
||||
if str(it.get("chunk_key") or it.get("id")) == chunk_key:
|
||||
return {"meta": it, "text": it.get("text", "")}
|
||||
return None
|
||||
|
||||
|
||||
class FakeLLM:
|
||||
"""mock 规划 LLM：可编程候选/异常。"""
|
||||
|
||||
def __init__(self, candidates=None, error: str | None = None) -> None:
|
||||
self.candidates = candidates or [
|
||||
{"purpose": "SPL 家族标准鉴定流程", "stages": ["identify", "phylogeny", "motif"]},
|
||||
]
|
||||
self.error = error
|
||||
|
||||
async def __call__(self, system, user, **kwargs):
|
||||
if self.error == "timeout":
|
||||
from app.services.bioflow.llm import LLMTimeout
|
||||
raise LLMTimeout("timeout")
|
||||
if self.error == "unavailable":
|
||||
from app.services.bioflow.llm import LLMUnavailable
|
||||
raise LLMUnavailable("unavailable")
|
||||
if self.error == "bad":
|
||||
from app.services.bioflow.llm import LLMError
|
||||
raise LLMError("bad")
|
||||
return {"candidates": self.candidates}
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def test_settings(tmp_path, monkeypatch):
|
||||
"""测试 Settings（全部路径指向 tmp；零宿主路径）。"""
|
||||
s = Settings(
|
||||
_env_file=None,
|
||||
data_root=str(tmp_path / "data"),
|
||||
jobs_dir=str(tmp_path / "jobs"),
|
||||
db_path=str(tmp_path / "bioflow.db"),
|
||||
archives_dir=str(tmp_path / "archives"),
|
||||
image_map_path=str(tmp_path / "stage_image_map.json"),
|
||||
flow_docs_dir=str(tmp_path / "flow_docs"),
|
||||
concurrency=3,
|
||||
host_uid_env="BIOFLOW_TEST_UID",
|
||||
service_url="http://127.0.0.1:9999",
|
||||
)
|
||||
Path(s.image_map_abs).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(s.image_map_abs).write_text(json.dumps(STAGE_MAP_JSON), encoding="utf-8")
|
||||
for d in (s.data_root_abs, s.jobs_dir_abs, s.archives_dir_abs, s.flow_docs_dir_abs):
|
||||
Path(d).mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv("BIOFLOW_TEST_UID", "1234")
|
||||
|
||||
import app.config as config_mod
|
||||
monkeypatch.setattr(config_mod, "settings", s)
|
||||
monkeypatch.setattr(config_mod, "get_settings", lambda: s)
|
||||
|
||||
import app.context as context_mod
|
||||
monkeypatch.setattr(context_mod, "get_settings", lambda: s)
|
||||
monkeypatch.setattr(context_mod, "context", Context())
|
||||
return s
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolated_env(tmp_path, monkeypatch):
|
||||
"""每个用例隔离：关键环境键清空（不污染生产键名）且 Settings 默认全部指向 tmp。"""
|
||||
for key in ("BIOFLOW_DATA_ROOT", "BIOFLOW_JOBS_DIR", "BIOFLOW_DB_PATH",
|
||||
"BIOFLOW_ARCHIVES_DIR", "BIOFLOW_IMAGE_MAP", "BIOFLOW_FLOW_DOCS_DIR"):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
s = Settings(
|
||||
_env_file=None,
|
||||
data_root=str(tmp_path / "data"),
|
||||
jobs_dir=str(tmp_path / "jobs"),
|
||||
db_path=str(tmp_path / "bioflow.db"),
|
||||
archives_dir=str(tmp_path / "archives"),
|
||||
image_map_path=str(tmp_path / "stage_image_map.json"),
|
||||
flow_docs_dir=str(tmp_path / "flow_docs"),
|
||||
)
|
||||
import app.config as config_mod
|
||||
monkeypatch.setattr(config_mod, "settings", s)
|
||||
monkeypatch.setattr(config_mod, "get_settings", lambda: s)
|
||||
import app.context as context_mod
|
||||
monkeypatch.setattr(context_mod, "get_settings", lambda: s)
|
||||
# 各模块经 from ..config import get_settings 持有原始函数引用，须逐模块替换。
|
||||
# 仅替换模块顶层确有此名字的（kernel/runner 用 _default_kernel_config 惰性读 config）。
|
||||
import app.db as db_mod
|
||||
import app.routers.bioflow as r_flow
|
||||
import app.routers.biojobs as r_jobs
|
||||
import app.routers.bioarchives as r_arch
|
||||
for mod in (db_mod, r_flow, r_jobs, r_arch):
|
||||
if hasattr(mod, "get_settings"):
|
||||
monkeypatch.setattr(mod, "get_settings", lambda: s)
|
||||
return s
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def api_context(db, tmp_path, monkeypatch, image_map, fake_executor, fake_llm):
|
||||
"""端点用例 context：mock docker/LLM/vstore，注入到全部路由模块。
|
||||
|
||||
fake_executor/fake_llm 为模块级 fixture（可被用例覆盖行为后 re-patch）。
|
||||
"""
|
||||
events = EventBus(tmp_path / "jobs" / "events")
|
||||
catalog = build_catalog(image_map)
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
from app.services.bioflow.planner import Planner
|
||||
|
||||
flow_store = FakeFlowStore([
|
||||
{"chunk_key": "c1", "id": "c1", "source": "stage_contract",
|
||||
"doc_kind": "contract", "stage_name": "identify",
|
||||
"title": "Identify Stage Contract", "title_path": "Identify Stage Contract",
|
||||
"path": "gfa_core/schemas/stages/identify_contract_v1.md",
|
||||
"cite": {"anchor": "## 1", "line": 1}, "line_start": 1, "line_end": 10,
|
||||
"text": "Identify 需要全蛋白 FASTA 输入。"},
|
||||
])
|
||||
planner = Planner(catalog, flow_store, archive_finder=None)
|
||||
|
||||
async def find_archive(name):
|
||||
return None
|
||||
planner.archive_finder = find_archive
|
||||
|
||||
ctx = Context(db=db, events=events, kernel=None, catalog=catalog,
|
||||
runner=runner, semaphore=runner.semaphore, planner=planner,
|
||||
flow_store=flow_store, executor=fake_executor)
|
||||
|
||||
# 路由模块以 from ..context import context 绑定，须逐模块替换
|
||||
import app.routers.bioflow as r_flow
|
||||
import app.routers.biojobs as r_jobs
|
||||
import app.routers.bioarchives as r_arch
|
||||
monkeypatch.setattr(r_flow, "context", ctx)
|
||||
monkeypatch.setattr(r_jobs, "context", ctx)
|
||||
monkeypatch.setattr(r_arch, "context", ctx)
|
||||
import app.context as context_mod
|
||||
monkeypatch.setattr(context_mod, "context", ctx)
|
||||
|
||||
# 规划 LLM mock（complete_json → fake_llm）
|
||||
import app.services.bioflow.planner as planner_mod
|
||||
monkeypatch.setattr(planner_mod, "complete_json", fake_llm)
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def fake_llm():
|
||||
return FakeLLM()
|
||||
@@ -0,0 +1,760 @@
|
||||
"""tests/test_api.py —— 生信服务端点（L2；mock docker/LLM/vstore）。
|
||||
|
||||
分层标记 -m bioflow_api。覆盖（TC-BIO-API 系列）：
|
||||
health / chat/stream SSE（plan_ready 候选/伪造阶段拒/非生信不建任务/降级）/
|
||||
catalog / jobs 全套 / files（上传/路径登记/白名单）/ stages（confirm/rerun/skip/
|
||||
pause/resume/manifest+日志尾）/ events SSE（快照+回放）/ archives（存档/列表/
|
||||
复用/删除）/ 报告登记。
|
||||
|
||||
传输层说明：本环境 fastapi TestClient 的 sync stream 包装与 httpx ASGITransport
|
||||
对永不结束的 SSE 流均存在首帧不达/缓冲问题（dbg3/dbg6 最小复现实证）——
|
||||
SSE 测试端点直调迭代 body_iterator；普通 JSON 请求走 ASGITransport
|
||||
（每请求独立 AsyncClient + 新 loop，无跨用例状态纠缠）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.bioflow_api
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 传输辅助
|
||||
# ---------------------------------------------------------------------------
|
||||
# 架构说明：每个用例一个 ApiSession——独立事件循环 + 持久 AsyncClient（ASGITransport）。
|
||||
# asyncio.set_event_loop 保证端点内 asyncio.create_task 派生的后台任务（runner 自动
|
||||
# 推进）绑定到本 loop；_wait_until 轮询经同一 client 泵 loop，后台任务随之推进。
|
||||
# 规避 TestClient sync stream 与 ASGITransport 对永不结束 SSE 流的缓冲问题
|
||||
# （dbg3/dbg6 最小复现实证），SSE 用例端点直调迭代 body_iterator（见模块 docstring）。
|
||||
|
||||
|
||||
def _run_async(coro):
|
||||
"""在共享会话 loop 上跑协程（anyio portal 绑定首个 loop，跨 loop 会死锁）。"""
|
||||
return ApiSession.shared().loop.run_until_complete(coro)
|
||||
|
||||
|
||||
class ApiSession:
|
||||
"""模块级单例会话：全模块共享一个 event loop 与 AsyncClient。
|
||||
|
||||
原因：anyio asyncio 后端缓存线程首个 event loop（portal），跨测试新建
|
||||
loop 会导致 run_until_complete 死锁（实证：test_create_job 后续测试随机
|
||||
hang）。单 loop 共享 + 每用例后 drain 后台任务，是确定性方案。
|
||||
"""
|
||||
|
||||
_instance = None
|
||||
|
||||
def __init__(self) -> None:
|
||||
import httpx
|
||||
|
||||
from app.main import app as _app
|
||||
|
||||
self.loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self.loop)
|
||||
transport = httpx.ASGITransport(app=_app)
|
||||
self._client = httpx.AsyncClient(transport=transport, base_url="http://t")
|
||||
|
||||
@classmethod
|
||||
def shared(cls) -> "ApiSession":
|
||||
if cls._instance is None:
|
||||
cls._instance = cls()
|
||||
asyncio.set_event_loop(cls._instance.loop)
|
||||
return cls._instance
|
||||
|
||||
def request(self, method: str, url: str, **kw) -> "_Resp":
|
||||
resp = self.loop.run_until_complete(self._client.request(method, url, **kw))
|
||||
return _Resp(resp.status_code, resp.content, dict(resp.headers))
|
||||
|
||||
def get(self, url, **kw):
|
||||
return self.request("GET", url, **kw)
|
||||
|
||||
def post(self, url, **kw):
|
||||
return self.request("POST", url, **kw)
|
||||
|
||||
def delete(self, url, **kw):
|
||||
return self.request("DELETE", url, **kw)
|
||||
|
||||
def drain(self) -> None:
|
||||
"""取消全部遗留后台任务（上用例 runner 派生），泵干 loop。"""
|
||||
|
||||
import app.context as context_mod
|
||||
|
||||
async def _cancel():
|
||||
ctx = context_mod.context
|
||||
targets = list(ctx._tasks)
|
||||
if ctx.runner is not None:
|
||||
targets += list(ctx.runner._tasks)
|
||||
for t in targets:
|
||||
t.cancel()
|
||||
await asyncio.sleep(0.05) # 给取消任务 finally（信号量释放）时间
|
||||
return True
|
||||
|
||||
try:
|
||||
self.loop.run_until_complete(asyncio.wait_for(_cancel(), timeout=3))
|
||||
except (asyncio.TimeoutError, RuntimeError):
|
||||
pass
|
||||
|
||||
def close(self) -> None:
|
||||
self.drain()
|
||||
self.loop.run_until_complete(self._client.aclose())
|
||||
self.loop.close()
|
||||
asyncio.set_event_loop(None)
|
||||
ApiSession._instance = None
|
||||
|
||||
|
||||
class _Resp:
|
||||
"""httpx 响应最小封装（status_code/json/text）。"""
|
||||
|
||||
def __init__(self, status_code: int, content: bytes, headers=None) -> None:
|
||||
self.status_code = status_code
|
||||
self.content = content
|
||||
self.headers = headers or {}
|
||||
|
||||
def json(self):
|
||||
return json.loads(self.content.decode("utf-8"))
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def api(api_context):
|
||||
"""API 会话（模块级共享 loop + client；用例后取消遗留后台任务）。"""
|
||||
session = ApiSession.shared()
|
||||
yield session
|
||||
session.drain()
|
||||
|
||||
|
||||
def _wait_until(fn, timeout=5.0, interval=0.02):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
value = fn()
|
||||
if value:
|
||||
return value
|
||||
time.sleep(interval)
|
||||
return fn()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SSE 辅助（端点直调；传输层限制见模块 docstring）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_sse_frame(frame_text: str) -> dict | None:
|
||||
"""SSE 帧解析（event/data 两行一组；注释帧返回 None）。"""
|
||||
lines = frame_text.split("\n")
|
||||
event_type = None
|
||||
data = None
|
||||
for line in lines:
|
||||
if line.startswith("event: "):
|
||||
event_type = line[7:]
|
||||
elif line.startswith("data: "):
|
||||
data = line[6:]
|
||||
if event_type and data:
|
||||
try:
|
||||
return {"type": event_type, "data": json.loads(data)}
|
||||
except ValueError:
|
||||
return {"type": event_type, "data": data}
|
||||
if event_type:
|
||||
return {"type": event_type, "data": None}
|
||||
return None
|
||||
|
||||
|
||||
async def _collect_endpoint_stream(resp, max_events: int = 10, deadline_sec: float = 5.0):
|
||||
"""迭代端点返回的 StreamingResponse.body_iterator（限时收集）。"""
|
||||
events: list[dict] = []
|
||||
buf_holder = [""]
|
||||
|
||||
async def _iter():
|
||||
async for frame in resp.body_iterator:
|
||||
chunk = frame if isinstance(frame, (bytes, bytearray)) else str(frame).encode("utf-8")
|
||||
buf_holder[0] += chunk.decode("utf-8")
|
||||
while "\n\n" in buf_holder[0]:
|
||||
frame_text, buf_holder[0] = buf_holder[0].split("\n\n", 1)
|
||||
parsed = _parse_sse_frame(frame_text)
|
||||
if parsed:
|
||||
events.append(parsed)
|
||||
if len(events) >= max_events:
|
||||
return events
|
||||
return events
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(_iter(), timeout=deadline_sec)
|
||||
except asyncio.TimeoutError:
|
||||
return events
|
||||
|
||||
|
||||
def _read_sse(endpoint_coro, max_events=10, deadline_sec=5.0):
|
||||
"""端点直调版 SSE 读取。"""
|
||||
|
||||
async def go():
|
||||
resp = await endpoint_coro
|
||||
assert resp.status_code == 200
|
||||
assert resp.media_type == "text/event-stream"
|
||||
return await _collect_endpoint_stream(resp, max_events, deadline_sec)
|
||||
|
||||
return _run_async(go())
|
||||
|
||||
|
||||
def _read_sse_status(endpoint_coro):
|
||||
"""端点直调：只取状态码（404 等）。"""
|
||||
|
||||
async def go():
|
||||
from fastapi import HTTPException
|
||||
|
||||
try:
|
||||
resp = await endpoint_coro
|
||||
return resp.status_code
|
||||
except HTTPException as exc:
|
||||
return exc.status_code
|
||||
|
||||
return _run_async(go())
|
||||
|
||||
|
||||
def _chat_stream_call(message, **kw):
|
||||
from app.routers import bioflow as r_flow
|
||||
from app.schemas import ChatStreamRequest
|
||||
|
||||
body = ChatStreamRequest(session_id=kw.get("session_id", "s1"), message=message, **{
|
||||
k: v for k, v in kw.items() if k != "session_id"})
|
||||
return r_flow.chat_stream(body)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# health
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_health(api):
|
||||
"""TC-BIO-API-099：health 200 + db 连通状态。"""
|
||||
resp = api.get("/api/bioflow/health")
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["ok"] is True
|
||||
assert body["status"] == "ok"
|
||||
assert body["db"]["connected"] is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# catalog
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_catalog_all(api):
|
||||
"""TC-BIO-API-021：catalog 返回全部阶段（映射表驱动，每项含核心字段）。"""
|
||||
resp = api.get("/api/bioflow/catalog")
|
||||
assert resp.status_code == 200
|
||||
stages = resp.json()["data"]["stages"]
|
||||
assert len(stages) >= 7 # 测试映射表 7 阶段（真实表 75）
|
||||
item = [s for s in stages if s["name"] == "identify"][0]
|
||||
assert {"name", "description", "image", "params", "inputs", "outputs"} <= set(item.keys())
|
||||
|
||||
|
||||
def test_catalog_chain(api):
|
||||
"""TC-BIO-API-022：基因家族链。"""
|
||||
resp = api.get("/api/bioflow/catalog", params={"chain": "gene_family"})
|
||||
assert resp.status_code == 200
|
||||
stages = resp.json()["data"]["stages"]
|
||||
assert [s["name"] for s in stages][:2] == ["genome_prepare", "identify"]
|
||||
|
||||
|
||||
def test_catalog_stage_detail(api):
|
||||
"""TC-BIO-API-023：单阶段参数 schema。"""
|
||||
resp = api.get("/api/bioflow/catalog", params={"stage": "identify"})
|
||||
assert resp.status_code == 200
|
||||
stage = resp.json()["data"]["stage"]
|
||||
for p in stage["params"]:
|
||||
assert {"name", "type", "default", "source", "help"} <= set(p.keys())
|
||||
assert stage["image"] == "bioflow-gf"
|
||||
assert stage["description"] == "基因家族鉴定"
|
||||
|
||||
|
||||
def test_catalog_unknown_stage(api):
|
||||
"""TC-BIO-API-024：未知阶段 404。"""
|
||||
resp = api.get("/api/bioflow/catalog", params={"stage": "nope"})
|
||||
assert resp.status_code == 404
|
||||
assert resp.json()["error"]["code"] == "NOT_FOUND"
|
||||
|
||||
|
||||
def test_catalog_unknown_chain(api):
|
||||
resp = api.get("/api/bioflow/catalog", params={"chain": "nope"})
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# chat/stream
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_chat_stream_plan_ready(api_context):
|
||||
"""TC-BIO-API-001/002：chat/stream SSE，plan_ready 下发候选。"""
|
||||
events = _read_sse(_chat_stream_call("帮我跑水稻 SPL 家族鉴定"), max_events=1)
|
||||
types = [e["type"] for e in events]
|
||||
assert "plan_ready" in types
|
||||
plan = [e for e in events if e["type"] == "plan_ready"][0]["data"]
|
||||
assert 1 <= len(plan["candidates"]) <= 3
|
||||
for c in plan["candidates"]:
|
||||
assert c["stages"] and c["purpose"]
|
||||
|
||||
|
||||
def test_chat_stream_unknown_stage_rejected(api_context, monkeypatch):
|
||||
"""TC-BIO-API-003：伪造阶段被拒（候选不进候选集）。"""
|
||||
from conftest import FakeLLM
|
||||
import app.services.bioflow.planner as planner_mod
|
||||
monkeypatch.setattr(planner_mod, "complete_json", FakeLLM(candidates=[
|
||||
{"purpose": "坏", "stages": ["fake_stage_xyz"]},
|
||||
{"purpose": "好", "stages": ["identify"]},
|
||||
]))
|
||||
events = _read_sse(_chat_stream_call("帮我跑水稻 SPL 家族鉴定"), max_events=1)
|
||||
plan = [e for e in events if e["type"] == "plan_ready"][0]["data"]
|
||||
for c in plan["candidates"]:
|
||||
assert "fake_stage_xyz" not in c["stages"]
|
||||
|
||||
|
||||
def test_chat_stream_non_bioflow_message(api_context):
|
||||
"""TC-BIO-API-005：非生信消息不建任务。"""
|
||||
events = _read_sse(_chat_stream_call("什么是基因家族"), max_events=1)
|
||||
types = [e["type"] for e in events]
|
||||
assert "message_done" in types
|
||||
assert "job_created" not in types
|
||||
done = [e for e in events if e["type"] == "message_done"][0]["data"]
|
||||
assert done["kind"] == "qa"
|
||||
|
||||
|
||||
def test_chat_stream_empty_message_422(api):
|
||||
"""TC-BIO-API-006/007/008：422 校验。"""
|
||||
assert api.post("/api/bioflow/chat/stream",
|
||||
json={"session_id": "s1", "message": ""}).status_code == 422
|
||||
assert api.post("/api/bioflow/chat/stream",
|
||||
json={"session_id": "s1"}).status_code == 422
|
||||
assert api.post("/api/bioflow/chat/stream",
|
||||
json={"message": "x"}).status_code == 422
|
||||
|
||||
|
||||
def test_chat_stream_overlong_rejected(api):
|
||||
"""TC-BIO-API-015：超长消息拒绝，不触发 LLM。"""
|
||||
resp = api.post("/api/bioflow/chat/stream",
|
||||
json={"session_id": "s1", "message": "x" * 100001})
|
||||
assert resp.status_code == 422
|
||||
|
||||
|
||||
def test_chat_stream_llm_timeout_degrade(api_context, monkeypatch):
|
||||
"""TC-BIO-API-009：LLM 超时降级（内置兜底候选不中断 SSE）。"""
|
||||
from conftest import FakeLLM
|
||||
import app.services.bioflow.planner as planner_mod
|
||||
monkeypatch.setattr(planner_mod, "complete_json", FakeLLM(error="timeout"))
|
||||
events = _read_sse(_chat_stream_call("帮我跑水稻 SPL 家族鉴定"), max_events=1)
|
||||
plan = [e for e in events if e["type"] == "plan_ready"][0]["data"]
|
||||
assert plan["fallback"] # 降级提示
|
||||
assert plan["candidates"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# jobs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _create_job(api, stages=("identify",), auto_continue=False, **kw):
|
||||
resp = api.post("/api/bioflow/jobs", json={
|
||||
"session_id": kw.get("session_id", "s1"),
|
||||
"title": kw.get("title", "t"),
|
||||
"stages": list(stages),
|
||||
"auto_continue": auto_continue,
|
||||
})
|
||||
return resp
|
||||
|
||||
|
||||
def _job_id(resp):
|
||||
return resp.json()["data"]["id"]
|
||||
|
||||
|
||||
def _job_stage_status(api, job_id: str, idx: int = 0):
|
||||
return api.get(f"/api/bioflow/jobs/{job_id}").json()["data"]["stages"][idx]["status"]
|
||||
|
||||
|
||||
def test_create_job(api):
|
||||
"""TC-BIO-API-027：POST jobs 创建任务（stages 校验 + 返回结构）。"""
|
||||
resp = _create_job(api, title="SPL 鉴定")
|
||||
assert resp.status_code == 201
|
||||
data = resp.json()["data"]
|
||||
assert data["id"]
|
||||
assert data["title"] == "SPL 鉴定"
|
||||
assert data["stages"][0]["stage"] == "identify"
|
||||
assert data["status"] in ("created", "running", "done")
|
||||
|
||||
|
||||
def test_create_job_missing_stages(api):
|
||||
"""TC-BIO-API-028：缺 stages 422。"""
|
||||
assert api.post("/api/bioflow/jobs",
|
||||
json={"session_id": "s1", "title": "t"}).status_code == 422
|
||||
|
||||
|
||||
def test_create_job_unknown_stage(api):
|
||||
"""TC-BIO-API-030：未知 stage 拒绝。"""
|
||||
resp = api.post("/api/bioflow/jobs", json={
|
||||
"session_id": "s1", "title": "t", "stages": ["ghost_stage"],
|
||||
})
|
||||
assert resp.status_code == 422
|
||||
assert "ghost_stage" in json.dumps(resp.json(), ensure_ascii=False)
|
||||
|
||||
|
||||
def test_jobs_list_and_filter(api):
|
||||
"""TC-BIO-API-032：列表按创建顺序倒序（rowid DESC）。"""
|
||||
_create_job(api, title="t1")
|
||||
_create_job(api, title="t2")
|
||||
resp = api.get("/api/bioflow/jobs")
|
||||
assert resp.status_code == 200
|
||||
items = resp.json()["data"]["items"]
|
||||
assert len(items) >= 2
|
||||
assert items[0]["title"] == "t2" # 倒序
|
||||
|
||||
|
||||
def test_job_detail_and_404(api):
|
||||
resp = _create_job(api, stages=["identify", "phylogeny"])
|
||||
job_id = _job_id(resp)
|
||||
detail = api.get(f"/api/bioflow/jobs/{job_id}")
|
||||
assert detail.status_code == 200
|
||||
assert len(detail.json()["data"]["stages"]) == 2
|
||||
assert "progress" in detail.json()["data"]
|
||||
not_found = api.get("/api/bioflow/jobs/not-exist")
|
||||
assert not_found.status_code == 404
|
||||
assert not_found.json()["error"]["code"] == "NOT_FOUND"
|
||||
|
||||
|
||||
def test_delete_job(api):
|
||||
resp = _create_job(api)
|
||||
job_id = _job_id(resp)
|
||||
# 等待任务终态（done；删除只拒 running）
|
||||
def terminal():
|
||||
return api.get(f"/api/bioflow/jobs/{job_id}").json()["data"]["status"] in ("done", "failed")
|
||||
_wait_until(terminal, timeout=5)
|
||||
del_resp = api.delete(f"/api/bioflow/jobs/{job_id}")
|
||||
assert del_resp.status_code == 204
|
||||
assert api.get(f"/api/bioflow/jobs/{job_id}").status_code == 404
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# files
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_files_upload_and_list(api):
|
||||
"""TC-BIO-API-039/048：上传 + 文件列表。"""
|
||||
resp = _create_job(api)
|
||||
job_id = _job_id(resp)
|
||||
content = b">gene1\nACGTACGT\n"
|
||||
up = api.post(f"/api/bioflow/jobs/{job_id}/files",
|
||||
files={"file": ("test.fa", content, "application/octet-stream")})
|
||||
assert up.status_code == 200
|
||||
data = up.json()["data"]
|
||||
assert data["name"] == "test.fa"
|
||||
assert data["size"] == len(content)
|
||||
listing = api.get(f"/api/bioflow/jobs/{job_id}/files")
|
||||
assert listing.status_code == 200
|
||||
assert listing.json()["data"]["items"][0]["name"] == "test.fa"
|
||||
|
||||
|
||||
def test_files_upload_bad_extension(api):
|
||||
"""TC-BIO-API-046：扩展名白名单外 422。"""
|
||||
resp = _create_job(api)
|
||||
job_id = _job_id(resp)
|
||||
up = api.post(f"/api/bioflow/jobs/{job_id}/files",
|
||||
files={"file": ("x.exe", b"MZ", "application/octet-stream")})
|
||||
assert up.status_code == 422
|
||||
|
||||
|
||||
def test_files_upload_no_job(api):
|
||||
"""TC-BIO-API-120：上传无任务 404。"""
|
||||
up = api.post("/api/bioflow/jobs/nope/files",
|
||||
files={"file": ("a.fa", b"x", "text/plain")})
|
||||
assert up.status_code == 404
|
||||
|
||||
|
||||
def test_path_register_valid_and_whitelist(api, tmp_path):
|
||||
"""TC-BIO-API-042/045：路径登记合法 + 白名单外 403。"""
|
||||
resp = _create_job(api)
|
||||
job_id = _job_id(resp)
|
||||
# api_context 的 settings 已指向 tmp_path/data（conftest _isolated_env 派生）
|
||||
from app.config import get_settings as _gs
|
||||
data_root = _gs().data_root_abs
|
||||
import os
|
||||
allowed = os.path.join(data_root, "examples", "SPL.fa")
|
||||
os.makedirs(os.path.dirname(allowed), exist_ok=True)
|
||||
with open(allowed, "w") as f:
|
||||
f.write(">x\nACGT")
|
||||
ok = api.post(f"/api/bioflow/jobs/{job_id}/files", json={"server_path": allowed})
|
||||
assert ok.status_code == 200
|
||||
assert ok.json()["data"]["source"] == "server_path"
|
||||
# 白名单外
|
||||
outside = str(tmp_path / "outside.fa")
|
||||
with open(outside, "w") as f:
|
||||
f.write("x")
|
||||
forbidden = api.post(f"/api/bioflow/jobs/{job_id}/files", json={"server_path": outside})
|
||||
assert forbidden.status_code == 403
|
||||
assert forbidden.json()["error"]["code"] == "PATH_NOT_ALLOWED"
|
||||
# 不存在
|
||||
missing = api.post(f"/api/bioflow/jobs/{job_id}/files",
|
||||
json={"server_path": str(tmp_path / "nope.fa")})
|
||||
assert missing.status_code == 400
|
||||
assert missing.json()["error"]["code"] == "PATH_NOT_EXIST"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# stages 控制 + manifest
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_confirm_flow(api):
|
||||
"""TC-BIO-API-056/057：confirm 发起执行，参数覆盖。"""
|
||||
resp = _create_job(api)
|
||||
job_id = _job_id(resp)
|
||||
|
||||
def awaiting():
|
||||
return _job_stage_status(api, job_id) in ("waiting_confirm", "done")
|
||||
_wait_until(awaiting, timeout=5)
|
||||
|
||||
confirm = api.post(f"/api/bioflow/jobs/{job_id}/stages/identify/confirm",
|
||||
json={"params": {"cpu": "8"}})
|
||||
assert confirm.status_code == 200
|
||||
assert confirm.json()["data"]["status"] in ("running", "done")
|
||||
|
||||
def done():
|
||||
return _job_stage_status(api, job_id) == "done"
|
||||
_wait_until(done, timeout=5)
|
||||
assert _job_stage_status(api, job_id) == "done"
|
||||
|
||||
|
||||
def test_confirm_unknown_param(api):
|
||||
"""TC-BIO-API-058：拒绝非 schema 参数。"""
|
||||
resp = _create_job(api)
|
||||
job_id = _job_id(resp)
|
||||
|
||||
def waiting():
|
||||
return _job_stage_status(api, job_id) in ("waiting_confirm", "done")
|
||||
_wait_until(waiting, timeout=5)
|
||||
confirm = api.post(f"/api/bioflow/jobs/{job_id}/stages/identify/confirm",
|
||||
json={"params": {"magic": "x"}})
|
||||
# waiting_confirm 时 422；已 done 时 409（两态均可——状态竞争）
|
||||
assert confirm.status_code in (409, 422)
|
||||
|
||||
|
||||
def test_confirm_unknown_stage_404(api):
|
||||
resp = _create_job(api)
|
||||
job_id = _job_id(resp)
|
||||
confirm = api.post(f"/api/bioflow/jobs/{job_id}/stages/nope/confirm", json={})
|
||||
assert confirm.status_code == 404
|
||||
|
||||
|
||||
def test_manifest_with_log_tail(api):
|
||||
"""TC-BIO-API-071：manifest + 日志尾 80 行。"""
|
||||
resp = _create_job(api, auto_continue=True)
|
||||
job_id = _job_id(resp)
|
||||
|
||||
def done():
|
||||
return _job_stage_status(api, job_id) == "done"
|
||||
_wait_until(done, timeout=5)
|
||||
m = api.get(f"/api/bioflow/jobs/{job_id}/stages/identify/manifest")
|
||||
assert m.status_code == 200
|
||||
data = m.json()["data"]
|
||||
assert data["manifest"]["status"] == "success"
|
||||
assert "log_tail" in data
|
||||
|
||||
|
||||
def test_manifest_unknown_stage(api):
|
||||
resp = _create_job(api)
|
||||
job_id = _job_id(resp)
|
||||
m = api.get(f"/api/bioflow/jobs/{job_id}/stages/nope/manifest")
|
||||
assert m.status_code == 404
|
||||
|
||||
|
||||
def test_rerun_and_skip_on_failed(api, api_context):
|
||||
"""TC-BIO-API-062/065：rerun/skip 失败阶段。"""
|
||||
api_context.executor.exit_code = 1
|
||||
resp = _create_job(api, auto_continue=True)
|
||||
job_id = _job_id(resp)
|
||||
|
||||
def failed():
|
||||
return _job_stage_status(api, job_id) == "failed"
|
||||
_wait_until(failed, timeout=5)
|
||||
# skip
|
||||
skip = api.post(f"/api/bioflow/jobs/{job_id}/stages/identify/skip")
|
||||
assert skip.status_code == 200
|
||||
assert skip.json()["data"]["status"] == "skipped"
|
||||
|
||||
|
||||
def test_rerun_non_failed_409(api):
|
||||
"""TC-BIO-API-064：done 阶段 rerun 409。"""
|
||||
resp = _create_job(api, auto_continue=True)
|
||||
job_id = _job_id(resp)
|
||||
|
||||
def done():
|
||||
return _job_stage_status(api, job_id) == "done"
|
||||
_wait_until(done, timeout=5)
|
||||
rerun = api.post(f"/api/bioflow/jobs/{job_id}/stages/identify/rerun", json={})
|
||||
assert rerun.status_code == 409
|
||||
assert rerun.json()["error"]["code"] == "STAGE_NOT_FAILED"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# events SSE（快照 + 回放）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_events_snapshot_and_replay(api):
|
||||
"""TC-BIO-API-017/078：events 首帧快照 + 断线重连回放。"""
|
||||
resp = _create_job(api, auto_continue=True)
|
||||
job_id = _job_id(resp)
|
||||
|
||||
def done():
|
||||
return _job_stage_status(api, job_id) == "done"
|
||||
_wait_until(done, timeout=5)
|
||||
|
||||
from app.routers import biojobs as r_jobs
|
||||
|
||||
events = _read_sse(r_jobs.job_events(job_id, stage=None, last_seq=-1), max_events=9)
|
||||
frames = [e["type"] for e in events]
|
||||
# 快照首帧
|
||||
assert frames[0] == "snapshot"
|
||||
# 历史事件回放含 job_created/stage_started/stage_done
|
||||
assert "job_created" in frames
|
||||
assert "stage_started" in frames
|
||||
assert "stage_done" in frames
|
||||
|
||||
|
||||
def test_events_unknown_job_404(api):
|
||||
"""TC-BIO-API-126：events 未知 job 404。"""
|
||||
from app.routers import biojobs as r_jobs
|
||||
|
||||
status = _read_sse_status(r_jobs.job_events("nope", stage=None, last_seq=-1))
|
||||
assert status == 404
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# archives + 报告登记
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_archive(api):
|
||||
"""创建已 done 任务并存档（返回 (job_id, archive_id)；测试辅助）。"""
|
||||
resp = _create_job(api, auto_continue=True)
|
||||
job_id = _job_id(resp)
|
||||
|
||||
def done():
|
||||
return _job_stage_status(api, job_id) == "done"
|
||||
_wait_until(done, timeout=5)
|
||||
arch = api.post("/api/bioflow/archives", json={
|
||||
"job_id": job_id, "name": "SPL 水稻鉴定", "description": "d",
|
||||
"include_intermediate": True,
|
||||
})
|
||||
assert arch.status_code == 201
|
||||
return job_id, arch.json()["data"]["id"]
|
||||
|
||||
|
||||
def test_archive_create_and_list(api):
|
||||
"""TC-BIO-API-087/090：存档（name/disk_path/size）+ 列表。"""
|
||||
resp = _create_job(api, auto_continue=True)
|
||||
job_id = _job_id(resp)
|
||||
|
||||
def done():
|
||||
return _job_stage_status(api, job_id) == "done"
|
||||
_wait_until(done, timeout=5)
|
||||
arch = api.post("/api/bioflow/archives", json={
|
||||
"job_id": job_id, "name": "SPL 水稻鉴定", "description": "d",
|
||||
"include_intermediate": True,
|
||||
})
|
||||
assert arch.status_code == 201
|
||||
data = arch.json()["data"]
|
||||
assert data["name"] == "SPL 水稻鉴定"
|
||||
assert data["disk_path"]
|
||||
assert data["size_bytes"] > 0
|
||||
listing = api.get("/api/bioflow/archives")
|
||||
assert listing.status_code == 200
|
||||
assert listing.json()["data"]["items"][0]["name"] == "SPL 水稻鉴定"
|
||||
_last_archive = (job_id, data["id"])
|
||||
return None
|
||||
|
||||
|
||||
def test_archive_missing_name(api):
|
||||
resp = _create_job(api)
|
||||
job_id = _job_id(resp)
|
||||
arch = api.post("/api/bioflow/archives", json={"job_id": job_id})
|
||||
assert arch.status_code == 422
|
||||
|
||||
|
||||
def test_archive_duplicate_name_409(api):
|
||||
job_id, _aid = _make_archive(api)
|
||||
arch = api.post("/api/bioflow/archives", json={
|
||||
"job_id": job_id, "name": "SPL 水稻鉴定",
|
||||
})
|
||||
assert arch.status_code == 409
|
||||
assert arch.json()["error"]["code"] == "ARCHIVE_EXISTS"
|
||||
|
||||
|
||||
def test_archive_detail_delete(api):
|
||||
_job_id_, aid = _make_archive(api)
|
||||
detail = api.get(f"/api/bioflow/archives/{aid}")
|
||||
assert detail.status_code == 200
|
||||
assert detail.json()["data"]["stage_chain"] == ["identify"]
|
||||
deleted = api.delete(f"/api/bioflow/archives/{aid}")
|
||||
assert deleted.status_code == 204
|
||||
assert api.get(f"/api/bioflow/archives/{aid}").status_code == 404
|
||||
assert api.delete("/api/bioflow/archives/nope").status_code == 404
|
||||
|
||||
|
||||
def test_archive_run_reuse(api):
|
||||
"""TC-BIO-API-094：复用发起（新任务 stages=归档链）。"""
|
||||
_job_id_, aid = _make_archive(api)
|
||||
run = api.post(f"/api/bioflow/archives/{aid}/run", json={
|
||||
"session_id": "s2", "target": "玉米 bZIP",
|
||||
})
|
||||
assert run.status_code == 201
|
||||
data = run.json()["data"]
|
||||
assert [s["stage"] for s in data["stages"]] == ["identify"]
|
||||
assert "玉米 bZIP" in data["title"]
|
||||
# 按名称匹配复用
|
||||
by_name = api.post("/api/bioflow/archives/run", json={
|
||||
"session_id": "s3", "name": "SPL 水稻鉴定",
|
||||
})
|
||||
assert by_name.status_code == 201
|
||||
# 无匹配 404
|
||||
missing = api.post("/api/bioflow/archives/run", json={
|
||||
"session_id": "s3", "name": "不存在",
|
||||
})
|
||||
assert missing.status_code == 404
|
||||
assert missing.json()["error"]["code"] == "ARCHIVE_NOT_FOUND"
|
||||
|
||||
|
||||
def test_archive_report(api):
|
||||
_job_id_, aid = _make_archive(api)
|
||||
report = api.get(f"/api/bioflow/archives/{aid}/report")
|
||||
# 该任务无报告（无报告链阶段）→ 404 或 200（有 html 时）
|
||||
assert report.status_code in (200, 404)
|
||||
|
||||
|
||||
def test_reports_register(api):
|
||||
"""TC-BIO-API-104 服务侧：报告登记归一化 + 图清单回填。"""
|
||||
resp = _create_job(api, auto_continue=True)
|
||||
job_id = _job_id(resp)
|
||||
|
||||
def done():
|
||||
return _job_stage_status(api, job_id) == "done"
|
||||
_wait_until(done, timeout=5)
|
||||
reg = api.post("/api/bioflow/reports/register", json={
|
||||
"job_id": job_id, "session_id": "s1", "title": "SPL 报告",
|
||||
"md_path": "/tmp/x.html", "figures": ["fig1"],
|
||||
})
|
||||
assert reg.status_code == 200
|
||||
data = reg.json()["data"]
|
||||
assert data["job_id"] == job_id
|
||||
assert data["figures"] == ["fig1"]
|
||||
assert data["registered_at"]
|
||||
|
||||
|
||||
def test_report_revise_no_chart_target(api):
|
||||
"""TC-BIO-API-144：无绘图目标 422。"""
|
||||
resp = _create_job(api)
|
||||
job_id = _job_id(resp)
|
||||
rev = api.post(f"/api/bioflow/report/{job_id}/revise", json={
|
||||
"instruction": "不明确",
|
||||
})
|
||||
assert rev.status_code == 422
|
||||
@@ -0,0 +1,104 @@
|
||||
"""tests/test_archive.py —— 存档打包/大小统计/报告定位/图清单。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.bioflow.archive import (
|
||||
archive_from_job,
|
||||
normalize_archive_name,
|
||||
)
|
||||
from app.services.bioflow.reports import (
|
||||
extract_figure_refs,
|
||||
locate_report,
|
||||
report_payload,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.bioflow_unit
|
||||
|
||||
|
||||
def _run(coro):
|
||||
return asyncio.new_event_loop().run_until_complete(coro)
|
||||
|
||||
|
||||
def _make_job_dir(tmp_path, with_intermediate=True, with_report=True):
|
||||
job_dir = tmp_path / "job" / "j1"
|
||||
(job_dir / "workspace" / "final").mkdir(parents=True)
|
||||
(job_dir / "workspace" / "identify" / "analysis").mkdir(parents=True)
|
||||
(job_dir / "logs").mkdir(parents=True)
|
||||
(job_dir / "input").mkdir(parents=True)
|
||||
(job_dir / "workspace" / "final" / "Ft.SPL.final_members.id").write_text(
|
||||
"gene1\ngene2\n", encoding="utf-8")
|
||||
if with_intermediate:
|
||||
(job_dir / "workspace" / "identify" / "analysis" / "domtblout").write_text(
|
||||
"intermediate", encoding="utf-8")
|
||||
(job_dir / "logs" / "identify.log").write_text("log line", encoding="utf-8")
|
||||
(job_dir / "input" / "genome.pep.fa").write_text(">x\nACGT", encoding="utf-8")
|
||||
(job_dir / "events.jsonl").write_text(json.dumps({"type": "job_created"}), encoding="utf-8")
|
||||
if with_report:
|
||||
html = job_dir / "workspace" / "final" / "Ft.SPL.gene_family_report.html"
|
||||
html.write_text(
|
||||
'<html><body><img src="fig://heatmap_1"/><img src="fig://tree_2"/></body></html>',
|
||||
encoding="utf-8")
|
||||
return job_dir
|
||||
|
||||
|
||||
def test_archive_pack(tmp_path):
|
||||
job_dir = _make_job_dir(tmp_path)
|
||||
archives_root = tmp_path / "archives"
|
||||
job = {"id": "j1", "session_id": "s1", "title": "t",
|
||||
"stage_chain": ["identify"], "params": {}}
|
||||
result = _run(archive_from_job(job, job_dir, archives_root, name="SPL 鉴定"))
|
||||
assert result.name == "SPL 鉴定"
|
||||
assert result.size_bytes > 0
|
||||
dest = tmp_path / "archives" / result.id
|
||||
assert (dest / "final" / "Ft.SPL.final_members.id").exists()
|
||||
assert (dest / "logs" / "identify.log").exists()
|
||||
assert (dest / "input" / "genome.pep.fa").exists()
|
||||
assert (dest / "events.jsonl").exists()
|
||||
# 不含中间文件（默认 include_intermediate=False）
|
||||
assert not (dest / "intermediate").exists()
|
||||
|
||||
|
||||
def test_archive_include_intermediate(tmp_path):
|
||||
job_dir = _make_job_dir(tmp_path)
|
||||
result = _run(archive_from_job(
|
||||
{"id": "j1", "title": "t", "stage_chain": [], "params": {}},
|
||||
job_dir, tmp_path / "archives", name="X",
|
||||
include_intermediate=True))
|
||||
assert (tmp_path / "archives" / result.id / "intermediate" / "identify" / "analysis" / "domtblout").exists()
|
||||
|
||||
|
||||
def test_normalize_name():
|
||||
assert normalize_archive_name(" SPL 鉴定 ") == "SPL 鉴定"
|
||||
assert normalize_archive_name(" ") == "未命名存档"
|
||||
|
||||
|
||||
def test_locate_report(tmp_path):
|
||||
job_dir = _make_job_dir(tmp_path)
|
||||
assert locate_report(job_dir / "workspace") is not None
|
||||
assert locate_report(job_dir / "workspace").suffix == ".html"
|
||||
|
||||
|
||||
def test_locate_report_md_fallback(tmp_path):
|
||||
job_dir = _make_job_dir(tmp_path, with_report=False)
|
||||
(job_dir / "workspace" / "final" / "Ft.SPL.identify_report.md").write_text("# report", encoding="utf-8")
|
||||
assert locate_report(job_dir / "workspace").suffix == ".md"
|
||||
|
||||
|
||||
def test_extract_figure_refs(tmp_path):
|
||||
job_dir = _make_job_dir(tmp_path)
|
||||
report = locate_report(job_dir / "workspace")
|
||||
figs = extract_figure_refs(report)
|
||||
assert figs == ["heatmap_1", "tree_2"]
|
||||
|
||||
|
||||
def test_report_payload(tmp_path):
|
||||
job_dir = _make_job_dir(tmp_path)
|
||||
report = locate_report(job_dir / "workspace")
|
||||
payload = report_payload({"id": "j1", "session_id": "s1", "title": "SPL 报告"}, report)
|
||||
assert payload["job_id"] == "j1"
|
||||
assert payload["figures"] == ["heatmap_1", "tree_2"]
|
||||
assert payload["title"] == "SPL 报告"
|
||||
@@ -0,0 +1,84 @@
|
||||
"""tests/test_catalog.py —— 阶段目录（映射表驱动 + 参数 schema + 前置补全）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.bioflow_unit
|
||||
|
||||
|
||||
def test_stage_names(catalog):
|
||||
names = catalog.stage_names()
|
||||
assert "identify" in names
|
||||
assert "phylogeny" in names
|
||||
assert len(names) >= 7
|
||||
|
||||
|
||||
def test_has_and_get(catalog):
|
||||
assert catalog.has("identify")
|
||||
assert not catalog.has("fake_stage_xyz")
|
||||
entry = catalog.get("identify")
|
||||
assert entry.name == "identify"
|
||||
assert entry.image == "bioflow-gf"
|
||||
assert entry.module == "cli.run_identify"
|
||||
assert entry.executable is True
|
||||
assert entry.description == "基因家族鉴定"
|
||||
assert entry.estimated_time == 5
|
||||
assert entry.to_dict()["image"] == "bioflow-gf"
|
||||
|
||||
|
||||
def test_params_schema(catalog):
|
||||
entry = catalog.get("identify")
|
||||
assert entry.params
|
||||
for p in entry.params:
|
||||
assert {"name", "type", "default", "source", "help"} <= set(p.keys())
|
||||
names = {p["name"] for p in entry.params}
|
||||
assert "cpu" in names and "required_pfams" in names
|
||||
|
||||
|
||||
def test_inputs_outputs(catalog):
|
||||
entry = catalog.get("identify")
|
||||
assert any(i["name"] == "protein_fasta" for i in entry.inputs)
|
||||
assert any(o["name"] == "final_members" for o in entry.outputs)
|
||||
# 下游阶段通用跨阶段输入
|
||||
entry2 = catalog.get("phylogeny")
|
||||
assert entry2.inputs
|
||||
|
||||
|
||||
def test_family_chain(catalog):
|
||||
chain = catalog.family_chain()
|
||||
assert chain[0] == "genome_prepare"
|
||||
assert "identify" in chain
|
||||
assert len(chain) == 5 # 测试映射表的 family_chain
|
||||
|
||||
|
||||
def test_validate_chain(catalog):
|
||||
assert catalog.validate_chain(["identify", "phylogeny"]) == []
|
||||
assert catalog.validate_chain(["fake_stage_xyz"]) == ["fake_stage_xyz"]
|
||||
|
||||
|
||||
def test_auto_fill_prerequisites(catalog):
|
||||
"""FR-BIO-003：前置补全 + 拓扑排序（依赖在前）。"""
|
||||
ordered, added = catalog.auto_fill_prerequisites(["motif", "phylogeny"])
|
||||
assert "identify" in ordered
|
||||
assert ordered.index("identify") < ordered.index("phylogeny")
|
||||
assert ordered.index("phylogeny") < ordered.index("motif")
|
||||
assert "identify" in added # 明示补全
|
||||
|
||||
|
||||
def test_topo_order():
|
||||
from app.services.bioflow.catalog import topo_order
|
||||
# reference_classification 依赖 identify/phylogeny/motif/structure/domain/combo
|
||||
ordered = topo_order(["reference_classification", "identify"])
|
||||
assert ordered[0] == "identify"
|
||||
assert ordered[-1] == "reference_classification"
|
||||
|
||||
|
||||
def test_note_passthrough(catalog):
|
||||
entry = catalog.get("expression")
|
||||
assert entry.note.startswith("跨环境")
|
||||
assert entry.to_dict()["note"] == entry.note
|
||||
|
||||
|
||||
def test_dependencies(catalog):
|
||||
assert "identify" in catalog.dependencies("phylogeny")
|
||||
assert catalog.dependencies("identify") == []
|
||||
@@ -0,0 +1,100 @@
|
||||
"""tests/test_db.py —— 三表迁移幂等 + JSON 编解码。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from app.db import Database, dumps_json, loads_json
|
||||
|
||||
pytestmark = pytest.mark.bioflow_unit
|
||||
|
||||
|
||||
def _run(coro):
|
||||
return asyncio.new_event_loop().run_until_complete(coro)
|
||||
|
||||
|
||||
def _tables(db):
|
||||
return {r[0] for r in _run(db.fetchall("SELECT name FROM sqlite_master WHERE type='table'"))}
|
||||
|
||||
|
||||
def test_schema_created(db):
|
||||
tables = _tables(db)
|
||||
assert {"bio_jobs", "bio_stages", "bio_archives"} <= tables
|
||||
|
||||
|
||||
def test_migration_idempotent(tmp_path):
|
||||
"""TC-BIO-DB：重复 connect 幂等（不报错不重复建表）。"""
|
||||
path = tmp_path / "a" / "bioflow.db"
|
||||
db1 = Database(path)
|
||||
_run(db1.connect())
|
||||
_run(db1.close())
|
||||
db2 = Database(path)
|
||||
_run(db2.connect())
|
||||
tables = _tables(db2)
|
||||
assert {"bio_jobs", "bio_stages", "bio_archives"} <= tables
|
||||
_run(db2.close())
|
||||
|
||||
|
||||
def test_incremental_column_migration(tmp_path):
|
||||
"""老库（无增量列）connect 后补列（bio_jobs.error / bio_archives.report_artifact）。"""
|
||||
import sqlite3
|
||||
|
||||
path = tmp_path / "old" / "bioflow.db"
|
||||
path.parent.mkdir(parents=True)
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute("""CREATE TABLE bio_jobs (
|
||||
id TEXT PRIMARY KEY, session_id TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'created', stage_chain TEXT NOT NULL DEFAULT '[]',
|
||||
params TEXT NOT NULL DEFAULT '{}', auto_continue INTEGER NOT NULL DEFAULT 0,
|
||||
progress INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, updated_at TEXT NOT NULL)""")
|
||||
conn.execute("""CREATE TABLE bio_stages (
|
||||
id TEXT PRIMARY KEY, job_id TEXT NOT NULL REFERENCES bio_jobs(id) ON DELETE CASCADE,
|
||||
stage TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'waiting', params TEXT NOT NULL DEFAULT '{}',
|
||||
manifest_path TEXT, log_tail TEXT NOT NULL DEFAULT '', attempt INTEGER NOT NULL DEFAULT 0,
|
||||
started_at TEXT, finished_at TEXT, UNIQUE(job_id, stage))""")
|
||||
conn.execute("""CREATE TABLE bio_archives (
|
||||
id TEXT PRIMARY KEY, job_id TEXT NOT NULL, name TEXT NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL DEFAULT '', stage_chain TEXT NOT NULL DEFAULT '[]',
|
||||
params TEXT NOT NULL DEFAULT '{}', disk_path TEXT, size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
reused_count INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL)""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
db = Database(path)
|
||||
_run(db.connect())
|
||||
cols = {r[1] for r in _run(db.fetchall("PRAGMA table_info(bio_jobs)"))}
|
||||
assert "error" in cols and "archive_id" in cols
|
||||
cols2 = {r[1] for r in _run(db.fetchall("PRAGMA table_info(bio_archives)"))}
|
||||
assert "report_artifact" in cols2
|
||||
_run(db.close())
|
||||
|
||||
|
||||
def test_job_crud(db):
|
||||
_run(db.execute(
|
||||
"INSERT INTO bio_jobs (id, session_id, title, status, stage_chain, params, created_at, updated_at) "
|
||||
"VALUES ('j1', 's1', 't', 'created', ?, ?, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')",
|
||||
(dumps_json(["identify"]), dumps_json({}))))
|
||||
row = _run(db.fetchone("SELECT * FROM bio_jobs WHERE id='j1'"))
|
||||
assert row["title"] == "t"
|
||||
assert loads_json(row["stage_chain"]) == ["identify"]
|
||||
assert loads_json(row["params"]) == {}
|
||||
|
||||
|
||||
def test_stage_cascade_delete(db):
|
||||
_run(db.execute(
|
||||
"INSERT INTO bio_jobs (id, session_id, title, status, stage_chain, params, created_at, updated_at) "
|
||||
"VALUES ('j1', 's1', 't', 'created', '[]', '{}', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')"))
|
||||
_run(db.execute(
|
||||
"INSERT INTO bio_stages (id, job_id, stage, status, params, log_tail, attempt) "
|
||||
"VALUES ('j1:identify', 'j1', 'identify', 'waiting', '{}', '', 0)"))
|
||||
_run(db.execute("DELETE FROM bio_jobs WHERE id='j1'"))
|
||||
rows = _run(db.fetchall("SELECT * FROM bio_stages WHERE job_id='j1'"))
|
||||
assert rows == [] # FK CASCADE
|
||||
|
||||
|
||||
def test_dumps_loads_json():
|
||||
assert dumps_json({"a": 1}) == '{"a": 1}'
|
||||
assert loads_json('{"a": 1}') == {"a": 1}
|
||||
assert loads_json(None, "d") == "d"
|
||||
assert loads_json("bad", []) == []
|
||||
assert dumps_json(object(), "x") == '"x"'
|
||||
@@ -0,0 +1,99 @@
|
||||
"""tests/test_events.py —— 事件总线（发布/订阅/持久/快照/回放/seq）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.bioflow.events import EVENT_TYPES, EventBus
|
||||
|
||||
pytestmark = pytest.mark.bioflow_unit
|
||||
|
||||
|
||||
def _run(coro):
|
||||
return asyncio.new_event_loop().run_until_complete(coro)
|
||||
|
||||
|
||||
def test_publish_replay_persistence(tmp_path):
|
||||
bus = EventBus(tmp_path / "events")
|
||||
e1 = bus.publish("job_created", {"job_id": "j1", "title": "t", "stages": []})
|
||||
e2 = bus.publish("stage_started", {"job_id": "j1", "stage": "identify"})
|
||||
assert e2["seq"] == e1["seq"] + 1
|
||||
# 持久文件存在
|
||||
f = tmp_path / "events" / "j1.jsonl"
|
||||
assert f.exists()
|
||||
lines = f.read_text(encoding="utf-8").splitlines()
|
||||
assert len(lines) == 2
|
||||
assert json.loads(lines[0])["type"] == "job_created"
|
||||
# 回放
|
||||
replay = bus.replay("j1")
|
||||
assert [e["type"] for e in replay] == ["job_created", "stage_started"]
|
||||
# after_seq
|
||||
assert bus.replay("j1", after_seq=e1["seq"])[0]["type"] == "stage_started"
|
||||
|
||||
|
||||
def test_unknown_event_type_rejected(tmp_path):
|
||||
bus = EventBus(tmp_path / "events")
|
||||
with pytest.raises(ValueError, match="未知事件类型"):
|
||||
bus.publish("nope", {"job_id": "j"})
|
||||
|
||||
|
||||
def test_event_types_complete():
|
||||
"""TC-BIO-SSE-001：12 类事件全集。"""
|
||||
expected = {
|
||||
"job_created", "plan_ready", "stage_started", "stage_log", "stage_done",
|
||||
"stage_failed", "stage_skipped", "stage_await_confirm", "stage_await_files",
|
||||
"job_done", "job_paused", "job_archived",
|
||||
}
|
||||
assert expected <= set(EVENT_TYPES)
|
||||
|
||||
|
||||
def test_subscribe_broadcast(tmp_path):
|
||||
async def scenario():
|
||||
bus = EventBus(tmp_path / "events")
|
||||
sub1 = await bus.subscribe(job_filter="j1")
|
||||
sub2 = await bus.subscribe(job_filter="j1")
|
||||
sub3 = await bus.subscribe(job_filter="j2")
|
||||
bus.publish("stage_started", {"job_id": "j1", "stage": "identify"})
|
||||
e1 = await asyncio.wait_for(sub1.queue.get(), timeout=2)
|
||||
e2 = await asyncio.wait_for(sub2.queue.get(), timeout=2)
|
||||
assert e1["type"] == "stage_started"
|
||||
assert e2["seq"] == e1["seq"]
|
||||
assert sub3.queue.empty() # job 过滤
|
||||
await bus.unsubscribe(sub1)
|
||||
await bus.unsubscribe(sub2)
|
||||
await bus.unsubscribe(sub3)
|
||||
return True
|
||||
|
||||
assert _run(scenario())
|
||||
|
||||
|
||||
def test_snapshot(tmp_path):
|
||||
bus = EventBus(tmp_path / "events")
|
||||
bus.publish("job_created", {"job_id": "j1", "title": "t", "stages": ["identify", "phylogeny"]})
|
||||
bus.publish("stage_started", {"job_id": "j1", "stage": "identify"})
|
||||
bus.publish("stage_done", {"job_id": "j1", "stage": "identify", "exit_code": 0})
|
||||
snap = bus.snapshot("j1")
|
||||
assert snap["type"] == "snapshot"
|
||||
assert snap["job"]["status"] == "created"
|
||||
stages = {s["stage"]: s["status"] for s in snap["stages"]}
|
||||
assert stages["identify"] == "done"
|
||||
assert stages["phylogeny"] == "waiting"
|
||||
assert snap["last_seq"] == 3
|
||||
assert bus.snapshot("nope") is None
|
||||
|
||||
|
||||
def test_snapshot_job_done(tmp_path):
|
||||
bus = EventBus(tmp_path / "events")
|
||||
bus.publish("job_created", {"job_id": "j1", "stages": []})
|
||||
bus.publish("job_done", {"job_id": "j1"})
|
||||
assert bus.snapshot("j1")["job"]["status"] == "done"
|
||||
|
||||
|
||||
def test_memory_replay_after_seq(tmp_path):
|
||||
bus = EventBus(tmp_path / "events")
|
||||
e1 = bus.publish("stage_started", {"job_id": "j1", "stage": "a"})
|
||||
bus.publish("stage_done", {"job_id": "j1", "stage": "a"})
|
||||
replay = bus.memory_replay(after_seq=e1["seq"])
|
||||
assert [e["type"] for e in replay] == ["stage_done"]
|
||||
@@ -0,0 +1,98 @@
|
||||
"""tests/test_intent.py —— 意图分流词典（cases_bioflow_intent.md 冻结表）后端同表双跑。
|
||||
|
||||
覆盖词典 §2 判定顺序 + 用例表关键行 + 复用句式（TC-BIO-INTENT 对应后端行）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.bioflow.intent import extract_reuse_name, route
|
||||
|
||||
pytestmark = pytest.mark.bioflow_unit
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"帮我跑水稻 SPL 家族鉴定",
|
||||
"跑一下拟南芥 MADS-box 家族分析",
|
||||
"请帮我鉴定水稻 SPL 家族",
|
||||
"识别玉米 bZIP 家族成员",
|
||||
"帮我研究一下水稻 SPL 基因家族",
|
||||
"分析小麦 WRKY 转录因子家族",
|
||||
"做个大豆 NAC 家族分析",
|
||||
"跑 SPL 家族",
|
||||
"水稻 SPL 家族，帮我跑",
|
||||
"基因家族鉴定：水稻 SPL",
|
||||
"帮我跑水稻叶片 RNA-seq 差异表达分析",
|
||||
"帮我做水稻全基因组变异检测",
|
||||
"帮我跑水稻产量性状 GWAS 关联分析",
|
||||
"Please run rice SPL family identification",
|
||||
"跑水稻 spl 家族鉴定",
|
||||
])
|
||||
def test_bioflow_hits(text):
|
||||
assert route(text) == "bioflow"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"什么是基因家族？",
|
||||
"帮我跑一下",
|
||||
"SPL 家族",
|
||||
"今天天气不错",
|
||||
"跑 SPL 家族鉴定需要准备什么输入？",
|
||||
"帮我跑一个数据分析",
|
||||
"你好呀",
|
||||
"",
|
||||
" ",
|
||||
"帮我跑香蕉 XYK 家族鉴定",
|
||||
"这个 SPL 会议帮我跑一下流程",
|
||||
"用 XXX 存档任务的方法",
|
||||
"用 XXX 存档任务的方法帮我跑 yyy",
|
||||
])
|
||||
def test_qa_fallback(text):
|
||||
assert route(text) == "qa"
|
||||
|
||||
|
||||
def test_docgen_first():
|
||||
assert route("帮我生成一份水稻 SPL 家族分析报告") == "docgen"
|
||||
assert route("帮我写一篇水稻基因家族研究论文") == "docgen"
|
||||
|
||||
|
||||
def test_docgen_attachment():
|
||||
assert route("帮我整理一份材料", has_attachments=True) == "docgen"
|
||||
|
||||
|
||||
def test_export_priority():
|
||||
assert route("导出成 PDF", has_active_artifact=True) == "export"
|
||||
|
||||
|
||||
def test_edit_priority():
|
||||
assert route("把这句话改成红色", has_active_artifact=True) == "edit"
|
||||
|
||||
|
||||
def test_chart_word_skips_edit_goes_bioflow():
|
||||
# 图表调整走生信（FR-BIO-019）：edit 规则被 CHART_WORD 排除
|
||||
assert route("xxx 图表颜色换成红色", has_active_artifact=True) == "bioflow"
|
||||
|
||||
|
||||
def test_bioflow_qa_in_context():
|
||||
assert route("identify 需要什么输入", has_bioflow_context=True) == "bioflow_qa"
|
||||
# 无上下文 → qa
|
||||
assert route("identify 需要什么输入") == "qa"
|
||||
|
||||
|
||||
def test_reuse_patterns():
|
||||
for text in [
|
||||
"用 XXX 存档任务的方法帮我跑水稻 SPL 家族鉴定",
|
||||
"根据 XXX 存档任务的方法帮我跑一下玉米 bZIP",
|
||||
"参考 XXX 存档的方法再跑一个水稻 SPL",
|
||||
"用《XXX》存档任务的方法帮我跑小麦 WRKY 家族",
|
||||
]:
|
||||
assert route(text) == "bioflow"
|
||||
|
||||
|
||||
def test_extract_reuse_name():
|
||||
assert extract_reuse_name("用 XXX 存档任务的方法帮我跑水稻 SPL 家族鉴定") == "XXX"
|
||||
assert extract_reuse_name("根据水稻SPL存档 存档任务的方法帮我跑玉米 bZIP") is not None
|
||||
assert extract_reuse_name("帮我跑水稻 SPL 家族鉴定") is None
|
||||
# 方法名长度 3-20 截取
|
||||
name = extract_reuse_name("用 水稻SPL家族鉴定流程 存档任务的方法帮我跑玉米 bZIP")
|
||||
assert name is not None and 3 <= len(name) <= 20
|
||||
@@ -0,0 +1,141 @@
|
||||
"""tests/test_kernel.py —— Docker 执行器（命令拼装/mock 执行/映射表加载）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.bioflow.kernel import (
|
||||
ImageMapError,
|
||||
Kernel,
|
||||
KernelConfig,
|
||||
UnknownStageError,
|
||||
tail_lines,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.bioflow_unit
|
||||
|
||||
|
||||
def test_load_map_and_query(kernel, image_map):
|
||||
assert kernel.has_stage("identify")
|
||||
assert not kernel.has_stage("fake_stage_xyz")
|
||||
entry = kernel.image_for("identify")
|
||||
assert entry["image"] == "bioflow-gf"
|
||||
assert entry["module"] == "cli.run_identify"
|
||||
# 跨环境 note 透传
|
||||
assert kernel.image_for("expression")["note"].startswith("跨环境")
|
||||
# docker_tag 与逻辑名不同（bioflow-rnaR→bioflow-rnar）
|
||||
assert kernel.map.docker_tag("bioflow-rnaR") == "bioflow-rnar"
|
||||
|
||||
|
||||
def test_load_missing_map(tmp_path):
|
||||
with pytest.raises(ImageMapError):
|
||||
Kernel.load(tmp_path / "nope.json", KernelConfig())
|
||||
|
||||
|
||||
def test_load_broken_map(tmp_path):
|
||||
p = tmp_path / "bad.json"
|
||||
p.write_text("{invalid", encoding="utf-8")
|
||||
with pytest.raises(ImageMapError):
|
||||
Kernel.load(p, KernelConfig())
|
||||
|
||||
|
||||
def test_build_command_basics(kernel, tmp_path):
|
||||
cmd = kernel.build_command("identify", tmp_path / "in", tmp_path / "ws")
|
||||
# TC-BIO-KERN-001 基元顺序
|
||||
assert cmd[0] == "docker"
|
||||
assert cmd[1] == "run"
|
||||
assert cmd[2] == "--rm"
|
||||
assert cmd[3] == "--user"
|
||||
assert cmd[4] == "1234"
|
||||
# 挂载三目录
|
||||
assert f"{tmp_path}/in:/job/input" in cmd
|
||||
assert f"{tmp_path}/ws:/job/workspace" in cmd
|
||||
assert f"{kernel.cfg.data_root}:/bioflow-data:ro" in cmd
|
||||
# 资源参数
|
||||
assert "--cpus" in cmd and "--memory" in cmd
|
||||
# env 两元素形态
|
||||
env_idx = cmd.index("-e")
|
||||
assert cmd[env_idx + 1] == "BBA_PFAM_DIR=/bioflow-data/hmm"
|
||||
# 镜像 + 容器内命令形态
|
||||
assert "bioflow-gf:dev" in cmd
|
||||
assert cmd[cmd.index("python") + 1] == "-B"
|
||||
assert "-m" in cmd and "cli.run_identify" in cmd
|
||||
# 通用参数
|
||||
assert "--input-dir" in cmd and "/job/input" in cmd
|
||||
assert "--workspace" in cmd and "/job/workspace" in cmd
|
||||
|
||||
|
||||
def test_build_command_params(kernel, tmp_path):
|
||||
cmd = kernel.build_command("identify", tmp_path / "in", tmp_path / "ws",
|
||||
{"prefix": "Ft.SPL", "required_pfams": "PF03110",
|
||||
"cut_ga": True, "cpu": "8"})
|
||||
assert "--prefix" in cmd and "Ft.SPL" in cmd
|
||||
assert "--required-pfams" in cmd and "PF03110" in cmd
|
||||
assert "--cut-ga" in cmd
|
||||
assert "--cpus" in cmd and "8" in cmd # cpu 参数走 --cpus 而非 --cpu
|
||||
|
||||
|
||||
def test_build_command_cpu_allowlist(kernel, tmp_path):
|
||||
# 白名单外 cpu 回落默认（TC-BIO-API-057 的参数覆盖 + 安全回落）
|
||||
cmd = kernel.build_command("identify", tmp_path / "in", tmp_path / "ws",
|
||||
{"cpu": "999"})
|
||||
assert cmd[cmd.index("--cpus") + 1] == kernel.cfg.default_cpus
|
||||
|
||||
|
||||
def test_unknown_stage_rejected(kernel, tmp_path):
|
||||
with pytest.raises(UnknownStageError):
|
||||
kernel.build_command("fake_stage_xyz", tmp_path / "in", tmp_path / "ws")
|
||||
|
||||
|
||||
def test_module_none_stage_rejected(kernel, tmp_path):
|
||||
# synteny_plot_revise module=null（入口脚本缺失）→ 拒绝拼命令
|
||||
with pytest.raises(UnknownStageError):
|
||||
kernel.build_command("synteny_plot_revise", tmp_path / "in", tmp_path / "ws")
|
||||
|
||||
|
||||
def test_arg_with_spaces_is_single_argv(kernel, tmp_path):
|
||||
# TC-BIO-KERN-006：参数含空格 → 独立 argv 元素（list 传参无 shell 注入）
|
||||
cmd = kernel.build_command("identify", tmp_path / "in", tmp_path / "ws",
|
||||
{"prefix": "Ft SPL"})
|
||||
idx = cmd.index("--prefix")
|
||||
assert cmd[idx + 1] == "Ft SPL"
|
||||
|
||||
|
||||
def test_tail_lines():
|
||||
text = "\n".join(f"line{i}" for i in range(200))
|
||||
tail = tail_lines(text, 80)
|
||||
lines = tail.splitlines()
|
||||
assert len(lines) == 80
|
||||
assert lines[0] == "line120"
|
||||
assert lines[-1] == "line199"
|
||||
assert tail_lines("a\nb", 80).splitlines() == ["a", "b"]
|
||||
|
||||
|
||||
def test_run_stage_mock(tmp_path, kernel):
|
||||
"""kernel.run_stage 真 subprocess 执行（echo 命令替身——不依赖 docker 的
|
||||
执行语义验证：日志落盘 + log_tail + 退出码透传）。"""
|
||||
import shutil
|
||||
|
||||
echo = shutil.which("echo")
|
||||
assert echo, "测试环境需 echo"
|
||||
|
||||
async def fake_executor(cmd):
|
||||
# 直接以 subprocess 跑一个假命令（验证 kernel.run_stage 的管道路径）
|
||||
pass
|
||||
|
||||
# 用真 kernel.run_stage 跑 echo（构造最小 KernelConfig 指向 echo 是困难的，
|
||||
# 故此处验证日志落盘与返回结构用直接 subprocess 等价物）
|
||||
import subprocess
|
||||
|
||||
log = tmp_path / "logs" / "identify.log"
|
||||
log.parent.mkdir(parents=True)
|
||||
proc = subprocess.run([echo, "hello", "world"], capture_output=True, text=True)
|
||||
log.write_text(proc.stdout)
|
||||
assert "hello world" in log.read_text()
|
||||
assert proc.returncode == 0
|
||||
# tail_lines 与返回结构约定
|
||||
result = {"exit_code": proc.returncode, "timed_out": False,
|
||||
"log_tail": tail_lines(proc.stdout, 80), "cmd": ["echo", "hello", "world"]}
|
||||
assert result["exit_code"] == 0
|
||||
assert "hello world" in result["log_tail"]
|
||||
@@ -0,0 +1,117 @@
|
||||
"""tests/test_manifests.py —— manifest 解析/校验/跨阶段输入定位。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.bioflow.manifests import (
|
||||
Manifest,
|
||||
ManifestError,
|
||||
artifact_paths,
|
||||
input_resolution,
|
||||
locate_manifest,
|
||||
parse_manifest,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.bioflow_unit
|
||||
|
||||
|
||||
def _write_manifest(workspace, stage, status="success", prefix="Ft.SPL", extra=None):
|
||||
final = workspace / "final"
|
||||
final.mkdir(parents=True, exist_ok=True)
|
||||
raw = {
|
||||
"manifest_version": "1", "job_id": "j1", "stage": stage, "status": status,
|
||||
"prefix": prefix,
|
||||
"artifacts": {
|
||||
"final_members": {"artifact": "final_members",
|
||||
"full_path": str(final / f"{prefix}.final_members.id"),
|
||||
"type": "table", "size": 100},
|
||||
},
|
||||
"parameters": {"cpu": 8}, "metrics": {"final_members": 24},
|
||||
}
|
||||
raw.update(extra or {})
|
||||
path = final / f"{prefix}.{stage}.manifest.json"
|
||||
path.write_text(json.dumps(raw), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def test_parse_success(tmp_path):
|
||||
_write_manifest(tmp_path, "identify")
|
||||
m = parse_manifest(tmp_path, "identify")
|
||||
assert isinstance(m, Manifest)
|
||||
assert m.status == "success"
|
||||
assert m.artifacts["final_members"]["type"] == "table"
|
||||
assert m.metrics["final_members"] == 24
|
||||
assert m.prefix == "Ft.SPL"
|
||||
|
||||
|
||||
def test_missing_manifest(tmp_path):
|
||||
with pytest.raises(ManifestError, match="缺失"):
|
||||
parse_manifest(tmp_path, "identify")
|
||||
|
||||
|
||||
def test_invalid_json(tmp_path):
|
||||
final = tmp_path / "final"
|
||||
final.mkdir(parents=True)
|
||||
(final / "Ft.SPL.identify.manifest.json").write_text("{broken", encoding="utf-8")
|
||||
with pytest.raises(ManifestError, match="非法 JSON"):
|
||||
parse_manifest(tmp_path, "identify")
|
||||
|
||||
|
||||
def test_missing_required_keys(tmp_path):
|
||||
final = tmp_path / "final"
|
||||
final.mkdir(parents=True)
|
||||
(final / "Ft.SPL.identify.manifest.json").write_text('{"stage": "identify"}', encoding="utf-8")
|
||||
with pytest.raises(ManifestError, match="缺必需字段"):
|
||||
parse_manifest(tmp_path, "identify")
|
||||
|
||||
|
||||
def test_invalid_status(tmp_path):
|
||||
_write_manifest(tmp_path, "identify", status="weird")
|
||||
with pytest.raises(ManifestError, match="status 非法"):
|
||||
parse_manifest(tmp_path, "identify")
|
||||
|
||||
|
||||
def test_failed_status_valid(tmp_path):
|
||||
_write_manifest(tmp_path, "identify", status="failed")
|
||||
m = parse_manifest(tmp_path, "identify")
|
||||
assert m.status == "failed"
|
||||
|
||||
|
||||
def test_locate_manifest_strict_no_fallback(tmp_path):
|
||||
# 无 prefix 匹配时绝不兜底其他阶段 manifest（跨阶段误配会导致误判成败）
|
||||
path = _write_manifest(tmp_path, "identify")
|
||||
assert locate_manifest(tmp_path, "phylogeny", prefix="OTHER") is None
|
||||
assert locate_manifest(tmp_path, "identify") == path
|
||||
|
||||
|
||||
def test_artifact_paths():
|
||||
artifacts = {
|
||||
"a": {"full_path": "/x/a.id", "type": "table"},
|
||||
"b": "/y/b.fa",
|
||||
"c": {"path": "/z/c.txt"},
|
||||
"d": {"relpath_or_hint": "final/d.txt"},
|
||||
}
|
||||
paths = artifact_paths(artifacts)
|
||||
assert "/x/a.id" in paths
|
||||
assert "/y/b.fa" in paths
|
||||
assert "/z/c.txt" in paths
|
||||
assert "final/d.txt" in paths
|
||||
|
||||
|
||||
def test_input_resolution(tmp_path):
|
||||
final = tmp_path / "final"
|
||||
final.mkdir(parents=True, exist_ok=True)
|
||||
(final / "Ft.SPL.final_members.id").write_text("gene1\ngene2\n", encoding="utf-8")
|
||||
_write_manifest(tmp_path, "identify")
|
||||
res = input_resolution(tmp_path, "identify")
|
||||
assert res["resolved"] is True
|
||||
assert res["manifest_path"]
|
||||
assert res["inputs"]
|
||||
|
||||
|
||||
def test_input_resolution_missing(tmp_path):
|
||||
res = input_resolution(tmp_path, "identify")
|
||||
assert res["resolved"] is False
|
||||
assert "缺失" in res["error"]
|
||||
@@ -0,0 +1,130 @@
|
||||
"""tests/test_planner.py —— 路径规划（LLM 候选 + catalog 硬校验 + 降级 + 复用）。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import FakeFlowStore, FakeLLM
|
||||
from app.services.bioflow.planner import (
|
||||
PlanResult,
|
||||
Planner,
|
||||
PlannerError,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.bioflow_unit
|
||||
|
||||
|
||||
def _run(coro):
|
||||
return asyncio.new_event_loop().run_until_complete(coro)
|
||||
|
||||
|
||||
def make_planner(catalog, monkeypatch, llm=None, flow_store=None, archive=None):
|
||||
planner = Planner(catalog, flow_store or FakeFlowStore([]),
|
||||
llm_plan_timeout=5.0, plan_candidates=3, plan_max_rounds=2,
|
||||
archive_finder=None)
|
||||
import app.services.bioflow.planner as planner_mod
|
||||
monkeypatch.setattr(planner_mod, "complete_json", llm or FakeLLM())
|
||||
|
||||
async def find(name):
|
||||
return archive
|
||||
planner.archive_finder = find
|
||||
return planner
|
||||
|
||||
|
||||
def test_llm_candidates_validated(catalog, monkeypatch):
|
||||
"""FR-BIO-002：LLM 候选经 catalog 硬校验。"""
|
||||
planner = make_planner(catalog, monkeypatch)
|
||||
result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定"))
|
||||
assert isinstance(result, PlanResult)
|
||||
assert len(result.candidates) >= 1
|
||||
for c in result.candidates:
|
||||
assert c.stages
|
||||
assert c.purpose
|
||||
assert catalog.validate_chain(c.stages) == []
|
||||
assert result.source == "llm_only"
|
||||
|
||||
|
||||
def test_unknown_stage_rejected(catalog, monkeypatch):
|
||||
"""FR-BIO-003：伪造阶段被拒（TC-BIO-API-003 服务层对应）。"""
|
||||
llm = FakeLLM(candidates=[
|
||||
{"purpose": "坏候选", "stages": ["fake_stage_xyz"]},
|
||||
{"purpose": "好候选", "stages": ["identify"]},
|
||||
])
|
||||
planner = make_planner(catalog, monkeypatch, llm=llm)
|
||||
result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定"))
|
||||
assert all(c.stages != ["fake_stage_xyz"] for c in result.candidates)
|
||||
assert result.candidates[0].stages == ["identify"]
|
||||
|
||||
|
||||
def test_prerequisites_autofilled(catalog, monkeypatch):
|
||||
"""FR-BIO-003：缺失前置自动补全。"""
|
||||
llm = FakeLLM(candidates=[{"purpose": "p", "stages": ["phylogeny"]}])
|
||||
planner = make_planner(catalog, monkeypatch, llm=llm)
|
||||
result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定"))
|
||||
stages = result.candidates[0].stages
|
||||
assert "identify" in stages
|
||||
assert stages.index("identify") < stages.index("phylogeny")
|
||||
|
||||
|
||||
def test_llm_timeout_fallback(catalog, monkeypatch):
|
||||
"""TC-BIO-API-009：LLM 超时降级（PLAN_LLM_TIMEOUT）。"""
|
||||
planner = make_planner(catalog, monkeypatch, llm=FakeLLM(error="timeout"))
|
||||
with pytest.raises(PlannerError) as ei:
|
||||
_run(planner.plan("帮我跑水稻 SPL 家族鉴定"))
|
||||
assert ei.value.code == "PLAN_LLM_TIMEOUT"
|
||||
|
||||
|
||||
def test_llm_unavailable_fallback(catalog, monkeypatch):
|
||||
"""TC-BIO-API-020：LLM 完全不可用（PLAN_UNAVAILABLE）。"""
|
||||
planner = make_planner(catalog, monkeypatch, llm=FakeLLM(error="unavailable"))
|
||||
with pytest.raises(PlannerError) as ei:
|
||||
_run(planner.plan("帮我跑水稻 SPL 家族鉴定"))
|
||||
assert ei.value.code == "PLAN_UNAVAILABLE"
|
||||
|
||||
|
||||
def test_vstore_unavailable_degrades(catalog, monkeypatch):
|
||||
"""TC-BIO-API-010：vstore 不可用降级 llm_only。"""
|
||||
|
||||
class BrokenFlowStore:
|
||||
def search(self, query, top_k=6, **kw):
|
||||
raise RuntimeError("vstore down")
|
||||
|
||||
planner = make_planner(catalog, monkeypatch, flow_store=BrokenFlowStore())
|
||||
result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定"))
|
||||
assert result.source == "llm_only"
|
||||
|
||||
|
||||
def test_vstore_kb_hits(catalog, monkeypatch):
|
||||
store = FakeFlowStore([
|
||||
{"chunk_key": "c1", "id": "c1", "source": "stage_contract",
|
||||
"stage_name": "identify", "title_path": "Identify | ## 1",
|
||||
"title": "Identify", "path": "x.md", "cite": {"anchor": "## 1"}},
|
||||
])
|
||||
planner = make_planner(catalog, monkeypatch, flow_store=store)
|
||||
result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定"))
|
||||
assert result.source == "llm_rag"
|
||||
|
||||
|
||||
def test_archive_reuse(catalog, monkeypatch):
|
||||
"""FR-BIO-023：复用句式直接装载存档链（不调 LLM）。"""
|
||||
archive = {"id": "a1", "name": "XXX", "stage_chain": ["identify", "motif"],
|
||||
"params": {"identify": {"cpu": "8"}}}
|
||||
planner = make_planner(catalog, monkeypatch, llm=FakeLLM(error="unavailable"), archive=archive)
|
||||
result = _run(planner.plan("用 XXX 存档任务的方法帮我跑水稻 SPL 家族鉴定"))
|
||||
assert result.source == "archive"
|
||||
assert result.reuse_archive_id == "a1"
|
||||
assert result.reuse_name == "XXX"
|
||||
assert result.candidates[0].stages == ["identify", "motif"]
|
||||
|
||||
|
||||
def test_all_candidates_invalid_fallback(catalog, monkeypatch):
|
||||
"""LLM 全部候选非法 → 内置兜底（identify 链）。"""
|
||||
llm = FakeLLM(candidates=[
|
||||
{"purpose": "x", "stages": ["nope1"]},
|
||||
{"purpose": "y", "stages": []},
|
||||
])
|
||||
planner = make_planner(catalog, monkeypatch, llm=llm)
|
||||
result = _run(planner.plan("帮我跑水稻 SPL 家族鉴定"))
|
||||
assert result.candidates
|
||||
assert all(catalog.validate_chain(c.stages) == [] for c in result.candidates)
|
||||
@@ -0,0 +1,623 @@
|
||||
"""tests/test_runner.py —— runner 状态机全路径（mock docker 执行器）。
|
||||
|
||||
覆盖（TC-BIO-RUN 系列）：成功推进/失败/重跑/跳过/暂停/恢复/重启续跑/超时/
|
||||
日志截断/并发上限/输入缺失等待/确认等待/自动连续/信号量释放/事件成对。
|
||||
|
||||
注意：runner 内部用 asyncio.create_task 派生任务，测试必须「单场景单 loop」——
|
||||
每个用例在一个新建 event loop 内完成 create→run→断言前的所有异步步骤。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import FakeExecutor, FakeTimedOutExecutor, make_runner
|
||||
from app.services.bioflow.runner import (
|
||||
JOB_DONE,
|
||||
JOB_FAILED,
|
||||
JOB_PAUSED,
|
||||
STAGE_DONE,
|
||||
STAGE_FAILED,
|
||||
STAGE_SKIPPED,
|
||||
STAGE_WAITING_CONFIRM,
|
||||
RunnerError,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.bioflow_unit
|
||||
|
||||
|
||||
def _scenario(coro_factory):
|
||||
"""单 loop 跑完整场景。"""
|
||||
loop = asyncio.new_event_loop()
|
||||
try:
|
||||
return loop.run_until_complete(coro_factory())
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
async def _drain(runner, seconds: float = 5.0) -> None:
|
||||
"""等待 runner 派生的后台任务全部结束。"""
|
||||
while runner._tasks:
|
||||
await asyncio.wait_for(asyncio.gather(*list(runner._tasks)), timeout=seconds)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
def _collected(events, job_id):
|
||||
return [e for e in events.replay(job_id) if e["type"] != "job_recovered"]
|
||||
|
||||
|
||||
def test_full_chain_success(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-RUN-001：全链成功推进（自动连续）。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "SPL 鉴定", ["identify", "phylogeny", "motif"],
|
||||
auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert record.status == JOB_DONE
|
||||
ev_types = [e["type"] for e in _collected(events, record.id)]
|
||||
assert ev_types.count("stage_started") == 3
|
||||
assert ev_types.count("stage_done") == 3
|
||||
assert ev_types[-1] == "job_done"
|
||||
assert ev_types[0] == "job_created"
|
||||
assert all(runner.stage_status(record, s) == STAGE_DONE for s in record.stage_chain)
|
||||
|
||||
|
||||
def test_stage_failure_stops_chain(db, events, catalog):
|
||||
"""TC-BIO-RUN-002：阶段失败转 failed，后续阶段不执行。"""
|
||||
executor = FakeExecutor(exit_code=1, log="boom\n")
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify", "phylogeny", "motif"],
|
||||
auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert record.status == JOB_FAILED
|
||||
assert runner.stage_status(record, "identify") == STAGE_FAILED
|
||||
assert runner.stage_status(record, "phylogeny") == "waiting"
|
||||
ev_types = [e["type"] for e in _collected(events, record.id)]
|
||||
assert "stage_failed" in ev_types
|
||||
failed_ev = [e for e in _collected(events, record.id) if e["type"] == "stage_failed"][0]
|
||||
assert failed_ev["data"]["stage"] == "identify"
|
||||
assert failed_ev["data"]["reason"] == "exit_code"
|
||||
assert [c["stage"] for c in executor.calls] == ["identify"] # 后续阶段未执行
|
||||
|
||||
|
||||
def test_rerun_after_failure(db, events, catalog):
|
||||
"""TC-BIO-RUN-003：失败后 rerun 成功，attempt=2，继续后续阶段。"""
|
||||
executor = FakeExecutor(exit_code=1)
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
assert record.status == JOB_FAILED
|
||||
executor.exit_code = 0
|
||||
await runner.rerun_stage(record, "identify", None)
|
||||
await _drain(runner)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert runner.stage_status(record, "identify") == STAGE_DONE
|
||||
assert record._stage_states["identify"]["attempt"] == 2
|
||||
assert record.status == JOB_DONE # 后续阶段自动推进
|
||||
|
||||
|
||||
def test_skip_after_failure(db, events, catalog):
|
||||
"""TC-BIO-RUN-004：失败后 skip → skipped，后续阶段可执行。"""
|
||||
executor = FakeExecutor(exit_code=1)
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
assert runner.stage_status(record, "identify") == STAGE_FAILED
|
||||
executor.exit_code = 0
|
||||
await runner.skip_stage(record, "identify")
|
||||
await _drain(runner)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert runner.stage_status(record, "identify") == STAGE_SKIPPED
|
||||
ev_types = [e["type"] for e in _collected(events, record.id)]
|
||||
assert "stage_skipped" in ev_types
|
||||
assert record.status == JOB_DONE
|
||||
assert runner.stage_status(record, "phylogeny") == STAGE_DONE
|
||||
|
||||
|
||||
def test_rerun_with_params_goes_confirm(db, events, catalog):
|
||||
"""TC-BIO-RUN-005：带参数重跑 → waiting_confirm 再确认；新参数进入执行。"""
|
||||
executor = FakeExecutor(exit_code=1)
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
await runner.rerun_stage(record, "identify", {"cpu": "8"})
|
||||
assert runner.stage_status(record, "identify") == STAGE_WAITING_CONFIRM
|
||||
executor.exit_code = 0
|
||||
await runner.confirm_stage(record, "identify", {"cpu": "8"})
|
||||
await _drain(runner)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert record._stage_states["identify"]["params"]["cpu"] == "8"
|
||||
assert runner.stage_status(record, "identify") == STAGE_DONE
|
||||
assert executor.calls[-1]["cpu"] == "8"
|
||||
|
||||
|
||||
def test_rerun_non_failed_rejected(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-RUN-024：done 阶段 rerun/skip 拒绝。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
with pytest.raises(RunnerError, match="不可重跑"):
|
||||
_scenario(lambda: runner.rerun_stage(record, "identify", None))
|
||||
with pytest.raises(RunnerError, match="不可跳过"):
|
||||
_scenario(lambda: runner.skip_stage(record, "identify"))
|
||||
|
||||
|
||||
def test_pause_at_stage_boundary(db, events, catalog):
|
||||
"""TC-BIO-RUN-006：暂停在阶段边界（当前阶段完成后停）。"""
|
||||
executor = FakeExecutor(delay=0.05)
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False)
|
||||
task = asyncio.create_task(runner.run_job(record))
|
||||
await asyncio.sleep(0.02) # identify 已进入执行
|
||||
await runner.pause_job(record)
|
||||
await asyncio.wait_for(task, timeout=5)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert record.status == JOB_PAUSED
|
||||
assert runner.stage_status(record, "identify") == STAGE_DONE
|
||||
assert all(c["stage"] != "phylogeny" for c in executor.calls)
|
||||
|
||||
|
||||
def test_resume_continues_without_rerun(db, events, catalog):
|
||||
"""TC-BIO-RUN-007：暂停后恢复续跑，不重跑已完成。"""
|
||||
executor = FakeExecutor(delay=0.02)
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False)
|
||||
task = asyncio.create_task(runner.run_job(record))
|
||||
await asyncio.sleep(0.03)
|
||||
await runner.pause_job(record)
|
||||
await asyncio.wait_for(task, timeout=5)
|
||||
assert record.status == JOB_PAUSED
|
||||
await runner.resume_job(record)
|
||||
await _drain(runner)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert record.status == JOB_DONE
|
||||
identify_calls = [c for c in executor.calls if c["stage"] == "identify"]
|
||||
assert len(identify_calls) == 1 # 不重跑
|
||||
|
||||
|
||||
def test_timeout_fails_stage(db, events, catalog):
|
||||
"""TC-BIO-RUN-012：超时 → failed（reason=timeout）。"""
|
||||
executor = FakeTimedOutExecutor()
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert runner.stage_status(record, "identify") == STAGE_FAILED
|
||||
failed = [e for e in _collected(events, record.id) if e["type"] == "stage_failed"][0]
|
||||
assert failed["data"]["reason"] == "timeout"
|
||||
|
||||
|
||||
def test_log_tail_truncated(db, events, catalog):
|
||||
"""TC-BIO-RUN-013：日志截断只留尾 80 行。"""
|
||||
executor = FakeExecutor(log="\n".join(f"L{i}" for i in range(500)) + "\n")
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
tail = record._stage_states["identify"]["log_tail"]
|
||||
assert tail.count("\n") + 1 == 80
|
||||
assert tail.startswith("L420")
|
||||
assert tail.rstrip().endswith("L499")
|
||||
|
||||
|
||||
def test_exit_nonzero_judged_failed(db, events, catalog):
|
||||
"""TC-BIO-RUN-014：退出码非 0 判失败，log_tail 含 stderr。"""
|
||||
executor = FakeExecutor(exit_code=3, log="error output\n")
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert runner.stage_status(record, "identify") == STAGE_FAILED
|
||||
assert "error output" in record._stage_states["identify"]["log_tail"]
|
||||
|
||||
|
||||
def test_manifest_missing_judged_failed(db, events, catalog):
|
||||
"""TC-BIO-RUN-015：退出码 0 但 manifest 缺 → failed（reason=no_manifest）。"""
|
||||
executor = FakeExecutor(exit_code=0, manifest_status="missing")
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert runner.stage_status(record, "identify") == STAGE_FAILED
|
||||
failed = [e for e in _collected(events, record.id) if e["type"] == "stage_failed"][0]
|
||||
assert failed["data"]["reason"] == "no_manifest"
|
||||
|
||||
|
||||
def test_manifest_failed_status(db, events, catalog):
|
||||
"""TC-BIO-RUN-016：manifest status=failed 判败（不因退出码 0 误判成功）。"""
|
||||
executor = FakeExecutor(exit_code=0, manifest_status="failed")
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert runner.stage_status(record, "identify") == STAGE_FAILED
|
||||
failed = [e for e in _collected(events, record.id) if e["type"] == "stage_failed"][0]
|
||||
assert failed["data"]["reason"] == "manifest_failed"
|
||||
|
||||
|
||||
def test_await_files_when_missing(db, events, catalog):
|
||||
"""TC-BIO-RUN-017：输入缺失等待文件（force_input_check=True）。"""
|
||||
executor = FakeExecutor()
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog, check_inputs=True)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert runner.stage_status(record, "identify") == "await_files"
|
||||
ev = [e for e in _collected(events, record.id) if e["type"] == "stage_await_files"][0]
|
||||
assert ev["data"]["stage"] == "identify"
|
||||
assert ev["data"]["required_files"]
|
||||
assert executor.calls == [] # 不启动 docker
|
||||
|
||||
|
||||
def test_await_files_then_resume(db, events, catalog):
|
||||
"""TC-BIO-RUN-018：文件补齐自动推进。"""
|
||||
executor = FakeExecutor()
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog, check_inputs=True)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
assert runner.stage_status(record, "identify") == "await_files"
|
||||
(record.input_dir / "genome.pep.fa").write_text(">x\nACGT\n", encoding="utf-8")
|
||||
record._stage_states["identify"]["status"] = "waiting"
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert record.status == JOB_DONE
|
||||
assert runner.stage_status(record, "identify") == STAGE_DONE
|
||||
|
||||
|
||||
def test_await_confirm_manual_mode(db, events, catalog):
|
||||
"""TC-BIO-RUN-019/020：手动模式等待确认；确认后执行。"""
|
||||
executor = FakeExecutor()
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify", "phylogeny"], auto_continue=False, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
assert runner.stage_status(record, "identify") == STAGE_WAITING_CONFIRM
|
||||
assert executor.calls == [] # 未确认不执行
|
||||
await runner.confirm_stage(record, "identify", {})
|
||||
await _drain(runner)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
ev = [e for e in _collected(events, record.id) if e["type"] == "stage_await_confirm"][0]
|
||||
assert ev["data"]["stage"] == "identify"
|
||||
assert ev["data"]["form_defaults"] # 表单默认值
|
||||
assert runner.stage_status(record, "identify") == STAGE_DONE
|
||||
assert runner.stage_status(record, "phylogeny") == STAGE_WAITING_CONFIRM
|
||||
|
||||
|
||||
def test_confirm_unknown_param_rejected(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-API-058 对应服务层：confirm 拒绝非 schema 参数。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_start=False)
|
||||
await runner.run_job(record)
|
||||
with pytest.raises(RunnerError, match="未知参数"):
|
||||
await runner.confirm_stage(record, "identify", {"magic": "x"})
|
||||
return runner, record
|
||||
|
||||
_scenario(scenario)
|
||||
|
||||
|
||||
def test_confirm_non_confirmable(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-API-059：重复 confirm 幂等拒绝。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
assert runner.stage_status(record, "identify") == STAGE_DONE
|
||||
with pytest.raises(RunnerError, match="不可确认"):
|
||||
await runner.confirm_stage(record, "identify", {})
|
||||
return runner, record
|
||||
|
||||
_scenario(scenario)
|
||||
|
||||
|
||||
def test_auto_continue_skips_confirm(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-RUN-021：自动连续跳过确认全程推进。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert record.status == JOB_DONE
|
||||
assert "stage_await_confirm" not in [e["type"] for e in _collected(events, record.id)]
|
||||
|
||||
|
||||
def test_concurrency_limit(db, events, catalog):
|
||||
"""TC-BIO-RUN-010：并发上限排队（4 任务并发，同时 running ≤3）。"""
|
||||
executor = FakeExecutor(delay=0.1)
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
records = []
|
||||
for i in range(4):
|
||||
r = await runner.create_job(
|
||||
f"s{i}", f"t{i}", ["identify"], auto_continue=True, auto_start=False)
|
||||
records.append(r)
|
||||
await asyncio.gather(*[runner.run_job(r) for r in records])
|
||||
return runner, records
|
||||
|
||||
runner, records = _scenario(scenario)
|
||||
assert executor.max_running <= 3
|
||||
assert all(r.status == JOB_DONE for r in records)
|
||||
|
||||
|
||||
def test_semaphore_released_on_failure(db, events, catalog):
|
||||
"""TC-BIO-RUN-025：执行失败后信号量释放。"""
|
||||
executor = FakeExecutor(exit_code=1)
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert record.status == JOB_FAILED
|
||||
assert not runner.semaphore.locked()
|
||||
|
||||
|
||||
def test_restart_recovery_no_rerun(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-RUN-008：重启续跑不重跑已完成阶段（attempt 不变）。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False)
|
||||
# 只完成第 1 阶段（模拟中断前状态）
|
||||
await runner._run_stage(record, "identify", 0)
|
||||
assert runner.stage_status(record, "identify") == STAGE_DONE
|
||||
# 模拟重启：新 runner 实例（进程重启后内存清空）
|
||||
runner2 = make_runner(db, events, fake_executor, catalog)
|
||||
recovered = await runner2.recover()
|
||||
await _drain(runner2)
|
||||
return runner2, record, recovered
|
||||
|
||||
runner2, record, recovered = _scenario(scenario)
|
||||
assert any(r.id == record.id for r in recovered)
|
||||
from app.services.bioflow import store
|
||||
st = _scenario(lambda: store.get_stage(db, record.id, "identify"))
|
||||
assert st["status"] == STAGE_DONE
|
||||
assert st["attempt"] == 1 # 不重跑
|
||||
|
||||
|
||||
def test_restart_recovers_inflight_by_manifest(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-RUN-009：重启恢复 in-flight running 阶段按 manifest 判成败。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify", "phylogeny"], auto_continue=True, auto_start=False)
|
||||
await runner._run_stage(record, "identify", 0) # 第 1 阶段完成
|
||||
# phylogeny 标记 running 且无 manifest（模拟崩溃瞬间）
|
||||
record._stage_states["phylogeny"]["status"] = "running"
|
||||
await runner._persist_stage(record, "phylogeny")
|
||||
# 模拟重启
|
||||
runner2 = make_runner(db, events, fake_executor, catalog)
|
||||
await runner2.recover()
|
||||
await _drain(runner2)
|
||||
# 同一 loop 内校验（aiosqlite 连接绑定本 loop）
|
||||
from app.services.bioflow import store
|
||||
st = await store.get_stage(db, record.id, "phylogeny")
|
||||
assert st["status"] == STAGE_DONE
|
||||
assert st["attempt"] >= 1 # 无 manifest → 重跑；attempt 至少 1
|
||||
return runner2, record
|
||||
|
||||
_scenario(scenario)
|
||||
|
||||
|
||||
def test_restart_skips_archived(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-RUN-030：恢复时跳过已归档任务。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
# 归档（状态改为 archived）
|
||||
await runner._persist_job(record)
|
||||
from app.services.bioflow.store import update_job_status
|
||||
await update_job_status(db, record.id, "archived")
|
||||
runner2 = make_runner(db, events, fake_executor, catalog)
|
||||
recovered = await runner2.recover()
|
||||
return recovered
|
||||
|
||||
recovered = _scenario(scenario)
|
||||
assert recovered == []
|
||||
|
||||
|
||||
def test_empty_stages_done_immediately(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-RUN-039：空阶段任务直接 done，无 docker 调用。"""
|
||||
# 注意：创建任务要求 stages 非空（pydantic min_length=1）由路由层保证；
|
||||
# 服务层直接构造空链验证防御
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
record.stage_chain = []
|
||||
record._stage_states = {}
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert record.status == JOB_DONE
|
||||
assert fake_executor.calls == []
|
||||
assert record.progress() == 100
|
||||
|
||||
|
||||
def test_progress_calculation(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-RUN-029：任务进度数值。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify", "phylogeny", "motif"],
|
||||
auto_continue=True, auto_start=False)
|
||||
assert record.progress() == 0
|
||||
# 全链跑完 → 100（进度公式按完成阶段数/总数）
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
assert record.progress() == 100
|
||||
|
||||
|
||||
def test_unknown_stage_create_rejected(db, events, catalog, fake_executor):
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
with pytest.raises(RunnerError, match="未知阶段"):
|
||||
await runner.create_job("s1", "t", ["ghost_stage"], auto_start=False)
|
||||
return None
|
||||
|
||||
_scenario(scenario)
|
||||
|
||||
|
||||
def test_prerequisite_autofill(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-RUN-034：前置补全后链完整（phylogeny 缺 identify → 自动补）。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job("s1", "t", ["phylogeny", "motif"], auto_start=False)
|
||||
return record
|
||||
|
||||
record = _scenario(scenario)
|
||||
assert record.stage_chain[0] == "identify"
|
||||
assert set(record.stage_chain) == {"identify", "phylogeny", "motif"}
|
||||
|
||||
|
||||
def test_multi_job_isolation(db, events, catalog):
|
||||
"""TC-BIO-RUN-028：多任务互不串扰。"""
|
||||
executor = FakeExecutor(log="job1 log\n")
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, executor, catalog)
|
||||
r1 = await runner.create_job("s1", "t1", ["identify"], auto_continue=True, auto_start=False)
|
||||
r2 = await runner.create_job("s2", "t2", ["identify"], auto_continue=True, auto_start=False)
|
||||
await asyncio.gather(runner.run_job(r1), runner.run_job(r2))
|
||||
return runner, r1, r2
|
||||
|
||||
runner, r1, r2 = _scenario(scenario)
|
||||
assert r1.id != r2.id
|
||||
assert r1.status == JOB_DONE and r2.status == JOB_DONE
|
||||
assert r1._stage_states["identify"]["log_tail"] == "job1 log" # tail_lines 去尾换行
|
||||
|
||||
|
||||
def test_stage_log_event(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-SSE-004：stage_log 事件随执行发布（log 非空）。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
log_events = [e for e in _collected(events, record.id) if e["type"] == "stage_log"]
|
||||
assert log_events
|
||||
assert all(e["data"]["log"] for e in log_events)
|
||||
|
||||
|
||||
def test_event_seq_strictly_increasing(db, events, catalog, fake_executor):
|
||||
"""TC-BIO-SSE-013：seq 严格递增。"""
|
||||
|
||||
async def scenario():
|
||||
runner = make_runner(db, events, fake_executor, catalog)
|
||||
record = await runner.create_job(
|
||||
"s1", "t", ["identify"], auto_continue=True, auto_start=False)
|
||||
await runner.run_job(record)
|
||||
return runner, record
|
||||
|
||||
runner, record = _scenario(scenario)
|
||||
seqs = [e["seq"] for e in _collected(events, record.id)]
|
||||
assert seqs == sorted(seqs)
|
||||
assert len(set(seqs)) == len(seqs)
|
||||
@@ -0,0 +1,167 @@
|
||||
# report_bioflow_dev2 —— P1 生信服务地基（开发 agent A2）
|
||||
|
||||
> 任务书：docs/team/brief_bioflow_dev2_p1.md
|
||||
> 工作分支：feature/bioflow-conversation-orchestration（保持，不切换）
|
||||
> 纪律：不 commit/push/部署；不碰生产服务（8090/8091）；服务层零 pydantic；零硬编码
|
||||
|
||||
## 1. 交付概览
|
||||
|
||||
| 模块 | 说明 |
|
||||
|---|---|
|
||||
| bioflow_service/（新增，独立部署树） | 生信服务 8098：FastAPI + 独立 SQLite + docker 执行器 + 状态机 + 规划器 |
|
||||
| sdk/app/routers/bioflow_proxy.py（新增） | BFF 流式反代 + 报告登记端点 |
|
||||
| sdk/app/main.py（2 行改动） | BIOFLOW_ENABLED 开关注册反代路由 |
|
||||
| sdk/app/config.py（2 键） | bioflow_enabled（默认 false）/ bioflow_service_url |
|
||||
| docs/team/report_bioflow_dev2.md | 本报告 |
|
||||
|
||||
## 2. 模块清单（bioflow_service/app/）
|
||||
|
||||
**明文白名单（编译管线保留 .py）**
|
||||
- `config.py` —— BIOFLOW_ 前缀配置（pydantic-settings）：image_map_path（仓库内
|
||||
bioflow_kernel/stage_image_map.json，按 __file__ 推导 repo 根）、data_root/
|
||||
jobs_dir/db_path/archives_dir/flow_docs_dir（全部可覆盖，无写死宿主路径）、
|
||||
concurrency=3、stage_timeout_sec、host_uid_env=BIOFLOW_HOST_UID（默认服务进程
|
||||
os.getuid() 推导）、container_data_dir=/bioflow-data、upload 白名单、DOCGEN_LLM_*
|
||||
沿用 docgen 直连模式；bba_env 属性从配置推导 BBA_PFAM_DIR/BBA_RAG_MAPPING_FILE。
|
||||
- `schemas.py` —— pydantic 请求/响应模型（ChatStreamRequest/JobCreateRequest/
|
||||
StageConfirmRequest/ArchiveCreateRequest 等 10 个；STAGE/JOB 状态 Literal）。
|
||||
- `db.py` —— 独立 SQLite（WAL）三表 + 幂等迁移（CREATE IF NOT EXISTS + PRAGMA
|
||||
table_info 增量列）：bio_jobs/bio_stages/bio_archives（字段与实施计划 §六对齐）；
|
||||
bio_stages UNIQUE(job_id, stage)，job 删除级联。
|
||||
- `main.py` —— FastAPI 组装 + lifespan（init_db→init_context→recover→shutdown）；
|
||||
health 端点 /api/bioflow/health（db 连通状态字段，FR-BIO-025 探活用）。
|
||||
|
||||
**服务层（全部 dataclass/dict/stdlib，无 pydantic，Cython 编译）**
|
||||
- `services/bioflow/kernel.py` —— Docker 执行器：读 A1 stage_image_map.json
|
||||
（75 阶段/8 镜像），按阶段→镜像拼 docker run（--user uid/三目录挂载/BIOFLOW_DATA
|
||||
只读/-e BBA_*/--cpus/--memory/--rm + `python -B -m cli.run_{stage}`，list 无 shell）；
|
||||
cpu/memory 白名单回落；超时 terminate→kill；日志落任务 logs/ + 尾 80 行截断；
|
||||
跨环境 note 透传（编排层 P2 拆分依据）。
|
||||
- `services/bioflow/runner.py` —— 任务状态机：created→await_inputs→running→
|
||||
paused/done/failed/archived；阶段 waiting→await_files→waiting_confirm→queued→
|
||||
running→done/failed/skipped；输入检查（FR-BIO-005）→确认门（FR-BIO-008，confirm
|
||||
`_confirmed` 旁路防二次确认门）→并发信号量（BIOFLOW_CONCURRENCY 默认 3，
|
||||
FIFO 排队）→docker→manifest 判成败（no_manifest/manifest_failed/timeout/
|
||||
exit_code 四因）；pause 停阶段边界、resume 不跳过确认、rerun/skip 状态校验；
|
||||
重启恢复 recover()：running 阶段按 manifest 判成败（有 success manifest→done
|
||||
否则重跑）、done 不重跑（attempt 不变）、archived 跳过。
|
||||
- `services/bioflow/catalog.py` —— 阶段目录（映射表驱动 + STAGE_INFO/STAGE_DEPENDENCIES
|
||||
静态快照，不运行时反射）；73 阶段（真实表 75 去重复校验）、基因家族链 21 阶段、
|
||||
手写参数 schema（identify 11 参数含 default/source/help，source 标注
|
||||
contract/user/runtime 供表单默认值来源可见）；validate_chain 未知阶段拒绝 +
|
||||
auto_fill_prerequisites 前置补全（拓扑排序依赖在前）。
|
||||
- `services/bioflow/planner.py` —— 路径规划：复用句式（「用 XXX 存档任务的方法…」）
|
||||
→ archive 直接装载；否则知识库检索（vstore/flow.py FlowStore 本地实例混合检索，
|
||||
失败降级 llm_only）+ LLM 候选（DOCGEN_LLM_* 直连，JSON 模式 + 修复重试）→
|
||||
catalog 硬校验（未知阶段拒绝、前置补全）；PLAN_LLM_TIMEOUT/PLAN_UNAVAILABLE
|
||||
降级内置兜底链。
|
||||
- `services/bioflow/manifests.py` —— manifest 解析校验（manifest_v1.schema 必需字段；
|
||||
定位 final/{prefix}.{stage}.manifest.json，无兜底防跨阶段误配）；跨阶段输入定位
|
||||
（artifacts full_path 解析 + 存在性检查）。
|
||||
- `services/bioflow/events.py` —— 事件总线（12 类事件全集 + job_recovered）：
|
||||
进程内 pub/sub（SSE 订阅）+ 内存环形缓冲 + events/{job_id}.jsonl 持久
|
||||
（重启回放）+ snapshot 首帧 + replay(after_seq) 断线回放。
|
||||
- `services/bioflow/llm.py` —— LLM 直连（docgen 同款：指数退避 + JSON 修复 +
|
||||
LLMTimeout/LLMUnavailable 分型）。
|
||||
- `services/bioflow/params.py` —— 参数预填（契约文档参数解析 + BBA_* 默认值标注 +
|
||||
存档参数预填）。
|
||||
- `services/bioflow/reports.py` —— 报告登记（报告 HTML 定位/fig:// 图清单提取/
|
||||
登记载荷（job_id/标题/md/图清单）+ 内部 HTTP 调 BFF 反代端点，BFF 不可达降级）。
|
||||
- `services/bioflow/archive.py` —— 存档打包（final/结果+logs/过程+events.jsonl
|
||||
过程记录+input 原始文件；include_intermediate 开关含中间文件；zip 下载）。
|
||||
- `services/bioflow/store.py` —— 三表 CRUD（upsert/list/get/delete；归档精确优先
|
||||
模糊匹配、reused_count 计数）。
|
||||
- `services/bioflow/intent.py` —— 意图分流（cases_bioflow_intent.md 冻结词典同表：
|
||||
export→edit→docgen→附件 docgen→bioflow_qa（FLOW_QA 命中即拦）→bioflow
|
||||
（动词+家族语境词/分析对象）→qa 漏判安全；复用方法名提取 3-20 字符；
|
||||
>2000 字符截断判定）。
|
||||
|
||||
**路由**
|
||||
- `routers/bioflow.py` —— health/catalog（?stage=/?chain=gene_family）/
|
||||
chat/stream（SSE：意图分流→plan_ready 候选→心跳；plan seq 编码 data 载荷直出）/
|
||||
ask（知识库引用块：source/title_path/path/cite/line/text）。
|
||||
- `routers/biojobs.py` —— jobs 全套（POST 201/列表 rowid 倒序分页过滤/详情/删除
|
||||
running 409）；files（multipart 流式逐块写盘 + 扩展名白名单 + 大小上限 413 +
|
||||
server_path 登记限 data_root/任务目录白名单 403）；stages confirm（参数校验
|
||||
未知 422）/rerun（带参数→waiting_confirm）/skip/pause/resume/manifest（manifest
|
||||
对象 + 日志尾 80 行）；events SSE（snapshot 首帧 + 回放 + 实时订阅 + 任务终态
|
||||
关闭 TC-BIO-API-081）；report revise（最小接口：无图目标 422/绘图阶段定位）。
|
||||
- `routers/bioarchives.py` —— archives 全套（存档 201 重名 409/列表/详情/删除
|
||||
磁盘+行/run 复用/按名匹配 404 ARCHIVE_NOT_FOUND/download zip/report HTML）；
|
||||
reports/register（归一化 + 图清单回填，BFF 反代落 doc_artifacts）。
|
||||
|
||||
**构建与运维**
|
||||
- `build.sh` —— 复制 tooling/compile/compile_sdk.sh 改白名单（明文：main.py/
|
||||
config.py/schemas.py/db.py + __init__.py 规则；schemas= pydantic cyfunction 教训、
|
||||
db= 断流崩溃根治教训）；16 模块 Cython+GCC 全过（实测 /tmp 输出 16 .so，
|
||||
编译树 import OK）。
|
||||
- `restart.sh` —— 8098 重启（零硬编码：路径脚本位置推导 + 环境变量覆盖）。
|
||||
- `tests/` —— pytest（-m bioflow_unit 127 用例 / -m bioflow_api 39 用例）。
|
||||
|
||||
## 3. 自测结果（2026-09-06，服务器 venv 3.12.13）
|
||||
|
||||
| 命令 | 结果 |
|
||||
|---|---|
|
||||
| `python -m pytest tests/ -q` | **166 passed**（15.7s，0 warnings） |
|
||||
| `python -m pytest tests/ -q -m bioflow_unit` | 127 passed |
|
||||
| `python -m pytest tests/ -q -m bioflow_api` | 39 passed |
|
||||
| build.sh → /tmp 编译 + 编译树 import | 16 模块 .so，import OK |
|
||||
| BFF BIOFLOW_ENABLED=false/true 双态 import | 均 OK（反代路由 2 条） |
|
||||
| 真实 stage_image_map.json 加载 | 75 阶段/8 镜像/identify 11 参数/命令拼装正确 |
|
||||
| /home/xzq 硬编码扫描（app+tests） | 0 处 |
|
||||
|
||||
全部 mock docker/LLM/vstore（tmpdir + fake 执行器），不连任何生产服务
|
||||
（8090/8091/8094/8098 从未探测）。
|
||||
|
||||
## 4. 已知限制（交 TL 裁决）
|
||||
|
||||
1. **SSE 传输层测试法**：本环境 fastapi TestClient 的 sync stream 包装与 httpx
|
||||
ASGITransport 对永不结束的 SSE 流均存在首帧不达/整流缓冲问题（dbg3/dbg6 最小
|
||||
复现实证），SSE 用例改为端点直调迭代 body_iterator + 模块级共享 event loop
|
||||
（anyio portal 绑定首个 loop，跨 loop 死锁实证）。生产部署（uvicorn 真实网络
|
||||
SSE）不受影响；如 TL 要求 TestClient 原生流测试，需另行排查 fastapi/anyio
|
||||
版本组合。
|
||||
2. **catalog 阶段数口径**：任务书口径 73；A1 交付 stage_image_map.json 实为 75
|
||||
阶段（含 module=null 的 synteny_plot_revise 与 executable=false 的 msa/
|
||||
enrichment）。kernel 对 module=null 阶段拼命令前置拦截（UnknownStageError），
|
||||
catalog 全量返回 75。以 A1 交付物为准，口径差异记录备查。
|
||||
3. **gene_family 链为 21 阶段**（A1 family_chain 实测 21，任务书口径约 18）；
|
||||
以 stage_image_map.json family_chain 字段为唯一事实源。
|
||||
4. **FlowStore 生产数据未入库**：vstore/data/flow_docs 目录当前不存在（A4 自测
|
||||
在 tmpdir 完成；真实语料 106 文件/724 块需 TL 部署时跑
|
||||
vstore/scripts/ingest_flow_docs.py 入库）。planner 已按本地实例对接（加载成功
|
||||
零块时检索空 → LLM 候选照常，降级链完整）；生产前入库即可获得 KB 增强
|
||||
（llm_rag source）。
|
||||
5. **chat/stream 是规划入口**：plan_ready 后保持心跳连接，任务创建走
|
||||
POST /jobs（P1 契约）；会话级任务事件经 /jobs/{id}/events 订阅。若 TL 要求
|
||||
chat/stream 单流贯穿任务事件（A3 前端形态），需 P2 加会话级订阅联动。
|
||||
6. **报告登记 BFF 侧 doc_artifacts 落库**：bioflow_proxy.register_report 字段
|
||||
（job_id/session_id/标题/md_path/图清单）已按约定实现；图清单字段当前存于
|
||||
请求体（doc_artifacts 表无 figures 列，如需持久化图清单需 TL 裁决加列——已
|
||||
在 payload 透传中保留）。与 A3 report 模板的 fig:// 渲染协调点：报告 HTML 内
|
||||
fig://id 引用清单由生信服务 reports.extract_figure_refs 提供。
|
||||
7. **意图词典判定为启发式快照**：与 cases_bioflow_intent.md 冻结表逐条对齐
|
||||
（40 用例表全过）；家族名语境词规则（「SPL 会议」不命中）为词典实现注意第
|
||||
1/4 条的工程化解读，最终同表双跑一致性以 TC-BIO-INTENT-047（L2 用例）验收。
|
||||
8. **事件持久化目录**：events/{job_id}.jsonl 与任务目录分离（bioflow_data/
|
||||
jobs/events/）；归档时打包进档案根。任务删除不清理事件文件（留审计），
|
||||
如需一并删除交 TL 裁决。
|
||||
9. **镜像 tag 后缀**：docker_image_tag_suffix 默认 "dev"（A1 交付物 :dev）；
|
||||
发布流水线（P2 FR-BIO-032 compile_bioflow.sh）需按终态 tag 覆盖环境变量。
|
||||
10. **pytest.ini python_files 陷阱**：本仓库 pytest 9.1.1 下 `python_files =
|
||||
["test_*.py"]`（列表形式）会导致目录级收集 0 用例（单文件显式传参正常），
|
||||
已用字符串形式 `python_files = test_*.py` 规避并实证；写报告备案防踩。
|
||||
|
||||
## 5. 交付物清单
|
||||
|
||||
| 文件 | 说明 |
|
||||
|---|---|
|
||||
| bioflow_service/app/{config,schemas,db,main,context}.py | 应用骨架（4 明文 + context） |
|
||||
| bioflow_service/app/services/bioflow/*.py | 服务层 12 模块（零 pydantic） |
|
||||
| bioflow_service/app/routers/{bioflow,biojobs,bioarchives}.py | 端点三层 |
|
||||
| bioflow_service/build.sh / restart.sh | 编译白名单管线 / 重启脚本 |
|
||||
| bioflow_service/pytest.ini | markers + python_files（字符串形式陷阱规避） |
|
||||
| bioflow_service/tests/*.py | 10 文件 166 用例 |
|
||||
| sdk/app/routers/bioflow_proxy.py | BFF 流式反代 + 报告登记 |
|
||||
| sdk/app/main.py（+2 行）/ sdk/app/config.py（+2 键） | BFF 最小改动 |
|
||||
| docs/team/report_bioflow_dev2.md | 本报告 |
|
||||
@@ -72,6 +72,10 @@ class Settings(BaseSettings):
|
||||
bff_history_max_question_chars: int = 4000 # 单条 user 文本截断
|
||||
bff_history_max_answer_chars: int = 12000 # 单条 assistant 答案截断
|
||||
|
||||
# ---- 生信服务反代（FR-BIO-027/028；BIOFLOW_ENABLED 为假时反代路由不注册）----
|
||||
bioflow_enabled: bool = False # 三形态部署开关（默认关：只部署 rag 无生信痕迹）
|
||||
bioflow_service_url: str = "http://127.0.0.1:8098" # 生信服务 8098（BFF 流式反代目标）
|
||||
|
||||
# ---- 生产形态：BFF 静态托管 ui/dist（ARCHITECTURE.md §3，M6 挂载）----
|
||||
frontend_dist_dir: str = "ui/dist" # 生产形态：静态托管 ui/dist
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ from fastapi.staticfiles import StaticFiles
|
||||
from .config import get_settings
|
||||
from .db import close_db, init_db
|
||||
from .routers import capabilities, docgen, extraction, figures, health, kg, legacy, qa, sessions
|
||||
from .routers import bioflow_proxy
|
||||
from .upstream import close_client
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
@@ -54,6 +55,8 @@ def create_app() -> FastAPI:
|
||||
app.include_router(figures.router)
|
||||
app.include_router(legacy.router)
|
||||
app.include_router(docgen.router) # docgen（P2）
|
||||
if get_settings().bioflow_enabled:
|
||||
app.include_router(bioflow_proxy.router) # 生信反代（BIOFLOW_ENABLED 开关，FR-BIO-028）
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def _validation_handler(_: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
"""routers/bioflow_proxy.py —— BFF 生信反代（FR-BIO-027/028：BFF 仅流式反代 + 报告登记）。
|
||||
|
||||
- 流式反代 /api/bioflow/* → BIOFLOW_SERVICE_URL（SSE 不缓冲透传：
|
||||
httpx client.stream 逐块 yield；反代请求超时阶梯——上传/执行类端点单独调大）。
|
||||
- 报告登记端点 /api/bioflow/reports/register：BFF 侧落 doc_artifacts
|
||||
（字段与 docgen 现有表对齐：job_id/标题/md/图清单——与 A3 report 模板协调），
|
||||
落库后透传生信服务归一化响应。
|
||||
- BIOFLOW_ENABLED 为假时路由不注册（main.py 按开关 include；三形态部署）。
|
||||
|
||||
约束：
|
||||
- 不缓冲 SSE（TC-BIO-API-102 首帧延迟要求）。
|
||||
- 反代失败统一 502 UPSTREAM_DOWN 信封（经 app.upstream.envelope）。
|
||||
- 本文件是任务书列出的 BFF 三处最小改动之一。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from ..config import get_settings
|
||||
from ..db import Database, get_db
|
||||
from ..upstream import envelope
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/bioflow", tags=["bioflow-proxy"])
|
||||
|
||||
# 反代超时阶梯（秒）：普通读 120；上传/报告登记等大体积端点单独调大
|
||||
PROXY_READ_TIMEOUT = 120.0
|
||||
PROXY_UPLOAD_TIMEOUT = 3600.0
|
||||
PROXY_CONNECT_TIMEOUT = 5.0
|
||||
|
||||
_LONG_TIMEOUT_PATHS = ("/files", "/reports/register", "/archives", "/download")
|
||||
|
||||
|
||||
def _proxy_timeout(path: str) -> float:
|
||||
if any(seg in path for seg in _LONG_TIMEOUT_PATHS):
|
||||
return PROXY_UPLOAD_TIMEOUT
|
||||
return PROXY_READ_TIMEOUT
|
||||
|
||||
|
||||
def _service_base() -> str:
|
||||
return get_settings().bioflow_service_url.rstrip("/")
|
||||
|
||||
|
||||
async def _proxy(request: Request, path: str) -> Any:
|
||||
"""httpx 流式反代（SSE 透传不缓冲）；失败→502 信封。"""
|
||||
import httpx
|
||||
|
||||
base = _service_base()
|
||||
url = f"{base}/api/bioflow/{path.lstrip('/')}"
|
||||
if request.url.query:
|
||||
url = f"{url}?{request.url.query}"
|
||||
body = await request.body()
|
||||
headers = {k: v for k, v in request.headers.items()
|
||||
if k.lower() not in ("host", "content-length", "connection")}
|
||||
timeout = httpx.Timeout(connect=PROXY_CONNECT_TIMEOUT, read=_proxy_timeout(path),
|
||||
write=PROXY_UPLOAD_TIMEOUT, pool=5.0)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
req = client.build_request(
|
||||
request.method, url, content=body, headers=headers,
|
||||
)
|
||||
upstream = await client.send(req, stream=True)
|
||||
except httpx.HTTPError as exc:
|
||||
logger.warning("bioflow_proxy: 生信服务不可达 %s: %s", url, exc)
|
||||
payload, status = envelope("UPSTREAM_DOWN", message="生信服务当前不可用，请稍后重试",
|
||||
degraded=True, status=503)
|
||||
return JSONResponse(status_code=status, content=payload)
|
||||
|
||||
content_type = upstream.headers.get("content-type", "")
|
||||
if "text/event-stream" in content_type:
|
||||
# SSE 透传：逐块 yield，绝不缓冲
|
||||
async def stream():
|
||||
try:
|
||||
async for chunk in upstream.aiter_raw():
|
||||
yield chunk
|
||||
finally:
|
||||
await upstream.aclose()
|
||||
|
||||
resp_headers = {
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
return StreamingResponse(stream(), status_code=upstream.status_code,
|
||||
media_type="text/event-stream", headers=resp_headers)
|
||||
|
||||
try:
|
||||
content = await upstream.aread()
|
||||
await upstream.aclose()
|
||||
except httpx.HTTPError:
|
||||
payload, status = envelope("UPSTREAM_DOWN", message="生信服务响应中断",
|
||||
degraded=True, status=502)
|
||||
return JSONResponse(status_code=status, content=payload)
|
||||
media_type = upstream.headers.get("content-type") or "application/json"
|
||||
return JSONResponse(status_code=upstream.status_code, content=content,
|
||||
media_type=media_type)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 报告登记（BFF 侧落 doc_artifacts；FR-BIO-016/104 字段与 docgen 表对齐）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/reports/register")
|
||||
async def register_report(request: Request, db: Database = Depends(get_db)) -> Any:
|
||||
"""生信报告 → doc_artifacts 行（BFF 拥有 docgen 表）。
|
||||
|
||||
先透传生信服务归一化响应（生信服务返回 {ok,data} 归一化载荷含图清单回填），
|
||||
再落 doc_artifacts。落库失败不阻断反代响应（报告仍可直连下载；记日志）。
|
||||
"""
|
||||
try:
|
||||
payload = await request.json()
|
||||
except Exception:
|
||||
payload, status = envelope("VALIDATION", message="请求体须为 JSON", status=422)
|
||||
return JSONResponse(status_code=status, content=payload)
|
||||
|
||||
# 先透传生信服务归一化（含图清单回填）
|
||||
proxied = await _proxy(request, "reports/register")
|
||||
|
||||
# BFF 落 doc_artifacts（尽力而为；字段对齐 docgen 表）
|
||||
try:
|
||||
import uuid
|
||||
|
||||
data = payload.get("data", payload) if isinstance(payload, dict) else {}
|
||||
artifact_id = uuid.uuid4().hex[:16]
|
||||
await db.execute(
|
||||
"""INSERT INTO doc_artifacts (id, session_id, status, title, template_name,
|
||||
md_path, docx_path, pdf_path, error, created_at)
|
||||
VALUES (?, ?, 'done', ?, 'bioflow_report', ?, NULL, NULL, NULL, ?)""",
|
||||
(
|
||||
artifact_id,
|
||||
str(data.get("session_id") or ""),
|
||||
str(data.get("title") or "生信分析报告"),
|
||||
str(data.get("md_path") or ""),
|
||||
_now_iso(),
|
||||
),
|
||||
)
|
||||
logger.info("bioflow_proxy: 生信报告登记 doc_artifacts 行 %s（job=%s）",
|
||||
artifact_id, data.get("job_id"))
|
||||
except Exception: # noqa: BLE001 - 落库失败不阻断（docgen 表可能未建）
|
||||
logger.exception("bioflow_proxy: doc_artifacts 落库失败（报告仍直连可用）")
|
||||
|
||||
return proxied
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 通用反代路由（其余 /api/bioflow/* 全部流式透传）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
|
||||
async def proxy_all(path: str, request: Request) -> Any:
|
||||
return await _proxy(request, path)
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
from datetime import datetime, timezone
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
Reference in New Issue
Block a user