adminandClaude Fable 5
dcb810a758
- stage_image_map 加 26 个 wgs/pop/gwas 阶段 cli_contract:
--workspace/--job-id/--prefix 公共三元组 + 上游 manifest 注入
(prev_manifest:<stage> 解析为 workspace/final/{prefix}.{stage}.manifest.json)
+ param:<name> 用户参数 + tool:<name> 镜像内工具
- kernel _contract_args 扩展 prev_manifest/tool 两种 source
- catalog 补 wgs/pop/gwas 参数 schema(确认表单收集参考基因组/样本清单/
表型文件/主成分等)
- 339 用例全绿;已部署 176
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
663 lines
36 KiB
Python
663 lines
36 KiB
Python
"""app/services/bioflow/catalog.py —— 阶段目录（stage_image_map.json + STAGE_INFO 移植）。
|
||
|
||
- 阶段列表/参数 schema/镜像映射的权威来源：A1 的 stage_image_map.json（不运行时反射）。
|
||
- 中文名/预计耗时：从内核 gfa_core/mas/stage_catalog.py STAGE_INFO 移植的静态快照
|
||
（同一仓库内文件，加载失败降级为空，不阻断）。
|
||
- 前置依赖：从内核 STAGE_DEPENDENCIES 移植（供 planner 前置补全）。
|
||
|
||
服务层纪律：纯 dataclass/dict/stdlib，无 pydantic。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
import re
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# ---- 从内核 stage_catalog.py 移植的静态快照（2026-09-06，A1 交付树）----
|
||
|
||
# 阶段中文名 + 预计耗时（分钟）
|
||
STAGE_INFO_SNAPSHOT: dict[str, dict[str, Any]] = {
|
||
"wgs_qc": {"name": "WGS质控", "estimated_time": 30},
|
||
"wgs_index": {"name": "WGS参考索引", "estimated_time": 15},
|
||
"wgs_align": {"name": "WGS序列比对", "estimated_time": 120},
|
||
"wgs_markdup": {"name": "WGS去重复", "estimated_time": 90},
|
||
"wgs_bam_qc": {"name": "WGS BAM质控", "estimated_time": 30},
|
||
"wgs_haplotype_calling": {"name": "WGS单样本变异检测", "estimated_time": 120},
|
||
"wgs_gvcf_combine": {"name": "WGS GVCF合并", "estimated_time": 60},
|
||
"wgs_joint_genotyping": {"name": "WGS联合分型", "estimated_time": 90},
|
||
"wgs_variant_select": {"name": "WGS变异类型选择", "estimated_time": 30},
|
||
"wgs_variant_filter": {"name": "WGS变异硬过滤", "estimated_time": 30},
|
||
"wgs_impute": {"name": "WGS基因型填充", "estimated_time": 60},
|
||
"wgs_maf_missing_filter": {"name": "WGS MAF与缺失率过滤", "estimated_time": 30},
|
||
"pop_pca": {"name": "群体PCA分析", "estimated_time": 30},
|
||
"pop_phylogeny": {"name": "群体系统树分析", "estimated_time": 45},
|
||
"pop_structure": {"name": "群体结构分析", "estimated_time": 90},
|
||
"pop_ld_decay": {"name": "群体LD衰减分析", "estimated_time": 60},
|
||
"pop_snp_density": {"name": "SNP密度分析", "estimated_time": 20},
|
||
"gwas_fmt": {"name": "GWAS VCF排序与HapMap转换", "estimated_time": 20},
|
||
"gwas_kinship": {"name": "GWAS TASSEL亲缘关系矩阵", "estimated_time": 20},
|
||
"gwas_trait": {"name": "GWAS表型数据准备", "estimated_time": 5},
|
||
"gwas_cov": {"name": "GWAS群体结构协变量准备", "estimated_time": 5},
|
||
"gwas_mlm": {"name": "GWAS TASSEL MLM/CMLM", "estimated_time": 60},
|
||
"gwas_plot": {"name": "GWAS MLM结果绘图", "estimated_time": 10},
|
||
"gwas_emmax": {"name": "GWAS EMMAX关联分析", "estimated_time": 60},
|
||
"gwas_emplot": {"name": "GWAS EMMAX结果绘图", "estimated_time": 10},
|
||
"gwas_glm": {"name": "GWAS TASSEL GLM", "estimated_time": 60},
|
||
"identify": {"name": "基因家族鉴定", "estimated_time": 5},
|
||
"identify_refine": {"name": "修订鉴定成员", "estimated_time": 5},
|
||
"subfamily_confirm": {"name": "确认亚家族分类", "estimated_time": 3},
|
||
"phylogeny": {"name": "系统发育分析", "estimated_time": 30},
|
||
"reference_classification": {"name": "参考亚家族分类", "estimated_time": 5},
|
||
"motif": {"name": "Motif分析", "estimated_time": 20},
|
||
"domain": {"name": "结构域分析", "estimated_time": 15},
|
||
"structure": {"name": "基因结构分析", "estimated_time": 10},
|
||
"promoter": {"name": "启动子分析", "estimated_time": 20},
|
||
"tfbs": {"name": "TF结合位点预测", "estimated_time": 30},
|
||
"synteny": {"name": "种内共线性分析", "estimated_time": 30},
|
||
"karyotype": {"name": "核型分析", "estimated_time": 10},
|
||
"interspecies_synteny": {"name": "物种间共线性", "estimated_time": 40},
|
||
"expression": {"name": "表达分析", "estimated_time": 15},
|
||
"kaks": {"name": "Ka/Ks分析", "estimated_time": 20},
|
||
"ppi": {"name": "PPI分析", "estimated_time": 10},
|
||
"pep": {"name": "蛋白特征分析", "estimated_time": 10},
|
||
"ortholog": {"name": "直系同源基因鉴定", "estimated_time": 15},
|
||
"msa": {"name": "多序列比对", "estimated_time": 10},
|
||
"enrichment": {"name": "富集分析", "estimated_time": 15},
|
||
"combo": {"name": "组合图", "estimated_time": 10},
|
||
"gene_family_report": {"name": "基因家族结果报告", "estimated_time": 2},
|
||
"genome_prepare": {"name": "基因组准备", "estimated_time": 10},
|
||
"ncbi_fetch_sequences": {"name": "NCBI序列下载", "estimated_time": 5},
|
||
"sequence_extract_by_id": {"name": "按ID提取序列", "estimated_time": 5},
|
||
"msa_region_plot": {"name": "MSA区间绘图", "estimated_time": 3},
|
||
"synteny_plot_revise": {"name": "种内共线性Circos重绘", "estimated_time": 3},
|
||
"promoter_plot_revise": {"name": "启动子自定义绘图", "estimated_time": 3},
|
||
"prepare_organize": {"name": "数据文件整理", "estimated_time": 5},
|
||
"prepare_normalize_gff": {"name": "GFF ID规范化", "estimated_time": 5},
|
||
"prepare_longest_transcript": {"name": "提取最长转录本", "estimated_time": 10},
|
||
"prepare_extract_pep": {"name": "从GFF提取PEP", "estimated_time": 10},
|
||
"prepare_extract_cds": {"name": "从GFF提取CDS", "estimated_time": 10},
|
||
"prepare_filter_fasta": {"name": "按ID提取FASTA", "estimated_time": 5},
|
||
"prepare_simplify_ncbi": {"name": "NCBI ID简化", "estimated_time": 5},
|
||
"rnaseq_qc": {"name": "RNA-seq质控", "estimated_time": 30},
|
||
"rnaseq_align": {"name": "RNA-seq比对", "estimated_time": 60},
|
||
"rnaseq_quant": {"name": "RNA-seq定量", "estimated_time": 30},
|
||
"rnaseq_sample_qc": {"name": "RNA-seq样本质量评估", "estimated_time": 10},
|
||
"rnaseq_de": {"name": "差异表达分析（自动生成FPKM表达图）", "estimated_time": 20},
|
||
"rnaseq_wgcna": {"name": "RNA-seq WGCNA分析", "estimated_time": 30},
|
||
"annotation": {"name": "背景注释", "estimated_time": 180},
|
||
"rnaseq_go_enrich": {"name": "RNA-seq GO富集分析", "estimated_time": 15},
|
||
"rnaseq_kegg_enrich": {"name": "RNA-seq KEGG富集分析", "estimated_time": 15},
|
||
"rnaseq_enrich": {"name": "RNA-seq富集分析", "estimated_time": 15},
|
||
"rnaseq_fgsea": {"name": "RNA-seq fgsea分析", "estimated_time": 15},
|
||
"rnaseq_candidate_family": {"name": "RNA-seq候选家族筛选", "estimated_time": 20},
|
||
"rnaseq_report": {"name": "RNA-seq结果报告", "estimated_time": 2},
|
||
}
|
||
|
||
# 前置依赖（STAGE_DEPENDENCIES 快照；planner 前置补全依据）
|
||
STAGE_DEPENDENCIES_SNAPSHOT: dict[str, list[str]] = {
|
||
"wgs_qc": [], "wgs_index": [], "wgs_align": ["wgs_qc", "wgs_index"],
|
||
"wgs_markdup": ["wgs_align"], "wgs_bam_qc": ["wgs_markdup", "wgs_index"],
|
||
"wgs_haplotype_calling": ["wgs_markdup", "wgs_bam_qc", "wgs_index"],
|
||
"wgs_gvcf_combine": ["wgs_haplotype_calling", "wgs_index"],
|
||
"wgs_joint_genotyping": ["wgs_gvcf_combine", "wgs_index"],
|
||
"wgs_variant_select": ["wgs_joint_genotyping"],
|
||
"wgs_variant_filter": ["wgs_variant_select"], "wgs_impute": ["wgs_variant_filter"],
|
||
"wgs_maf_missing_filter": ["wgs_impute"],
|
||
"pop_pca": [], "pop_phylogeny": [], "pop_structure": [], "pop_ld_decay": [],
|
||
"pop_snp_density": [], "gwas_fmt": [], "gwas_kinship": ["gwas_fmt"],
|
||
"gwas_trait": [], "gwas_cov": [], "gwas_mlm": ["gwas_fmt", "gwas_kinship", "gwas_trait", "gwas_cov"],
|
||
"gwas_plot": ["gwas_mlm"], "gwas_emmax": ["gwas_fmt", "gwas_trait"],
|
||
"gwas_emplot": ["gwas_emmax"], "gwas_glm": ["gwas_fmt", "gwas_trait", "gwas_cov"],
|
||
"identify": [], "identify_refine": [], "subfamily_confirm": ["reference_classification"],
|
||
"genome_prepare": [], "ncbi_fetch_sequences": [], "sequence_extract_by_id": [],
|
||
"msa_region_plot": [], "tree_plot_revise": [], "synteny_plot_revise": [],
|
||
"promoter_plot_revise": [], "prepare_organize": [], "prepare_normalize_gff": [],
|
||
"prepare_longest_transcript": [], "prepare_extract_pep": [], "prepare_extract_cds": [],
|
||
"prepare_filter_fasta": [], "prepare_simplify_ncbi": [], "rnaseq_qc": [],
|
||
"phylogeny": ["identify"],
|
||
"reference_classification": ["identify", "phylogeny", "motif", "structure", "domain", "combo"],
|
||
"synteny": ["identify"], "pep": ["identify"], "msa": ["identify"],
|
||
"motif": ["identify", "phylogeny"], "domain": ["identify", "phylogeny"],
|
||
"structure": ["identify", "phylogeny"], "promoter": ["identify", "phylogeny"],
|
||
"tfbs": ["genome_prepare", "identify"], "expression": ["identify", "phylogeny"],
|
||
"enrichment": ["identify", "phylogeny"], "ortholog": ["identify"],
|
||
"interspecies_synteny": ["identify"], "karyotype": ["identify", "synteny"],
|
||
"ppi": ["identify"], "kaks": ["identify"],
|
||
"combo": ["identify", "phylogeny", "motif", "structure", "domain"],
|
||
"rnaseq_align": ["rnaseq_qc"], "rnaseq_quant": ["rnaseq_align"],
|
||
"rnaseq_sample_qc": ["rnaseq_quant"], "rnaseq_de": ["rnaseq_sample_qc"],
|
||
"rnaseq_wgcna": ["rnaseq_sample_qc"], "annotation": [],
|
||
"rnaseq_enrich": ["rnaseq_de"], "rnaseq_fgsea": ["rnaseq_de"],
|
||
"rnaseq_go_enrich": ["rnaseq_de"], "rnaseq_kegg_enrich": ["rnaseq_de"],
|
||
"rnaseq_candidate_family": ["rnaseq_de", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea"],
|
||
"rnaseq_report": [],
|
||
}
|
||
|
||
GENE_FAMILY_CHAIN = [
|
||
"genome_prepare", "identify", "identify_refine", "phylogeny",
|
||
"reference_classification", "subfamily_confirm", "motif", "domain", "structure",
|
||
"combo", "promoter", "tfbs", "synteny", "interspecies_synteny", "karyotype",
|
||
"pep", "ortholog", "kaks", "ppi", "gene_family_report", "expression",
|
||
]
|
||
|
||
CHAIN_ALIASES = {"gene_family": GENE_FAMILY_CHAIN, "gf": GENE_FAMILY_CHAIN,
|
||
"wgs": None, "rnaseq": None}
|
||
|
||
|
||
@dataclass
|
||
class StageDef:
|
||
"""阶段目录条目（catalog 端点返回形态）。"""
|
||
|
||
name: str
|
||
description: str
|
||
image: str
|
||
module: str
|
||
executable: bool
|
||
estimated_time: int
|
||
params: list[dict[str, Any]] = field(default_factory=list)
|
||
inputs: list[dict[str, Any]] = field(default_factory=list)
|
||
outputs: list[dict[str, Any]] = field(default_factory=list)
|
||
note: str = ""
|
||
dependencies: list[str] = field(default_factory=list)
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
return {
|
||
"name": self.name,
|
||
"description": self.description,
|
||
"image": self.image,
|
||
"module": self.module,
|
||
"executable": self.executable,
|
||
"estimated_time": self.estimated_time,
|
||
"params": self.params,
|
||
"inputs": self.inputs,
|
||
"outputs": self.outputs,
|
||
"note": self.note,
|
||
"dependencies": self.dependencies,
|
||
}
|
||
|
||
|
||
# ---- 基因家族链参数 schema（手写；参数 help 文案自入口 argparse 提取整理）----
|
||
|
||
_GENOME_PREPARE_PARAMS = [
|
||
{"name": "species", "type": "string", "default": "", "source": "user",
|
||
"help": "目标物种代码（如 Ft），基因组数据文件名中的物种标识"},
|
||
{"name": "suffix", "type": "string", "default": "", "source": "user",
|
||
"help": "文件后缀过滤（如 .fa.gz），留空自动匹配"},
|
||
]
|
||
_IDENTIFY_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀，例如 Ft.SPL（输出文件命名空间）"},
|
||
{"name": "required_pfams", "type": "string", "default": "", "source": "contract",
|
||
"help": "逗号分隔的 Pfam accession，如 PF03110（家族识别条件）"},
|
||
{"name": "dom_i_evalue_max", "type": "float", "default": 1e-3, "source": "contract",
|
||
"help": "domain i-Evalue threshold（结构域独立 E 值阈值）"},
|
||
{"name": "hmm_cov_min", "type": "float", "default": 0.3, "source": "contract",
|
||
"help": "HMM coverage threshold（HMM 覆盖度阈值）"},
|
||
{"name": "blast_evalue", "type": "float", "default": 1e-7, "source": "contract",
|
||
"help": "Blast E-value threshold（BLAST E 值阈值）"},
|
||
{"name": "blast_identity_min", "type": "float", "default": 30.0, "source": "contract",
|
||
"help": "minimum BLAST percent identity（参考蛋白命中最低一致度）"},
|
||
{"name": "hmm_combine_mode", "type": "choice", "choices": ["intersection", "union"],
|
||
"default": "union", "source": "contract",
|
||
"help": "多 HMM 文件整合方式（交集/并集），默认并集"},
|
||
{"name": "hmm_blast_merge_mode", "type": "choice", "choices": ["intersection", "union"],
|
||
"default": "", "source": "contract", "help": "HMM 和 BLAST 结果整合方式"},
|
||
{"name": "cut_ga", "type": "boolean", "default": False, "source": "contract",
|
||
"help": "use Pfam GA thresholds（使用 Pfam GA 阈值）"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数（--cpus 注入）"},
|
||
{"name": "memory", "type": "choice", "choices": ["16g", "32g", "64g"], "default": "32g",
|
||
"source": "runtime", "help": "容器内存上限（--memory 注入）"},
|
||
]
|
||
_PHYLOGENY_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀（同 identify）"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
{"name": "memory", "type": "choice", "choices": ["16g", "32g", "64g"], "default": "32g",
|
||
"source": "runtime", "help": "容器内存上限"},
|
||
]
|
||
_MOTIF_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
{"name": "memory", "type": "choice", "choices": ["16g", "32g", "64g"], "default": "32g",
|
||
"source": "runtime", "help": "容器内存上限"},
|
||
]
|
||
_DOMAIN_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_STRUCTURE_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_COMBO_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_PROMOTER_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "upstream", "type": "integer", "default": 2000, "source": "contract",
|
||
"help": "启动子上游区间长度（bp）"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_TFBS_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_SYNTENY_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_INTERSPECIES_SYNTENY_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "ref_species", "type": "string", "default": "", "source": "user",
|
||
"help": "参考物种代码（跨物种共线性）"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_KARYOTYPE_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_PEP_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_ORTHOLOG_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "ref_species", "type": "string", "default": "", "source": "user",
|
||
"help": "参考物种代码（直系同源鉴定）"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_KAKS_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_PPI_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "species_code", "type": "string", "default": "", "source": "user",
|
||
"help": "物种代码（STRING 蛋白互作）"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_REFERENCE_CLASSIFICATION_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_SUBFAMILY_CONFIRM_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_EXPRESSION_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_REPORT_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
]
|
||
_RNASEQ_QC_PARAMS = [
|
||
{"name": "samples", "type": "string", "default": "", "source": "user",
|
||
"help": "样本列表（逗号分隔）"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "8",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_RNASEQ_ALIGN_PARAMS = [
|
||
{"name": "samples", "type": "string", "default": "", "source": "user",
|
||
"help": "样本列表（逗号分隔）"},
|
||
{"name": "ref_genome", "type": "string", "default": "", "source": "user",
|
||
"help": "参考基因组路径/索引名"},
|
||
{"name": "hisat2_index", "type": "string", "default": "", "source": "user",
|
||
"help": "HISAT2 索引前缀（容器内可访问路径，如 /bioflow-data/hmm/hisat2/xxx）"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "8",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_RNASEQ_QUANT_PARAMS = [
|
||
{"name": "samples", "type": "string", "default": "", "source": "user",
|
||
"help": "样本列表（逗号分隔）"},
|
||
{"name": "gtf", "type": "string", "default": "", "source": "user",
|
||
"help": "参考基因组 GTF 注释（容器内可访问路径）"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "6",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_RNASEQ_SAMPLE_QC_PARAMS = [
|
||
{"name": "samples", "type": "string", "default": "", "source": "user",
|
||
"help": "样本列表（逗号分隔）"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_RNASEQ_DE_PARAMS = [
|
||
{"name": "design", "type": "string", "default": "", "source": "user",
|
||
"help": "实验设计矩阵文件（对比组声明）"},
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_RNASEQ_WGCNA_PARAMS = [
|
||
{"name": "design", "type": "string", "default": "", "source": "user",
|
||
"help": "实验设计矩阵文件"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_RNASEQ_ENRICH_PARAMS = [
|
||
{"name": "gene_list", "type": "string", "default": "", "source": "contract",
|
||
"help": "差异基因列表（上游 DE 结果）"},
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_RNASEQ_CANDIDATE_FAMILY_PARAMS = [
|
||
{"name": "family_pfams", "type": "string", "default": "", "source": "user",
|
||
"help": "候选家族 Pfam accession"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4",
|
||
"source": "runtime", "help": "容器 CPU 数"},
|
||
]
|
||
_RNASEQ_REPORT_PARAMS = [
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract",
|
||
"help": "分析前缀"},
|
||
]
|
||
|
||
# wgs/pop/gwas 参数 schema(契约 param: 键 → 确认表单字段)
|
||
_CPU4 = {"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"],
|
||
"default": "4", "source": "runtime", "help": "容器 CPU 数"}
|
||
|
||
|
||
def _u(name: str, help_: str) -> dict[str, str]:
|
||
return {"name": name, "type": "string", "default": "", "source": "user", "help": help_}
|
||
|
||
|
||
_WGS_COMMON = [_u("prefix", "分析前缀"), _CPU4]
|
||
_WGS_QC_PARAMS = [*_WGS_COMMON, _u("wgs_samples", "WGS 样本清单(逗号分隔或路径)")]
|
||
_WGS_INDEX_PARAMS = [*_WGS_COMMON, _u("reference_fasta", "参考基因组 FASTA 路径(容器内可访问)")]
|
||
_WGS_ALIGN_PARAMS = [*_WGS_COMMON]
|
||
_WGS_MARKDUP_PARAMS = [*_WGS_COMMON,
|
||
{"name": "max_jobs", "type": "string", "default": "8", "source": "user",
|
||
"help": "并行任务上限"},
|
||
{"name": "remove_duplicates", "type": "choice", "choices": ["true"],
|
||
"default": "true", "source": "user", "help": "去重复(固定 true)"}]
|
||
_WGS_BAM_QC_PARAMS = [*_WGS_COMMON,
|
||
{"name": "threads", "type": "string", "default": "8", "source": "user",
|
||
"help": "线程数"},
|
||
{"name": "max_jobs", "type": "string", "default": "8", "source": "user",
|
||
"help": "并行任务上限"}]
|
||
_WGS_HAPLOTYPE_PARAMS = [*_WGS_COMMON,
|
||
{"name": "native_pair_hmm_threads", "type": "string", "default": "4",
|
||
"source": "user", "help": "GATK PairHMM 线程"},
|
||
{"name": "max_jobs", "type": "string", "default": "8", "source": "user",
|
||
"help": "并行任务上限"}]
|
||
_WGS_IMPUTE_PARAMS = [*_WGS_COMMON, _u("beagle_jar", "Beagle JAR 路径(容器内可访问)")]
|
||
_POP_COMMON = [*_WGS_COMMON, _u("vcf", "VCF 文件路径(容器内可访问)")]
|
||
_GWAS_TRAIT_PARAMS = [*_WGS_COMMON,
|
||
_u("phenotype", "表型数据文件路径"),
|
||
_u("sample_column", "样本列名"),
|
||
_u("trait_column", "性状列名"),
|
||
_u("trait_name", "性状名")]
|
||
_GWAS_COV_PARAMS = [*_WGS_COMMON,
|
||
_u("q", "Q 矩阵路径"),
|
||
_u("fam", "FAM 文件路径"),
|
||
_u("k", "主成分数 K")]
|
||
|
||
# 通用输入/输出描述（契约文档摘要；契约文件为权威，这里给目录层最小描述）
|
||
_IDENTIFY_INPUTS = [
|
||
{"name": "protein_fasta", "type": "fasta", "required": True,
|
||
"description": "物种全蛋白质 FASTA（input_dir 内查找）"},
|
||
{"name": "hmm_file", "type": "hmm", "required": False,
|
||
"description": "用户上传 HMM 优先；否则按 required_pfams 从 Pfam-A.hmm 获取（BBA_PFAM_DIR）"},
|
||
]
|
||
_IDENTIFY_OUTPUTS = [
|
||
{"name": "final_members", "type": "table", "description": "final/{prefix}.identified_members.id"},
|
||
{"name": "final_proteins", "type": "fasta", "description": "final/{prefix}.identified_members.pep.fa"},
|
||
{"name": "identify_report", "type": "markdown", "description": "final/{prefix}.identify_report.md"},
|
||
{"name": "manifest", "type": "manifest", "description": "final/{prefix}.identify.manifest.json"},
|
||
]
|
||
_GENOME_PREPARE_INPUTS = [
|
||
{"name": "genome_fasta", "type": "fasta", "required": False,
|
||
"description": "{species}.genome.fasta.gz"},
|
||
{"name": "pep_fasta", "type": "fasta", "required": False,
|
||
"description": "{species}.pep.fasta.gz"},
|
||
{"name": "cds_fasta", "type": "fasta", "required": False,
|
||
"description": "{species}.cds.fasta.gz"},
|
||
{"name": "gff3", "type": "gff3", "required": False, "description": "{species}.gff3.gz"},
|
||
]
|
||
_GENOME_PREPARE_OUTPUTS = [
|
||
{"name": "normalized_pep", "type": "fasta", "description": "{species}.pep.fasta"},
|
||
{"name": "normalized_cds", "type": "fasta", "description": "{species}.cds.fasta"},
|
||
{"name": "normalized_gff", "type": "gff3", "description": "{species}_final.gff3"},
|
||
{"name": "manifest", "type": "manifest", "description": "{prefix}.genome_prepare.manifest.json"},
|
||
]
|
||
|
||
# 下游阶段通用描述（目录层最小；契约文档权威）
|
||
_DOWNSTREAM_INPUTS = [
|
||
{"name": "final_members", "type": "table", "required": True,
|
||
"description": "上游 identify 的 final_members.id（跨阶段输入）"},
|
||
]
|
||
_DOWNSTREAM_OUTPUTS = [
|
||
{"name": "manifest", "type": "manifest", "description": "final/{prefix}.{stage}.manifest.json"},
|
||
{"name": "report", "type": "markdown", "description": "final/{prefix}.{stage}_report.md"},
|
||
]
|
||
|
||
# stage → (params, inputs, outputs) 手写表
|
||
_STAGE_PARAM_SCHEMAS: dict[str, tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]] = {
|
||
"genome_prepare": (_GENOME_PREPARE_PARAMS, _GENOME_PREPARE_INPUTS, _GENOME_PREPARE_OUTPUTS),
|
||
"identify": (_IDENTIFY_PARAMS, _IDENTIFY_INPUTS, _IDENTIFY_OUTPUTS),
|
||
"identify_refine": ([
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", "source": "runtime", "help": "容器 CPU 数"},
|
||
], _IDENTIFY_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"phylogeny": (_PHYLOGENY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"reference_classification": (_REFERENCE_CLASSIFICATION_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"subfamily_confirm": (_SUBFAMILY_CONFIRM_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"motif": (_MOTIF_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"domain": (_DOMAIN_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"structure": (_STRUCTURE_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"combo": (_COMBO_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"promoter": (_PROMOTER_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"tfbs": (_TFBS_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"synteny": (_SYNTENY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"interspecies_synteny": (_INTERSPECIES_SYNTENY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"karyotype": (_KARYOTYPE_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"pep": (_PEP_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"ortholog": (_ORTHOLOG_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"kaks": (_KAKS_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"ppi": (_PPI_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"expression": (_EXPRESSION_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"gene_family_report": (_REPORT_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"msa": ([
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "4", "source": "runtime", "help": "容器 CPU 数"},
|
||
], _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"enrichment": ([
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"},
|
||
], _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_qc": (_RNASEQ_QC_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_qc": (_WGS_QC_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_index": (_WGS_INDEX_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_align": (_WGS_ALIGN_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_markdup": (_WGS_MARKDUP_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_bam_qc": (_WGS_BAM_QC_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_haplotype_calling": (_WGS_HAPLOTYPE_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_gvcf_combine": (_WGS_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_joint_genotyping": (_WGS_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_variant_select": (_WGS_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_variant_filter": (_WGS_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_impute": (_WGS_IMPUTE_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"wgs_maf_missing_filter": (_WGS_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"pop_pca": (_POP_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"pop_phylogeny": (_POP_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"pop_structure": (_POP_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"pop_ld_decay": (_POP_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"pop_snp_density": (_POP_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"gwas_fmt": (_POP_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"gwas_kinship": (_POP_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"gwas_trait": (_GWAS_TRAIT_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"gwas_cov": (_GWAS_COV_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"gwas_mlm": (_WGS_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"gwas_plot": (_WGS_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"gwas_emmax": (_WGS_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"gwas_emplot": (_WGS_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"gwas_glm": (_WGS_COMMON, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_align": (_RNASEQ_ALIGN_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_quant": (_RNASEQ_QUANT_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_sample_qc": (_RNASEQ_SAMPLE_QC_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_de": (_RNASEQ_DE_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_wgcna": (_RNASEQ_WGCNA_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_enrich": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_go_enrich": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_kegg_enrich": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_fgsea": (_RNASEQ_ENRICH_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_candidate_family": (_RNASEQ_CANDIDATE_FAMILY_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"rnaseq_report": (_RNASEQ_REPORT_PARAMS, _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
"annotation": ([
|
||
{"name": "prefix", "type": "string", "default": "", "source": "contract", "help": "分析前缀"},
|
||
{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"], "default": "8", "source": "runtime", "help": "容器 CPU 数"},
|
||
], _DOWNSTREAM_INPUTS, _DOWNSTREAM_OUTPUTS),
|
||
}
|
||
|
||
|
||
class StageCatalog:
|
||
"""阶段目录（数据来自 stage_image_map.json + 手写快照，不运行时反射）。"""
|
||
|
||
def __init__(self, image_map: Any) -> None:
|
||
self._map = image_map # ImageMap 实例（kernel 模块）
|
||
|
||
# ---- 查询 ----
|
||
|
||
def stage_names(self) -> list[str]:
|
||
return self._map.all_stage_names()
|
||
|
||
def family_chain(self) -> list[str]:
|
||
return list(self._map.family_chain) or GENE_FAMILY_CHAIN
|
||
|
||
def has(self, stage: str) -> bool:
|
||
return stage in self._map.stages
|
||
|
||
def validate_chain(self, stages: list[str]) -> list[str]:
|
||
"""链硬校验：返回未知阶段名列表（空=全合法）。"""
|
||
return [s for s in stages if not self.has(s)]
|
||
|
||
def get(self, stage: str) -> StageDef | None:
|
||
"""返回阶段目录条目（未知阶段 None）。"""
|
||
if not self.has(stage):
|
||
return None
|
||
entry = self._map.image_for(stage) or {}
|
||
params, inputs, outputs = _STAGE_PARAM_SCHEMAS.get(
|
||
stage, ([], _DOWNSTREAM_INPUTS if stage not in ("identify", "genome_prepare") else [], [])
|
||
)
|
||
if stage not in _STAGE_PARAM_SCHEMAS:
|
||
params, inputs, outputs = (
|
||
[{"name": "cpu", "type": "choice", "choices": ["4", "8", "16", "32"],
|
||
"default": "4", "source": "runtime", "help": "容器 CPU 数"}],
|
||
_DOWNSTREAM_INPUTS if stage not in ("identify", "genome_prepare") else [],
|
||
_DOWNSTREAM_OUTPUTS,
|
||
)
|
||
info = STAGE_INFO_SNAPSHOT.get(stage, {})
|
||
return StageDef(
|
||
name=stage,
|
||
description=str(info.get("name") or stage),
|
||
image=str(entry.get("image") or ""),
|
||
module=str(entry.get("module") or ""),
|
||
executable=bool(entry.get("executable")),
|
||
estimated_time=int(info.get("estimated_time") or 0),
|
||
params=params,
|
||
inputs=inputs,
|
||
outputs=outputs,
|
||
note=str(entry.get("note") or ""),
|
||
dependencies=STAGE_DEPENDENCIES_SNAPSHOT.get(stage, []),
|
||
)
|
||
|
||
def resolve_chain(self, chain: str | None) -> list[str] | None:
|
||
"""chain 别名解析（gene_family 链 21 阶段）；None 表示无该链。"""
|
||
if not chain:
|
||
return None
|
||
alias = CHAIN_ALIASES.get(chain)
|
||
if alias is None:
|
||
return None
|
||
return list(alias)
|
||
|
||
def auto_fill_prerequisites(self, stages: list[str]) -> tuple[list[str], list[str]]:
|
||
"""前置补全（FR-BIO-003）：缺失的前置阶段自动补全。
|
||
|
||
返回 (补全后的链, 被补全的阶段名列表)。
|
||
链内阶段按依赖拓扑顺序排列（依赖在前）。
|
||
"""
|
||
missing: list[str] = []
|
||
added: list[str] = []
|
||
chain_set = set(stages)
|
||
# 反复补全直到闭合（依赖的依赖也补）
|
||
changed = True
|
||
while changed:
|
||
changed = False
|
||
for s in list(stages):
|
||
for dep in STAGE_DEPENDENCIES_SNAPSHOT.get(s, []):
|
||
if dep not in chain_set and self.has(dep):
|
||
chain_set.add(dep)
|
||
missing.append(dep)
|
||
added.append(dep)
|
||
changed = True
|
||
ordered = topo_order([s for s in GENE_FAMILY_CHAIN if s in chain_set]
|
||
or list(chain_set))
|
||
return ordered, added
|
||
|
||
def dependencies(self, stage: str) -> list[str]:
|
||
return STAGE_DEPENDENCIES_SNAPSHOT.get(stage, [])
|
||
|
||
|
||
def topo_order(stages: list[str]) -> list[str]:
|
||
"""按依赖拓扑排序（依赖在前，保持输入相对顺序稳定）。"""
|
||
result: list[str] = []
|
||
seen: set[str] = set()
|
||
|
||
def visit(s: str) -> None:
|
||
if s in seen:
|
||
return
|
||
seen.add(s)
|
||
for dep in STAGE_DEPENDENCIES_SNAPSHOT.get(s, []):
|
||
visit(dep)
|
||
result.append(s)
|
||
|
||
for s in stages:
|
||
visit(s)
|
||
return result
|
||
|
||
|
||
def build_catalog(image_map: Any) -> StageCatalog:
|
||
return StageCatalog(image_map)
|