From 03b11dedda6497acccde297dc6f14b3f7fa70bda Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 6 Sep 2026 03:00:09 +0800 Subject: [PATCH] =?UTF-8?q?chore(bioflow):=20=E5=88=A0=E9=99=A4=E5=89=AF?= =?UTF-8?q?=E6=9C=AC=E4=B8=AD=2013=20=E4=B8=AA=20dev=20=E5=8E=86=E5=8F=B2?= =?UTF-8?q?=E5=A4=87=E4=BB=BD=E6=96=87=E4=BB=B6(.bak/.orig)=E2=80=94?= =?UTF-8?q?=E2=80=94=E5=89=AF=E6=9C=AC=E5=85=A8=E6=A0=91=E9=9B=B6=E7=A1=AC?= =?UTF-8?q?=E7=BC=96=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- ...epare.py.bak_family_allpep_20260504_193257 | 1542 ------ ...ntify.py.bak_family_allpep_20260504_193257 | 1802 ------- ...n_interspecies_synteny.py.bak_blast_engine | 1715 ------- ...nteny.py.bak_family_allpep_20260504_193257 | 1518 ------ ...nterspecies_synteny.py.bak_highlight_local | 1542 ------ ...es_synteny.py.bak_pairwise_20260504_165043 | 1265 ----- ...interspecies_synteny.py.bak_species_select | 1609 ------ .../cli/run_rnaseq_qc.py.bak.20260430_050459 | 617 --- .../cli/run_rnaseq_qc.py.bak_20260813_225803 | 1072 ---- .../run_structure.py.bak_20260719_target_only | 1329 ----- .../gfa_core/mas/agents/diagnosis.py.orig | 4545 ----------------- .../gfa_core/mas/tools/stage_status.py.orig | 1092 ---- .../gfa_core/mas/tools/stage_tools.py.orig | 1044 ---- 13 files changed, 20692 deletions(-) delete mode 100755 bioflow_kernel/cli/run_genome_prepare.py.bak_family_allpep_20260504_193257 delete mode 100755 bioflow_kernel/cli/run_identify.py.bak_family_allpep_20260504_193257 delete mode 100755 bioflow_kernel/cli/run_interspecies_synteny.py.bak_blast_engine delete mode 100755 bioflow_kernel/cli/run_interspecies_synteny.py.bak_family_allpep_20260504_193257 delete mode 100755 bioflow_kernel/cli/run_interspecies_synteny.py.bak_highlight_local delete mode 100755 bioflow_kernel/cli/run_interspecies_synteny.py.bak_pairwise_20260504_165043 delete mode 100755 bioflow_kernel/cli/run_interspecies_synteny.py.bak_species_select delete mode 100755 bioflow_kernel/cli/run_rnaseq_qc.py.bak.20260430_050459 delete mode 100755 bioflow_kernel/cli/run_rnaseq_qc.py.bak_20260813_225803 delete mode 100755 bioflow_kernel/cli/run_structure.py.bak_20260719_target_only delete mode 100755 bioflow_kernel/gfa_core/mas/agents/diagnosis.py.orig delete mode 100755 bioflow_kernel/gfa_core/mas/tools/stage_status.py.orig delete mode 100755 bioflow_kernel/gfa_core/mas/tools/stage_tools.py.orig diff --git a/bioflow_kernel/cli/run_genome_prepare.py.bak_family_allpep_20260504_193257 b/bioflow_kernel/cli/run_genome_prepare.py.bak_family_allpep_20260504_193257 deleted file mode 100755 index 2b88727..0000000 --- a/bioflow_kernel/cli/run_genome_prepare.py.bak_family_allpep_20260504_193257 +++ /dev/null @@ -1,1542 +0,0 @@ -#!/usr/bin/env python3 -""" -Genome Prepare V2: 多物种基因组数据预处理 -支持批量处理多个物种的基因组数据 - -输入：input/job_id/ 下的多个物种原始基因组数据 - - {species}.cds.fasta.gz / {species}.pep.fasta.gz (序列文件) - - {species}.gff3.gz (注释文件) - - {species}.genome.fasta.gz (基因组文件，可选) - -输出： - - {species}.cds.fasta (标准化CDS序列) - - {species}.pep.fasta (标准化蛋白序列) - - {species}.genome.fasta (标准化基因组序列，如有) - - {species}_final.gff3 (标准化GFF注释) - - {prefix}.genome_prepare.manifest.json (机器清单，符合schema v1) - - {prefix}.genome_prepare_report.md (分析报告) - -多物种命名规则： - - 文件名应包含物种标识，如： - - Athaliana_167_TAIR10.protein_primaryTranscriptOnly.fa.gz (拟南芥) - - FtChromosomeV2.fasta.gz (苦荞) - - Osativa_323_v7.protein_primaryTranscriptOnly.fa.gz (水稻) - -依赖工具： - - python (converted GFF helper scripts) - - gffread (从基因组+GFF提取蛋白序列) - - pigz/gzip (解压.gz文件) -""" - -import os -import argparse -import json -import hashlib -import logging -import re -import subprocess -import sys -import shutil -from datetime import datetime -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple -from dataclasses import dataclass, field - -# FIX: 添加 pandas 用于读取 xlsx -try: - import pandas as pd -except ImportError: - pd = None - -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - - -# 添加conda环境bin目录到PATH -CONDA_BIN = "/home/xia-lab/miniconda3/envs/test4py311/bin" -if CONDA_BIN not in os.environ.get("PATH", ""): - os.environ["PATH"] = CONDA_BIN + ":" + os.environ.get("PATH", "") - - -@dataclass -class SpeciesFiles: - """表示一个物种的文件集合""" - species_code: str - cds: Optional[Path] = None - pep: Optional[Path] = None - gff: Optional[Path] = None - genome: Optional[Path] = None - db_type: str = "unknown" - - def is_valid(self) -> bool: - """检查是否有足够的文件进行分析 - - FIX: 目标物种需要 PEP + GFF，参考物种只需要 PEP - """ - # 至少有 PEP 文件 - if self.pep is None: - return False - - # 如果有 GFF，是完整的目标物种 - # 如果没有 GFF，是参考物种（轻量级处理） - return True - - def __str__(self) -> str: - files = [] - if self.cds: files.append("CDS") - if self.pep: files.append("PEP") - if self.gff: files.append("GFF") - if self.genome: files.append("GENOME") - return f"{self.species_code}({','.join(files)})" - - -def compute_file_hash(filepath: Path) -> str: - """计算文件的SHA256哈希值""" - sha256_hash = hashlib.sha256() - try: - with open(filepath, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - except Exception: - return "" - - -def get_tool_versions() -> Dict[str, str]: - """获取工具版本信息""" - versions = {} - versions["python"] = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - - - try: - subprocess.run(['seqtk'], capture_output=True, check=False) - versions["seqtk"] = "available" - except: - versions["seqtk"] = "not_found" - - try: - result = subprocess.run(['gffread', '--version'], capture_output=True, text=True, check=False) - versions["gffread"] = (result.stdout.strip() or result.stderr.strip() or "unknown").split('\n')[0] - except: - versions["gffread"] = "not_found" - - return versions - - -class MultiSpeciesDetector: - """多物种文件检测器""" - - # 物种代码映射 - SPECIES_PATTERNS = { - # 模式: (物种代码, 物种名称) - r'^[Aa]th.*TAIR': ('Ath', 'Arabidopsis thaliana'), - r'^[Ff]t.*Chromosome': ('Ft', 'Fagopyrum tataricum'), - r'^[Oo]s.*': ('Os', 'Oryza sativa'), - r'^[Zz]m.*': ('Zm', 'Zea mays'), - r'^[Ss]l.*': ('Sl', 'Solanum lycopersicum'), - r'^[Gg]m.*': ('Gm', 'Glycine max'), - r'^[Bb]d.*': ('Bd', 'Brachypodium distachyon'), - r'^[Hh]v.*': ('Hv', 'Hordeum vulgare'), - r'^[Ss]b.*': ('Sb', 'Sorghum bicolor'), - r'^[Pp]t.*': ('Pt', 'Populus trichocarpa'), - r'^[Vv].*': ('Vv', 'Vitis vinifera'), - r'^[Cc]r.*': ('Cr', 'Chlamydomonas reinhardtii'), - # 新增物种 - r'^[Ff]vesca.*|^[Ff]v.*': ('Fv', 'Fragaria vesca (Strawberry)'), - r'^[Mm]alus.*|^[Mm]d.*': ('Md', 'Malus domestica (Apple)'), - r'^[Pp]runus.*|^[Pp]p.*': ('Pp', 'Prunus persica (Peach)'), - r'^[Pp]yrus.*|^[Pp]b.*': ('Pb', 'Pyrus communis (Pear)'), - r'^[Rr]ubus.*|^[Rr]o.*': ('Ro', 'Rubus occidentalis (Raspberry)'), - r'^[Oo]BDH.*': ('OBDH', 'OBDH (Unknown species)'), - } - - @classmethod - def detect_species(cls, filename: str) -> Tuple[str, str]: - """从文件名检测物种代码""" - for pattern, (code, name) in cls.SPECIES_PATTERNS.items(): - if re.match(pattern, filename): - return code, name - - # 默认：提取前两个大写字母作为物种代码 - match = re.match(r'^([A-Z][a-z])', filename) - if match: - return match.group(1), f"Unknown ({filename[:10]}...)" - - return "UNK", "Unknown" - - @classmethod - def detect_db_type(cls, filename: str) -> str: - """检测数据库类型""" - name_lower = filename.lower() - - # JGI特征 - if 'primarytranscriptonly' in name_lower or '_tair' in name_lower: - return "jgi" - - # Ensembl特征 - if '.cds.all.fa' in name_lower or '.pep.all.fa' in name_lower: - return "ensembl" - - # NCBI特征 - if 'gcf_' in name_lower or '_cds_from_genomic' in name_lower: - return "ncbi" - - return "unknown" - - @classmethod - def scan_input_dir(cls, input_dir: Path) -> Dict[str, SpeciesFiles]: - """扫描输入目录，识别所有物种 - - FIX: 支持递归扫描子目录，从文件夹名获取 Species_Code - """ - species_map: Dict[str, SpeciesFiles] = {} - - # FIX: 递归扫描所有文件（包括子目录） - for file in input_dir.rglob("*"): - if not file.is_file(): - continue - - name = file.name - name_lower = name.lower() - - # Skip generated files, merged reference files, and config.xlsx. - if '_final.gff3' in name or name.startswith('Ref_combined.') or name == 'config.xlsx': - continue - - # FIX: 从父文件夹名获取 Species_Code（如果不在根目录） - if file.parent != input_dir: - # 子目录中的文件，使用文件夹名作为 Species_Code - # FIX: 排除特殊文件夹 - parent_name = file.parent.name - if parent_name in ['genome_prepare', 'final', 'logs', 'analysis']: - continue - species_code = parent_name - else: - # 根目录中的文件，从文件名检测 - species_code, _ = cls.detect_species(name) - - # Classify file before creating a species entry; ignore logs, HMMs, reports, etc. - file_kind = None - if ('cds' in name_lower or 'cdna' in name_lower or 'rna' in name_lower) and ('.fa' in name_lower or '.fasta' in name_lower or '.fna' in name_lower): - file_kind = 'cds' - elif ('protein' in name_lower or 'pep' in name_lower or name_lower.endswith('.faa') or name_lower.endswith('.faa.gz')) and ('.fa' in name_lower or '.fasta' in name_lower): - file_kind = 'pep' - elif name_lower.endswith('.gff3') or name_lower.endswith('.gff') or '.gff3.' in name_lower or '.gff.' in name_lower: - file_kind = 'gff' - elif ('genome' in name_lower or 'chromosome' in name_lower or 'genomic' in name_lower) and ('.fa' in name_lower or '.fasta' in name_lower or '.fna' in name_lower): - file_kind = 'genome' - - if file_kind is None: - continue - - if species_code not in species_map: - species_map[species_code] = SpeciesFiles( - species_code=species_code, - db_type="unknown" - ) - - if file_kind == 'cds': - species_map[species_code].cds = file - elif file_kind == 'pep': - species_map[species_code].pep = file - elif file_kind == 'gff': - species_map[species_code].gff = file - elif file_kind == 'genome': - species_map[species_code].genome = file - - return species_map - - -def organize_files_by_config(input_dir: Path) -> Dict[str, Any]: - """ - 根据 config.xlsx 组织文件到对应 Species_Code 文件夹 - - FIX: 新增函数，实现文件分类移动和重命名 - FIX: 新增 Ref 列支持，区分目标物种和参考物种 - - Args: - input_dir: 输入目录路径（input//） - - Returns: - 包含组织结果的字典 - """ - config_path = input_dir / "config.xlsx" - - if not config_path.exists(): - logger.warning(f"config.xlsx not found in {input_dir}, skipping organization") - return {"organized": False, "reason": "config.xlsx not found"} - - if pd is None: - logger.error("pandas not available, cannot read config.xlsx") - return {"organized": False, "reason": "pandas not available"} - - try: - # 读取 config.xlsx - df = pd.read_excel(config_path) - logger.info(f"Read config.xlsx with {len(df)} entries") - - # 检查必要列 - required_cols = ['Original_File_Name', 'Species_Code', 'Gene_Family', 'File_Type'] - missing_cols = [col for col in required_cols if col not in df.columns] - if missing_cols: - raise ValueError(f"Missing columns in config.xlsx: {missing_cols}") - - # FIX: 检查 Ref 列，如果没有则默认 FALSE - has_ref_col = 'Ref' in df.columns - if not has_ref_col: - logger.warning("Ref column not found, defaulting to FALSE for all entries") - df['Ref'] = False - - # 创建 Species_Code 文件夹 - species_codes = df['Species_Code'].unique() - for code in species_codes: - if pd.isna(code) or code == '-': - continue - species_dir = input_dir / str(code) - species_dir.mkdir(exist_ok=True) - logger.info(f"Created directory: {species_dir}") - - # 移动并重命名文件 - moved_files = [] - missing_files = [] - gene_family = "unknown" - - for _, row in df.iterrows(): - original_name = str(row['Original_File_Name']) - species_code = str(row['Species_Code']) - gene_family = str(row['Gene_Family']) - file_type = str(row['File_Type']) - is_ref = bool(row['Ref']) if has_ref_col else False - - # 跳过无效的 Species_Code - if pd.isna(species_code) or species_code == '-': - logger.info(f"Skipping file with no species code: {original_name}") - continue - - # FIX: 记录 gene_family（从第一个有效行获取） - if gene_family == "unknown" and not pd.isna(row.get('Gene_Family')): - gene_family = str(row['Gene_Family']) - - # Karpathy: 生成新文件名，保留原始文件的后缀（包括 .gz） - # 修订 2026-04-27: 保留 .gz 后缀，让 decompress_if_needed 正确识别 - if file_type.lower() == 'gff': - # Preserve GFF/GFF3 and .gz so decompress_if_needed can still work. - original_lower = original_name.lower() - if original_lower.endswith('.gff3.gz'): - new_name = f"{species_code}.{gene_family}.gff3.gz" - elif original_lower.endswith('.gff.gz'): - new_name = f"{species_code}.{gene_family}.gff.gz" - elif original_lower.endswith('.gff3'): - new_name = f"{species_code}.{gene_family}.gff3" - else: - new_name = f"{species_code}.{gene_family}.gff" - else: - # FASTA 文件可能是 .fasta, .fa, .fasta.gz, .fa.gz 等 - if original_name.endswith('.gz'): - # 提取基础后缀（如 .fasta.gz → .fasta） - base_name = original_name.replace('.gz', '') - # 提取文件类型部分（cds, pep, genome） - new_base = f"{species_code}.{gene_family}.{file_type}.fasta" - new_name = f"{new_base}.gz" - else: - new_name = f"{species_code}.{gene_family}.{file_type}.fasta" - - source = input_dir / original_name - target = input_dir / species_code / new_name - - file_record = { - 'original': original_name, - 'new': new_name, - 'species': species_code, - 'is_ref': is_ref # FIX: ????????? - } - if source.exists(): - # ?????????????????????????? - if target.exists(): - logger.warning(f"Target exists, removing: {target}") - target.unlink() - shutil.move(str(source), str(target)) - moved_files.append(file_record) - logger.info(f"Moved & Renamed: {original_name} -> {species_code}/{new_name}") - elif target.exists(): - # ???????????????????????????????????? - moved_files.append(file_record) - logger.info(f"Standardized file already exists: {species_code}/{new_name}") - else: - missing_files.append(original_name) - logger.warning(f"Source file not found: {original_name}") - - # FIX: 分离目标物种和参考物种 - target_species = [f['species'] for f in moved_files if not f.get('is_ref', False)] - ref_species = [f['species'] for f in moved_files if f.get('is_ref', False)] - - return { - "organized": True, - "species_codes": list(species_codes), - "target_species": list(set(target_species)), - "ref_species": list(set(ref_species)), - "moved_files": moved_files, - "missing_files": missing_files, - "gene_family": gene_family if 'gene_family' in locals() else "unknown" - } - - except Exception as e: - logger.error(f"Failed to organize files: {e}") - return {"organized": False, "reason": str(e)} - - -def merge_species_files(input_dir: Path, genome_prepare_dir: Path, org_result: Dict[str, Any]) -> Dict[str, Any]: - """ - 合并参考物种的标准化文件 - - 修订 2026-04-26: 只合并参考物种，目标物种保持单独文件 - - FIX: 新增函数，实现文件合并逻辑 - FIX: 直接从 config.xlsx 读取参考物种信息 - - Args: - input_dir: 输入目录路径 - genome_prepare_dir: genome_prepare 目录路径 - org_result: organize_files_by_config 的返回结果 - - Returns: - 包含合并结果的字典 - """ - gene_family = org_result.get("gene_family", "unknown") - target_species = org_result.get("target_species", []) - ref_species = org_result.get("ref_species", []) - - # FIX: 直接从 config.xlsx 读取参考物种信息 - config_path = input_dir / "config.xlsx" - if config_path.exists() and pd is not None: - try: - df = pd.read_excel(config_path) - if 'Ref' in df.columns: - all_species = df['Species_Code'].unique() - target_species = [] - ref_species = [] - for spc in all_species: - if pd.isna(spc) or spc == '-': - continue - species_records = df[df['Species_Code'] == spc] - if species_records['Ref'].any(): - ref_species.append(spc) - else: - target_species.append(spc) - logger.info(f"Re-extracted from config.xlsx for merge: target={target_species}, ref={ref_species}") - except Exception as e: - logger.warning(f"Failed to re-extract species from config.xlsx: {e}") - - merged_files = {} - - # 定义文件类型 - file_types = ['cds', 'pep', 'gff'] - - # Karpathy: 合并参考物种文件 -> Ref_combined.{GeneFamily}.* - # 修订 2026-04-27: 从子目录读取参考物种文件进行合并 - if ref_species: - for file_type in ['cds', 'pep']: # 参考物种通常不需要 gff - output_name = f"Ref_combined.{gene_family}.{file_type}.fasta" - output_path = input_dir / output_name - - # 修订: 在子目录中查找参考物种文件（因为参考物种不再输出到根目录） - files_to_merge = [] - for spc in ref_species: - # 优先在子目录中查找 - spc_file_in_subdir = input_dir / spc / f"{spc}.{gene_family}.{file_type}.fasta" - # 兼容：也在根目录查找（旧版本兼容） - spc_file_in_root = input_dir / f"{spc}.{gene_family}.{file_type}.fasta" - - if spc_file_in_subdir.exists(): - files_to_merge.append(spc_file_in_subdir) - logger.info(f"Found reference {file_type} in subdirectory: {spc_file_in_subdir}") - elif spc_file_in_root.exists(): - files_to_merge.append(spc_file_in_root) - logger.info(f"Found reference {file_type} in root: {spc_file_in_root}") - - if files_to_merge: - # 合并文件 - with open(output_path, 'w') as outfile: - for file_path in files_to_merge: - with open(file_path, 'r') as infile: - outfile.write(infile.read()) - if not infile.read().endswith('\n'): - outfile.write('\n') - - merged_files[f"ref_{file_type}"] = str(output_path) - logger.info(f"Merged reference {file_type}: {output_name} ({len(files_to_merge)} files)") - - return { - "merged": True, - "target_species": target_species, - "ref_species": ref_species, - "merged_files": merged_files - } - - -def generate_species_config(input_dir: Path, genome_prepare_dir: Path, org_result: Dict[str, Any]) -> Path: - """ - 生成 species_config.json 配置文件 - - FIX: 新增函数，用于 R 脚本读取物种信息 - FIX: 直接从 config.xlsx 读取参考物种信息 - - Args: - input_dir: 输入目录路径 - genome_prepare_dir: genome_prepare 目录路径 - org_result: organize_files_by_config 的返回结果 - - Returns: - 生成的配置文件路径 - """ - gene_family = org_result.get("gene_family", "unknown") - target_species = org_result.get("target_species", []) - ref_species = org_result.get("ref_species", []) - - # FIX: 直接从 config.xlsx 读取参考物种信息（因为 organize_files_by_config 可能遗漏） - config_path = input_dir / "config.xlsx" - if config_path.exists() and pd is not None: - try: - df = pd.read_excel(config_path) - if 'Ref' in df.columns: - # 重新提取目标物种和参考物种 - all_species = df['Species_Code'].unique() - target_species = [] - ref_species = [] - for spc in all_species: - if pd.isna(spc) or spc == '-': - continue - species_records = df[df['Species_Code'] == spc] - if species_records['Ref'].any(): - ref_species.append(spc) - else: - target_species.append(spc) - logger.info(f"Re-extracted from config.xlsx: target={target_species}, ref={ref_species}") - except Exception as e: - logger.warning(f"Failed to re-extract species from config.xlsx: {e}") - - # 预定义颜色 - colors = ["#E41A1C", "#377EB8", "#4DAF4A", "#984EA3", "#FF7F00", "#FFFF33", "#A65628", "#F781BF"] - - # 构建物种映射 - species_mapping = {} - all_species = target_species + ref_species - - for i, spc in enumerate(all_species): - color = colors[i % len(colors)] - species_mapping[spc] = { - "type": "target" if spc in target_species else "reference", - "color": color, - "pattern": f"^{spc}" - } - - config = { - "gene_family": gene_family, - "target": { - "prefix": f"Multi.{gene_family}", - "species": target_species, - "files": { - "cds": None, # 修订 2026-04-26: 不再合并目标物种 - "pep": None, - "gff": None - } - }, - "reference": { - "prefix": f"Ref_combined.{gene_family}", # 修订 2026-04-26: Ref. -> Ref_combined. - "species": ref_species, - "files": { - "cds": f"Ref_combined.{gene_family}.cds.fasta", - "pep": f"Ref_combined.{gene_family}.pep.fasta" - } - }, - "species_mapping": species_mapping - } - - config_path = genome_prepare_dir / "species_config.json" - with open(config_path, 'w') as f: - json.dump(config, f, indent=2) - - logger.info(f"Generated species_config.json: {config_path}") - return config_path - - -def _extract_gene_family_from_filename(filename: str) -> str: - """从文件名提取 Gene_Family - - 假设文件名格式: {Species_Code}.{Gene_Family}.{File_Type}.fasta - 例如: Lc.SUTs.pep.fasta -> SUTs - """ - parts = filename.split('.') - if len(parts) >= 3: - return parts[1] # Gene_Family 是第二部分 - return "unknown" - - -def validate_input_directory(input_dir: Path) -> Dict[str, Any]: - """ - 验证输入目录并获取必要信息 - - Args: - input_dir: 输入目录路径（input//） - - Returns: - 包含输入文件信息的字典 - - Raises: - FileNotFoundError: 输入目录不存在 - ValueError: 输入目录无效 - """ - print(f"正在验证输入目录: {input_dir}") - - # 1. 检查输入目录是否存在 - if not input_dir.exists(): - raise FileNotFoundError(f"输入目录不存在: {input_dir}") - - if not input_dir.is_dir(): - raise ValueError(f"输入路径不是目录: {input_dir}") - - print(f" 输入目录存在: {input_dir}") - - # 2. 扫描物种文件 - species_map = MultiSpeciesDetector.scan_input_dir(input_dir) - - if not species_map: - raise ValueError(f"在输入目录中未找到有效的物种文件: {input_dir}") - - print(f" 检测到 {len(species_map)} 个物种") - for code, species in species_map.items(): - print(f" - {species}") - - # 3. 验证每个物种的文件完整性 - valid_species = {} - invalid_species = [] - - for code, species in species_map.items(): - if species.is_valid(): - valid_species[code] = species - else: - invalid_species.append(code) - print(f" ⚠️ 物种 {code} 文件不完整: {species}") - - if not valid_species: - raise ValueError("没有有效的物种文件（需要至少PEP文件）") - - print(f" 有效物种: {len(valid_species)}/{len(species_map)}") - - return { - "species_map": valid_species, - "invalid_species": invalid_species, - "total_files": len(list(input_dir.iterdir())), - "input_dir": input_dir - } - - -def prepare_directories(workspace: Path) -> Dict[str, Path]: - """创建Genome Prepare阶段的标准目录结构""" - paths = { - "work_dir": workspace / "genome_prepare", - "final_dir": workspace, - "logs_dir": workspace / "logs", - } - - print("创建目录结构:") - for name, path in paths.items(): - path.mkdir(parents=True, exist_ok=True) - print(f" {name}: {path}") - - return paths - - -class MultiSpeciesGenomePreparer: - """??????????""" - - def __init__(self, input_dir: Path, output_dir: Path, scripts_dir: Path, paths: Dict[str, Path]): - self.input_dir = input_dir - self.output_dir = output_dir - self.scripts_dir = scripts_dir - self.paths = paths - self.work_dir = paths["work_dir"] - - self.cmd_history = [] - self.results = [] - - def _extract_gene_family(self, species: SpeciesFiles) -> str: - """???????? Gene_Family?""" - for file_attr in [species.pep, species.cds, species.gff, species.genome]: - if file_attr: - return _extract_gene_family_from_filename(file_attr.name) - return "unknown" - - def _is_reference_file(self, species: SpeciesFiles) -> bool: - """???????????? config.xlsx ?? Ref ???""" - config_path = self.input_dir / "config.xlsx" - - if not config_path.exists() or pd is None: - for file_attr in [species.pep, species.cds]: - if file_attr and 'ref' in file_attr.name.lower(): - return True - return False - - try: - df = pd.read_excel(config_path) - if 'Ref' not in df.columns: - return False - - species_records = df[df['Species_Code'] == species.species_code] - if not species_records.empty: - return species_records['Ref'].any() - - return False - except Exception as e: - logger.warning(f"Failed to read config.xlsx for ref check: {e}") - return False - - def run_command(self, cmd: List[str], description: str = "") -> Tuple[int, str, str]: - """????????""" - cmd_str = ' '.join(str(c) for c in cmd) - self.cmd_history.append(cmd_str) - logger.info(f"Running: {description or cmd_str}") - - try: - result = subprocess.run(cmd, capture_output=True, text=True, cwd=self.work_dir, check=False) - return result.returncode, result.stdout, result.stderr - except Exception as e: - logger.error(f"Command failed: {e}") - return -1, "", str(e) - - def run_command_to_file(self, cmd: List[str], output_file: Path, description: str = "") -> Tuple[int, str, str]: - """?????? stdout ?????????? shell ???""" - cmd_str = ' '.join(str(c) for c in cmd) + f" > {output_file}" - self.cmd_history.append(cmd_str) - logger.info(f"Running: {description or cmd_str}") - - try: - with open(output_file, 'wb') as out: - result = subprocess.run(cmd, stdout=out, stderr=subprocess.PIPE, cwd=self.work_dir, check=False) - stderr = result.stderr.decode(errors='replace') if isinstance(result.stderr, bytes) else (result.stderr or "") - return result.returncode, "", stderr - except Exception as e: - logger.error(f"Command failed: {e}") - return -1, "", str(e) - - def decompress_if_needed(self, filepath: Path) -> Path: - """????? .gz ???????????????""" - if filepath.suffix == '.gz': - decompressed_path = filepath.parent / filepath.stem - logger.info(f"Decompressing {filepath.name}...") - - decompressed = False - try: - result = subprocess.run(['pigz', '-d', '-c', str(filepath)], capture_output=True, text=False, check=False) - if result.returncode == 0: - with open(decompressed_path, 'wb') as f: - f.write(result.stdout) - logger.info(f"Decompressed to {decompressed_path.name}") - decompressed = True - except FileNotFoundError: - pass - - if not decompressed: - result = subprocess.run(['gzip', '-d', '-c', str(filepath)], capture_output=True, text=False, check=False) - if result.returncode == 0: - with open(decompressed_path, 'wb') as f: - f.write(result.stdout) - logger.info(f"Decompressed to {decompressed_path.name}") - decompressed = True - - if decompressed: - try: - filepath.unlink() - logger.info(f"Removed compressed file: {filepath.name}") - except Exception as e: - logger.warning(f"Failed to remove compressed file {filepath.name}: {e}") - return decompressed_path - - return filepath - - def detect_db_type_from_gff(self, gff_file: Optional[Path]) -> str: - """??? GFF ??????????""" - if not gff_file or not gff_file.exists(): - return "unknown_user" - - ensembl_score = 0 - jgi_score = 0 - ncbi_score = 0 - checked = 0 - - with open(gff_file, 'r', errors='replace') as handle: - for line in handle: - if line.startswith('#') or not line.strip(): - continue - fields = line.rstrip('\n').split('\t') - if len(fields) < 9: - continue - attrs = fields[8] - checked += 1 - - if 'ID=transcript:' in attrs or 'Parent=gene:' in attrs or 'ID=CDS:' in attrs: - ensembl_score += 1 - if '.TAIR' in attrs or 'longest=1' in attrs or 'pacid=' in attrs: - jgi_score += 1 - if 'ID=rna-' in attrs or 'ID=cds-' in attrs or 'gbkey=' in attrs or 'Genbank:' in attrs: - ncbi_score += 1 - - if checked >= 500: - break - - scores = {"ensembl": ensembl_score, "jgi": jgi_score, "ncbi": ncbi_score} - db_type, score = max(scores.items(), key=lambda item: item[1]) - return db_type if score > 0 else "unknown_user" - - def _ensure_script(self, name: str) -> Path: - script = self.scripts_dir / name - if not script.exists(): - raise FileNotFoundError(f"Required script not found: {script}") - return script - - def _ensure_tool(self, name: str) -> None: - if shutil.which(name) is None: - raise RuntimeError(f"Required tool not found in PATH: {name}") - - def _copy_if_needed(self, src: Optional[Path], dst: Path) -> None: - if not src: - return - if src.resolve() == dst.resolve(): - return - shutil.copy2(src, dst) - - def _read_fasta_ids(self, fasta_file: Path) -> List[str]: - ids = [] - with open(fasta_file, 'r', errors='replace') as handle: - for line in handle: - if line.startswith('>'): - ids.append(line[1:].split()[0]) - return ids - - def _write_ids_from_fasta(self, fasta_file: Path, output_file: Path) -> int: - ids = self._read_fasta_ids(fasta_file) - with open(output_file, 'w') as out: - for seq_id in ids: - out.write(f"{seq_id}\n") - return len(ids) - - def _write_ids_from_longest_table(self, table_file: Path, output_file: Path) -> int: - count = 0 - with open(table_file, 'r') as table, open(output_file, 'w') as out: - for line in table: - parts = line.split() - if len(parts) >= 3: - out.write(parts[2] + '\n') - count += 1 - return count - - def _write_map_from_longest_table(self, table_file: Path, output_file: Path) -> int: - count = 0 - with open(table_file, 'r') as table, open(output_file, 'w') as out: - for line in table: - parts = line.split() - if len(parts) >= 3: - out.write(f"{parts[1]}\t{parts[2]}\n") - count += 1 - return count - - def _preprocess_gff(self, gff_file: Path, gff_tmp: Path, db_type: str) -> None: - with open(gff_file, 'r', errors='replace') as infile, open(gff_tmp, 'w') as outfile: - for line in infile: - if db_type == "jgi": - line = re.sub(r'\.TAIR\d*', '', line) - elif db_type == "ensembl": - line = re.sub(r'([=,])(gene|transcript|CDS|cds):', r'\1', line) - elif db_type == "ncbi": - line = re.sub(r'([=,])gene-', r'\1', line) - line = re.sub(r'([=,])rna-', r'\1', line) - line = re.sub(r'([=,])cds-', r'\1', line) - else: - line = re.sub(r'\.TAIR\d*', '', line) - line = re.sub(r'([=,])(gene|transcript|CDS|cds):', r'\1', line) - outfile.write(line) - - def _normalize_ncbi_cds_headers(self, cds_file: Path, output_file: Path) -> Path: - pattern = re.compile(r'^>\S+_cds_(\S+)_[0-9]+\s+') - with open(cds_file, 'r', errors='replace') as infile, open(output_file, 'w') as outfile: - for line in infile: - if line.startswith('>'): - match = pattern.match(line) - if match: - line = '>' + match.group(1) + ' ' + line[match.end():] - outfile.write(line) - return output_file - - def _simplify_ncbi_gff_attrs(self, gff_file: Path) -> None: - """Trim verbose NCBI annotations while keeping structural GFF relationships.""" - keep_order = [ - 'ID', 'Parent', 'Name', 'Alias', 'gene', 'locus_tag', - 'protein_id', 'transcript_id', 'gbkey', 'gene_biotype' - ] - keep = set(keep_order) - tmp_file = gff_file.with_suffix(gff_file.suffix + '.slim_tmp') - with open(gff_file, 'r', errors='replace') as infile, open(tmp_file, 'w') as outfile: - for line in infile: - if line.startswith('#') or not line.strip(): - outfile.write(line) - continue - fields = line.rstrip('\n').split('\t') - if len(fields) != 9: - outfile.write(line) - continue - attrs = {} - for item in fields[8].split(';'): - if not item or '=' not in item: - continue - key, value = item.split('=', 1) - if key in keep and value: - attrs[key] = value - slim_attrs = [] - for key in keep_order: - if key in attrs: - slim_attrs.append(f'{key}={attrs[key]}') - fields[8] = ';'.join(slim_attrs) if slim_attrs else fields[8] - outfile.write('\t'.join(fields) + '\n') - tmp_file.replace(gff_file) - - def _extract_pep_if_needed(self, species: SpeciesFiles, pep_file: Optional[Path], genome_file: Optional[Path], gff_file: Optional[Path]) -> Optional[Path]: - if pep_file or not genome_file or not gff_file: - return pep_file - logger.info("No protein file found, attempting to extract from genome+GFF using gffread...") - self._ensure_tool('gffread') - pep_temp = self.work_dir / f"{species.species_code}_extracted.pep.fa" - cmd = ['gffread', '-y', str(pep_temp), '-g', str(genome_file), str(gff_file)] - rc, _, stderr = self.run_command(cmd, f"Extract protein from genome for {species.species_code}") - if rc == 0 and pep_temp.exists() and pep_temp.stat().st_size > 0: - logger.info(f"Successfully extracted protein sequences to {pep_temp.name}") - return pep_temp - raise RuntimeError(f"No protein file and failed to extract from genome+GFF: {stderr}") - - def _build_artifacts(self, cds_out: Path, pep_out: Path, gff_out: Path, genome_out: Path, genome_file: Optional[Path]) -> Dict[str, Any]: - artifacts: Dict[str, Any] = {} - for key, path, artifact_type in [("cds", cds_out, "fasta"), ("pep", pep_out, "fasta"), ("gff", gff_out, "gff3")]: - if path.exists(): - artifacts[key] = {"artifact": key, "path": str(path.name), "full_path": str(path.absolute()), "type": artifact_type, "size": path.stat().st_size, "sha256": compute_file_hash(path), "mtime": path.stat().st_mtime} - if genome_file and genome_out.exists(): - artifacts["genome"] = {"artifact": "genome", "path": str(genome_out.name), "full_path": str(genome_out.absolute()), "type": "fasta", "size": genome_out.stat().st_size, "sha256": compute_file_hash(genome_out), "mtime": genome_out.stat().st_mtime} - return artifacts - - def _process_reference(self, species: SpeciesFiles, db_type: str, cds_file: Optional[Path], pep_file: Optional[Path]) -> Dict[str, Any]: - id_source = cds_file if cds_file else pep_file - gene_count = self._write_ids_from_fasta(id_source, self.work_dir / f"{species.species_code}_mRNA.id") if id_source else 0 - logger.info("Reference file detected, using lightweight processing") - artifacts = {} - if cds_file: - artifacts["ref_cds"] = {"artifact": "ref_cds", "path": str(cds_file.name), "full_path": str(cds_file.absolute()), "type": "fasta", "size": cds_file.stat().st_size, "sha256": compute_file_hash(cds_file), "mtime": cds_file.stat().st_mtime} - if pep_file: - artifacts["ref_pep"] = {"artifact": "ref_pep", "path": str(pep_file.name), "full_path": str(pep_file.absolute()), "type": "fasta", "size": pep_file.stat().st_size, "sha256": compute_file_hash(pep_file), "mtime": pep_file.stat().st_mtime} - return {"species": species.species_code, "success": True, "db_type": db_type, "gene_count": gene_count, "is_reference": True, "artifacts": artifacts, "files": {"cds": str(cds_file) if cds_file else None, "pep": str(pep_file) if pep_file else None}} - - def _process_jgi_or_unknown(self, species: SpeciesFiles, db_type: str, gff_file: Path, cds_file: Optional[Path], pep_file: Optional[Path], genome_file: Optional[Path], outputs: Dict[str, Path]) -> Tuple[int, Dict[str, Any]]: - filter_script = self._ensure_script('gff_filter_bymRNAID.py') - if not cds_file and not pep_file: - raise RuntimeError("No CDS or PEP file available") - if not pep_file: - pep_file = self._extract_pep_if_needed(species, pep_file, genome_file, gff_file) - id_source = cds_file if cds_file else pep_file - mrna_id_file = self.work_dir / f"{species.species_code}_mRNA.id" - gene_count = self._write_ids_from_fasta(id_source, mrna_id_file) - logger.info(f"Extracted {gene_count} gene IDs from {id_source.name}") - gff_tmp = self.work_dir / f"{species.species_code}.gff3.tmp" - self._preprocess_gff(gff_file, gff_tmp, db_type) - gene_table = self.work_dir / f"{species.species_code}_geneID_mrnaID.table" - cmd = [sys.executable, str(filter_script), str(gff_tmp.resolve()), str(mrna_id_file.resolve()), str(gene_table.resolve()), str(outputs['gff'].resolve())] - rc, _, stderr = self.run_command(cmd, f"Filter GFF for {species.species_code} ({db_type})") - if rc != 0: - raise RuntimeError(f"GFF filtering failed: {stderr}") - self._copy_if_needed(cds_file if cds_file else pep_file, outputs['cds']) - self._copy_if_needed(pep_file, outputs['pep']) - self._copy_if_needed(genome_file, outputs['genome']) - return gene_count, self._build_artifacts(outputs['cds'], outputs['pep'], outputs['gff'], outputs['genome'], genome_file) - - def _process_ensembl(self, species: SpeciesFiles, gff_file: Path, cds_file: Optional[Path], pep_file: Optional[Path], genome_file: Optional[Path], outputs: Dict[str, Path]) -> Tuple[int, Dict[str, Any]]: - self._ensure_tool('seqtk') - longest_script = self._ensure_script('gff_filter_longest.py') - if not cds_file or not pep_file: - raise RuntimeError("Ensembl processing requires CDS and PEP files") - gff_tmp = self.work_dir / f"{species.species_code}.gff3.tmp" - self._preprocess_gff(gff_file, gff_tmp, "ensembl") - longest_table = self.work_dir / f"{species.species_code}_gene_mrna_cds.ids" - cmd = [sys.executable, str(longest_script), str(gff_tmp.resolve()), str(longest_table.resolve()), str(outputs['gff'].resolve())] - rc, _, stderr = self.run_command(cmd, f"Select longest CDS and filter GFF for {species.species_code} (ensembl)") - if rc != 0: - raise RuntimeError(f"Longest CDS filtering failed: {stderr}") - mrna_id_file = self.work_dir / f"{species.species_code}_mRNA.id" - gene_count = self._write_ids_from_longest_table(longest_table, mrna_id_file) - logger.info(f"Selected {gene_count} longest CDS IDs from {longest_table.name}") - for source, output, label in [(cds_file, outputs['cds'], 'CDS'), (pep_file, outputs['pep'], 'PEP')]: - rc, _, stderr = self.run_command_to_file(['seqtk', 'subseq', str(source), str(mrna_id_file)], output, f"Extract Ensembl {label} for {species.species_code}") - if rc != 0: - raise RuntimeError(f"seqtk subseq failed for {label}: {stderr}") - self._copy_if_needed(genome_file, outputs['genome']) - return gene_count, self._build_artifacts(outputs['cds'], outputs['pep'], outputs['gff'], outputs['genome'], genome_file) - - def _process_ncbi(self, species: SpeciesFiles, gff_file: Path, cds_file: Optional[Path], pep_file: Optional[Path], genome_file: Optional[Path], outputs: Dict[str, Path]) -> Tuple[int, Dict[str, Any]]: - self._ensure_tool('seqtk') - longest_script = self._ensure_script('gff_filter_longest.py') - map_script = self._ensure_script('map_gff_ids.py') - if not cds_file or not pep_file: - raise RuntimeError("NCBI processing requires CDS and PEP files") - normalized_cds = self.work_dir / f"{species.species_code}_all.cds.fasta" - self._normalize_ncbi_cds_headers(cds_file, normalized_cds) - gff_tmp = self.work_dir / f"{species.species_code}.gff3.tmp" - self._preprocess_gff(gff_file, gff_tmp, "ncbi") - longest_table = self.work_dir / f"{species.species_code}_gene_mrna_cds.ids" - cmd = [sys.executable, str(longest_script), str(gff_tmp.resolve()), str(longest_table.resolve()), str(outputs['gff'].resolve())] - rc, _, stderr = self.run_command(cmd, f"Select longest CDS and filter GFF for {species.species_code} (ncbi)") - if rc != 0: - raise RuntimeError(f"Longest CDS filtering failed: {stderr}") - mrna_map_file = self.work_dir / f"{species.species_code}_mRNA.mapid" - self._write_map_from_longest_table(longest_table, mrna_map_file) - cmd = [sys.executable, str(map_script), str(mrna_map_file.resolve()), str(outputs['gff'].resolve())] - rc, _, stderr = self.run_command(cmd, f"Map NCBI mRNA IDs to CDS IDs for {species.species_code}") - if rc != 0: - raise RuntimeError(f"NCBI GFF ID mapping failed: {stderr}") - self._simplify_ncbi_gff_attrs(outputs['gff']) - mrna_id_file = self.work_dir / f"{species.species_code}_mRNA.id" - gene_count = self._write_ids_from_longest_table(longest_table, mrna_id_file) - logger.info(f"Selected {gene_count} longest NCBI CDS IDs from {longest_table.name}") - for source, output, label in [(normalized_cds, outputs['cds'], 'CDS'), (pep_file, outputs['pep'], 'PEP')]: - rc, _, stderr = self.run_command_to_file(['seqtk', 'subseq', str(source), str(mrna_id_file)], output, f"Extract NCBI {label} for {species.species_code}") - if rc != 0: - raise RuntimeError(f"seqtk subseq failed for {label}: {stderr}") - self._copy_if_needed(genome_file, outputs['genome']) - return gene_count, self._build_artifacts(outputs['cds'], outputs['pep'], outputs['gff'], outputs['genome'], genome_file) - - def process_species(self, species: SpeciesFiles) -> Dict[str, Any]: - """???????""" - logger.info(f"\n{'='*60}") - logger.info(f"Processing species: {species.species_code}") - logger.info(f"{'='*60}") - gene_family = self._extract_gene_family(species) - outputs = {"cds": self.input_dir / f"{species.species_code}.{gene_family}.cds.fasta", "pep": self.input_dir / f"{species.species_code}.{gene_family}.pep.fasta", "genome": self.input_dir / f"{species.species_code}.{gene_family}.genome.fasta", "gff": self.input_dir / f"{species.species_code}.{gene_family}_final.gff3"} - is_ref = self._is_reference_file(species) - try: - gff_file = self.decompress_if_needed(species.gff) if species.gff else None - genome_file = self.decompress_if_needed(species.genome) if species.genome else None - cds_file = self.decompress_if_needed(species.cds) if species.cds else None - pep_file = self.decompress_if_needed(species.pep) if species.pep else None - db_type = self.detect_db_type_from_gff(gff_file) - species.db_type = db_type - logger.info(f"Detected database type from GFF for {species.species_code}: {db_type}") - if is_ref: - return self._process_reference(species, db_type, cds_file, pep_file) - if not gff_file: - raise RuntimeError("Target species requires a GFF file for genome_prepare") - if db_type == "jgi": - gene_count, artifacts = self._process_jgi_or_unknown(species, db_type, gff_file, cds_file, pep_file, genome_file, outputs) - elif db_type == "ensembl": - gene_count, artifacts = self._process_ensembl(species, gff_file, cds_file, pep_file, genome_file, outputs) - elif db_type == "ncbi": - gene_count, artifacts = self._process_ncbi(species, gff_file, cds_file, pep_file, genome_file, outputs) - else: - gene_count, artifacts = self._process_jgi_or_unknown(species, "unknown_user", gff_file, cds_file, pep_file, genome_file, outputs) - logger.info(f"{species.species_code} processing complete") - return {"species": species.species_code, "success": True, "db_type": db_type, "gene_count": gene_count, "artifacts": artifacts, "files": {"cds": str(outputs['cds']), "pep": str(outputs['pep']), "gff": str(outputs['gff']), "genome": str(outputs['genome']) if genome_file else None}} - except Exception as e: - logger.error(f"{species.species_code} processing failed: {e}") - return {"species": species.species_code, "success": False, "db_type": getattr(species, 'db_type', 'unknown_user'), "error": str(e)} - - def process_all(self) -> List[Dict[str, Any]]: - """???????""" - species_map = MultiSpeciesDetector.scan_input_dir(self.input_dir) - logger.info(f"\nDetected {len(species_map)} species:") - for code, species in species_map.items(): - logger.info(f" - {species}") - results = [] - for species in species_map.values(): - result = self.process_species(species) - results.append(result) - return results - -def scan_hmm_files(input_dir: Path) -> List[Dict[str, Any]]: - """扫描 HMM 文件 - - FIX: 新增函数，扫描 input_dir 下的 .hmm 文件 - """ - hmm_files = [] - for file in input_dir.iterdir(): - if file.is_file() and file.suffix == '.hmm': - hmm_files.append({ - "path": str(file.name), - "full_path": str(file.absolute()), - "sha256": compute_file_hash(file), - "size": file.stat().st_size, - "mtime": file.stat().st_mtime - }) - return hmm_files - - -def write_manifest( - output_dir: Path, - job_id: str, - run_id: str, - prefix: str, - start_time: datetime, - end_time: datetime, - results: List[Dict[str, Any]], - tool_versions: Dict[str, str], - cmd_history: List[str], - input_dir: Path, - status: str = "success", - merge_result: Optional[Dict[str, Any]] = None, - species_config: Optional[Path] = None, - workspace_dir: Optional[Path] = None # REVISION 2026-04-28: 新增workspace_dir参数 -) -> Path: - """ - 写入符合schema v1的manifest文件 - - 注意：artifacts必须是字典格式，不是列表 - """ - elapsed_sec = (end_time - start_time).total_seconds() - success_count = sum(1 for r in results if r.get("success")) - total_count = len(results) - - # 确定状态（确保不为None） - if success_count == 0: - status = "failed" - elif success_count < total_count: - status = "partial_success" - elif status is None: - status = "success" - - # 收集所有 artifacts - all_artifacts = {} - for result in results: - if result.get("success") and "artifacts" in result: - species = result.get("species", "unknown") - for key, artifact in result["artifacts"].items(): - artifact_key = f"{species}_{key}" - all_artifacts[artifact_key] = artifact - - # FIX: 扫描 HMM 文件 - hmm_files = scan_hmm_files(input_dir) - - # FIX: 构建 inputs 列表（从子目录收集） - inputs_list = [] - for species_dir in input_dir.iterdir(): - if species_dir.is_dir() and not species_dir.name.startswith('.'): - for file in species_dir.iterdir(): - if file.is_file(): - inputs_list.append({ - "path": f"{species_dir.name}/{file.name}", - "full_path": str(file.absolute()), - "sha256": compute_file_hash(file), - "size": file.stat().st_size, - "mtime": file.stat().st_mtime - }) - - # FIX: 添加合并后的文件信息 - merged_data = {} - if merge_result and merge_result.get("merged"): - merged_data = { - "target": { - "species": merge_result.get("target_species", []), - "files": merge_result.get("merged_files", {}) - }, - "reference": { - "species": merge_result.get("ref_species", []), - "files": {k: v for k, v in merge_result.get("merged_files", {}).items() if k.startswith("ref_")} - } - } - - manifest_data = { - # 基础标识字段（必需） - "manifest_version": "1.0", - "job_id": job_id, - "stage": "genome_prepare_v2", - "run_id": run_id, - "prefix": prefix, - - # FIX: 添加物种配置路径 - "species_config": str(species_config) if species_config else None, - - # 状态和时间字段（必需） - "timestamp": end_time.isoformat(), - "status": status, - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - "elapsed_sec": round(elapsed_sec, 2), - - # 执行信息（必需） - "cmd": cmd_history, - "tool_versions": tool_versions, - - # 目录路径（必需） - "workspace_root": str(output_dir.parent), - "workspace": str(output_dir), - "input_dir": str(input_dir), - - # 输入文件（必需） - "inputs": inputs_list, - - # FIX: HMM 文件单独记录 - "hmm": hmm_files, - - # 输出产物（必需）- 字典格式 - "artifacts": all_artifacts, - - # FIX: 添加合并后的文件信息 - "merged": merged_data, - - # 参数和指标（必需） - "parameters": { - "total_species": total_count, - "success_species": success_count, - "failed_species": total_count - success_count - }, - "metrics": { - "species_results": results - }, - - # 修订 2026-04-26: 更新合并文件信息 - "merged_files": { - "target": { - "prefix": None, # 不再合并目标物种 - "cds": None, - "pep": None, - "gff": None - }, - "reference": { - "prefix": f"Ref_combined.{prefix}", # Ref. -> Ref_combined. - "cds": f"Ref_combined.{prefix}.cds.fasta" if (input_dir / f"Ref_combined.{prefix}.cds.fasta").exists() else None, - "pep": f"Ref_combined.{prefix}.pep.fasta" if (input_dir / f"Ref_combined.{prefix}.pep.fasta").exists() else None - } - }, - - # 溯源信息（可选但推荐） - "provenance": { - "derived_from": "raw_genome_data", - "stage_sequence": ["genome_prepare_v2"] - } - } - - # 确保输出到 final/ 子目录 - final_dir = output_dir / "final" - final_dir.mkdir(parents=True, exist_ok=True) - - manifest_path = final_dir / f"{prefix}.genome_prepare.manifest.json" - - print(f"生成Manifest文件: {manifest_path}") - - # 确保JSON严格性：无注释，确保ASCII - with open(manifest_path, "w", encoding='utf-8') as f: - json.dump(manifest_data, f, indent=2, ensure_ascii=False) - - # 确认文件已生成 - if manifest_path.exists(): - print(f" ✅ Manifest文件已生成: {manifest_path} ({manifest_path.stat().st_size} 字节)") - - # REVISION 2026-04-28: 同时复制manifest到input_dir/final/目录 - # 因为实际输出文件都在input_dir，后端API会去input_dir查找 - input_final_dir = input_dir / "final" - input_final_dir.mkdir(parents=True, exist_ok=True) - input_manifest = input_final_dir / manifest_path.name - try: - shutil.copy2(str(manifest_path), str(input_manifest)) - print(f" ✅ Manifest已复制到input目录: {input_manifest}") - except Exception as e: - print(f" ⚠️ Manifest复制到input目录失败: {e}") - - # REVISION 2026-04-28: 同时复制manifest到workspace/final/目录 - # 供后端API查找（pipeline_backend.py去workspace查找manifest） - if workspace_dir: - workspace_final_dir = workspace_dir / "final" - workspace_final_dir.mkdir(parents=True, exist_ok=True) - workspace_manifest = workspace_final_dir / manifest_path.name - try: - shutil.copy2(str(manifest_path), str(workspace_manifest)) - print(f" ✅ Manifest已复制到workspace目录: {workspace_manifest}") - except Exception as e: - print(f" ⚠️ Manifest复制到workspace目录失败: {e}") - else: - print(f" ❌ Manifest文件生成失败: {manifest_path}") - - return manifest_path - - -def write_report( - output_dir: Path, - prefix: str, - results: List[Dict[str, Any]], - tool_versions: Dict[str, str], - parameters: Dict[str, Any] -) -> Path: - """写入人可读的分析报告""" - # 确保输出到 final/ 子目录 - final_dir = output_dir / "final" - final_dir.mkdir(parents=True, exist_ok=True) - - report_path = final_dir / f"{prefix}.genome_prepare_report.md" - - success_count = sum(1 for r in results if r.get("success")) - total_count = len(results) - - print(f"生成分析报告: {report_path}") - - with open(report_path, 'w') as f: - f.write(f"# Genome Prepare V2 分析报告 - {prefix}\n\n") - f.write(f"**生成时间**: {datetime.now().isoformat()}\n\n") - - f.write("## 1. 分析概述\n") - f.write(f"- **分析阶段**: Genome Prepare V2\n") - f.write(f"- **分析状态**: {'✅ 成功' if success_count == total_count else '⚠️ 部分成功' if success_count > 0 else '❌ 失败'}\n") - f.write(f"- **总物种数**: {total_count}\n") - f.write(f"- **成功**: {success_count}\n") - f.write(f"- **失败**: {total_count - success_count}\n\n") - - f.write("## 2. 工具版本\n") - for tool, version in tool_versions.items(): - f.write(f"- **{tool}**: {version}\n") - f.write("\n") - - f.write("## 3. 执行参数\n") - for param, value in parameters.items(): - f.write(f"- **{param}**: {value}\n") - f.write("\n") - - f.write("## 4. 物种详细信息\n\n") - for result in results: - species = result.get("species", "Unknown") - success = result.get("success", False) - - f.write(f"### {species}\n") - f.write(f"- **状态**: {'✅ 成功' if success else '❌ 失败'}\n") - - if success: - f.write(f"- **数据库类型**: {result.get('db_type', 'unknown')}\n") - f.write(f"- **基因数量**: {result.get('gene_count', 'N/A')}\n") - - # 输出文件信息（使用 artifacts 格式） - artifacts = result.get("artifacts", {}) - if artifacts: - f.write("- **输出文件**:\n") - for artifact_name, artifact_info in artifacts.items(): - if isinstance(artifact_info, dict): - f.write(f" - **{artifact_name}**: {artifact_info.get('path', 'N/A')}\n") - if artifact_info.get('size'): - f.write(f" - 大小: {artifact_info['size']} 字节\n") - if artifact_info.get('sha256'): - f.write(f" - SHA256: {artifact_info['sha256'][:16]}...\n") - else: - f.write(f"- **错误信息**: {result.get('error', '未知错误')}\n") - - f.write("\n") - - f.write("## 5. 输出文件汇总\n") - for result in results: - if result.get("success"): - species = result.get("species", "Unknown") - artifacts = result.get("artifacts", {}) - for artifact_name, artifact_info in artifacts.items(): - if isinstance(artifact_info, dict) and artifact_info.get('path'): - f.write(f"- `{artifact_info['path']}`\n") - f.write("\n") - - return report_path - - -def main(): - parser = argparse.ArgumentParser( - description="Genome Prepare V2 - 多物种基因组数据预处理", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -示例： - python run_genome_prepare_v2.py -i /path/to/input -o /path/to/output -p multi_species - -输入要求： - - 输入目录应包含原始基因组数据文件 - - 支持的文件格式：.fasta, .fa, .gff3, .gff (可带.gz压缩) - - 文件名应包含物种标识（如 Ath, Ft, Os 等） - -输出产物： - - {species}.cds.fasta (标准化CDS序列) - - {species}.pep.fasta (标准化蛋白序列) - - {species}.genome.fasta (标准化基因组，如有) - - {species}_final.gff3 (标准化GFF注释) - - {prefix}.genome_prepare.manifest.json (机器清单) - - {prefix}.genome_prepare_report.md (分析报告) - """ - ) - parser.add_argument("-i", "--input", required=True, help="输入目录路径（包含原始基因组数据）") - parser.add_argument("-o", "--output", required=True, help="输出目录路径（准备的fasta文件输出位置）") - parser.add_argument("-w", "--workspace", help="工作目录路径（用于final/logs等标准输出，默认与--output相同）") - parser.add_argument("-p", "--prefix", default="multi_species", help="输出文件前缀（默认：multi_species）") - parser.add_argument("--scripts-dir", default="/home/xia-lab/wutx/00test/bbav5_copy_yang_7870/scripts", help="Perl脚本目录") - - args = parser.parse_args() - - input_dir = Path(args.input) - output_dir = Path(args.output) - - # REVISION 2026-04-28: 如果prefix为空或默认值，从config.xlsx读取Gene_Family生成 - # 规则: 取config.xlsx第三列"Gene_Family"的第一个值，格式为"Multi.{Gene_Family}" - if not args.prefix or args.prefix == "multi_species" or args.prefix.strip() == "": - config_path = input_dir / "config.xlsx" - if config_path.exists() and pd is not None: - try: - df = pd.read_excel(config_path) - if 'Gene_Family' in df.columns and len(df) > 0: - # 数据从第二行开始（pandas读取后第一行是数据，索引0） - gene_family = str(df['Gene_Family'].iloc[0]).strip() - if gene_family and gene_family != 'nan': - args.prefix = f"Multi.{gene_family}" - logger.info(f"Auto-generated prefix from config.xlsx: {args.prefix}") - else: - logger.warning("Gene_Family value is empty, using default prefix") - else: - logger.warning("Gene_Family column not found in config.xlsx, using default prefix") - except Exception as e: - logger.warning(f"Failed to read config.xlsx for prefix generation: {e}") - else: - logger.warning("config.xlsx not found, using default prefix") - output_dir.mkdir(parents=True, exist_ok=True) - # FIX: 支持双输出位置，workspace用于标准输出，output用于准备的fasta文件 - # FIX: 标准化输出到 input_dir 根目录（平铺），而不是 output_dir - workspace_dir = Path(args.workspace) if args.workspace else output_dir - workspace_dir.mkdir(parents=True, exist_ok=True) - scripts_dir = Path(args.scripts_dir) - - # FIX: 创建 genome_prepare 过程文件夹 - genome_prepare_dir = input_dir / "genome_prepare" - genome_prepare_dir.mkdir(exist_ok=True) - - start_time = datetime.now() - tool_versions = get_tool_versions() - - # FIX: 步骤0 - 根据 config.xlsx 组织文件 - print("\n📋 根据 config.xlsx 组织文件...") - try: - org_result = organize_files_by_config(input_dir) - if org_result.get("organized"): - print(f" ✅ 文件组织完成") - print(f" 物种: {org_result.get('species_codes', [])}") - print(f" 移动: {len(org_result.get('moved_files', []))} 个文件") - if org_result.get('missing_files'): - print(f" 缺失: {len(org_result['missing_files'])} 个文件") - else: - print(f" ⚠️ 跳过文件组织: {org_result.get('reason', 'unknown')}") - except Exception as e: - print(f" ⚠️ 文件组织失败: {e}") - - # 1. 验证输入目录 - print("\n🔍 验证输入目录...") - try: - input_info = validate_input_directory(input_dir) - print(f" ✅ 输入验证通过，检测到 {len(input_info['species_map'])} 个有效物种") - except Exception as e: - print(f" ❌ 输入验证失败: {e}") - sys.exit(1) - - # 2. 准备目录结构 - print("\n📁 准备目录结构...") - print(f" 工作目录(workspace): {workspace_dir}") - print(f" 输出目录(output): {output_dir}") - paths = prepare_directories(workspace_dir) - - # 3. 获取工具版本 - print("\n🔧 检查工具版本...") - tool_versions = get_tool_versions() - print(f" Python: {tool_versions.get('python', 'unknown')}") - print(f" Perl: {tool_versions.get('perl', 'unknown')}") - print(f" gffread: {tool_versions.get('gffread', 'unknown')}") - print(f" seqtk: {tool_versions.get('seqtk', 'unknown')}") - - # 4. 创建预处理器并执行 - # FIX: 使用 input_dir 作为输出目录（标准化文件输出到 input 根目录） - # FIX: 修改 work_dir 为 genome_prepare_dir - paths["work_dir"] = genome_prepare_dir - preparer = MultiSpeciesGenomePreparer(input_dir, input_dir, scripts_dir, paths) - - try: - # 处理所有物种 - print("\n🚀 开始处理物种...") - results = preparer.process_all() - - # 记录结束时间 - end_time = datetime.now() - - # 生成运行ID - run_id = f"genome_prepare_{start_time.strftime('%Y%m%d_%H%M%S_%f')[:-3]}" - job_id = output_dir.name - - # FIX: 步骤5 - 合并物种文件 - merge_result = None - config_path = None - print("\n🔗 合并物种文件...") - try: - if org_result.get("organized"): - merge_result = merge_species_files(input_dir, genome_prepare_dir, org_result) - if merge_result.get("merged"): - print(f" ✅ 文件合并完成") - print(f" 目标物种: {merge_result.get('target_species', [])}") - print(f" 参考物种: {merge_result.get('ref_species', [])}") - - # FIX: 生成 species_config.json - config_path = generate_species_config(input_dir, genome_prepare_dir, org_result) - print(f" ✅ 物种配置已生成: {config_path}") - else: - print(f" ⚠️ 文件合并失败") - else: - print(f" ⚠️ 跳过文件合并（未组织文件）") - except Exception as e: - print(f" ⚠️ 文件合并失败: {e}") - - # 6. 生成manifest和报告 - print("\n📝 生成输出文件...") - print(f" DEBUG: prefix={args.prefix}, job_id={job_id}, run_id={run_id}") - manifest_file = write_manifest( - output_dir=output_dir, - job_id=job_id, - run_id=run_id, - prefix=args.prefix, - start_time=start_time, - end_time=end_time, - results=results, - tool_versions=tool_versions, - cmd_history=preparer.cmd_history, - input_dir=input_dir, - merge_result=merge_result, - species_config=config_path, - workspace_dir=workspace_dir # REVISION 2026-04-28: 传入workspace_dir用于复制manifest - ) - - report_file = write_report( - output_dir=output_dir, - prefix=args.prefix, - results=results, - tool_versions=tool_versions, - parameters={"input_dir": str(input_dir), "output_dir": str(output_dir)} - ) - - # 6. 输出总结 - success_count = sum(1 for r in results if r.get("success")) - print(f"\n{'='*60}") - print(f"🎉 Genome Prepare V2 完成!") - print(f"{'='*60}") - print(f"总物种数: {len(results)}") - print(f"成功: {success_count}") - print(f"失败: {len(results) - success_count}") - print(f"Manifest: {manifest_file}") - print(f"报告: {report_file}") - - # 根据成功率返回退出码 - if success_count == 0: - sys.exit(1) - elif success_count < len(results): - sys.exit(2) # 部分成功 - - except Exception as e: - print(f"\n❌ Genome prepare 失败: {e}") - import traceback - traceback.print_exc() - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/bioflow_kernel/cli/run_identify.py.bak_family_allpep_20260504_193257 b/bioflow_kernel/cli/run_identify.py.bak_family_allpep_20260504_193257 deleted file mode 100755 index afacaed..0000000 --- a/bioflow_kernel/cli/run_identify.py.bak_family_allpep_20260504_193257 +++ /dev/null @@ -1,1802 +0,0 @@ -#!/usr/bin/env python3 -""" -identify stage (web version) - 修订版 -使用原始用户上传文件，而不是预处理文件 - -Main tasks: -- find HMM: user uploaded, or fetch from Pfam-A.hmm by required_pfams. -- hmmsearch with HMM vs protein sequences. -- domain QC and final member selection. - -Inputs: -- input// (包含用户上传的所有原始文件) -- 不再使用 workspace//prepared/ 中的预处理文件 - -Outputs: -- workspace//final/.identify_report.md -- workspace//final/.final_members.pep.fa -- workspace//final/.final_members.id -- workspace//final/.identify.manifest.json - -Design: -- 直接从 input// 查找原始蛋白质序列文件 -- 用户上传的HMM优先使用 -- 如果没有HMM，则从Pfam数据库获取 -- 直接使用原始文件进行hmmsearch和blast分析 -""" -import os -import gzip -import shutil -import psutil -import argparse -import json -import logging -import re -import shlex -import subprocess -import sys -import time -import hashlib -import platform -from datetime import datetime, timedelta -from pathlib import Path -from typing import List, Optional, Tuple, Dict, Any, Union, Set - - -# 添加conda环境bin目录到PATHCONDA_BIN = "/home/xia-lab/miniconda3/envs/test4py311/bin"if CONDA_BIN not in os.environ.get("PATH", ""): os.environ["PATH"] = CONDA_BIN + ":" + os.environ.get("PATH", "") -def get_pfam_from_rag(gene_family_name: str, rag_db_path: Path = None) -> str: - """从RAG数据库获取Pfam ID""" - if rag_db_path is None: - # 使用相对于脚本的路径 - script_dir = Path(__file__).parent.parent - rag_db_path = script_dir / "data" / "rag_db" / "pfam_mapping.json" - - if not rag_db_path.exists(): - print(f"⚠️ RAG数据库不存在: {rag_db_path}") - return None - - try: - with open(rag_db_path, "r", encoding="utf-8") as f: - rag_db = json.load(f) - - # 精确匹配 - family_lower = gene_family_name.lower() - for key, value in rag_db.items(): - if key.lower() == family_lower: - print(f"✅ 从RAG数据库找到精确匹配: {key} -> {value}") - return value - - # 模糊匹配（包含关系） - for key, value in rag_db.items(): - if family_lower in key.lower() or key.lower() in family_lower: - print(f"🔍 RAG数据库模糊匹配: {key} -> {value}") - return value - - print(f"❌ RAG数据库中未找到基因家族: {gene_family_name}") - return None - except Exception as e: - print(f"❌ 读取RAG数据库失败: {e}") - return None - - -def parse_gene_family_from_prefix(prefix: str) -> str: - """从prefix中提取基因家族名称""" - # 例如: "Ft.SPL" -> "SPL", "SPL.Ft" -> "SPL" - parts = prefix.split('.') - for part in parts: - if len(part) > 1 and not part[0].isdigit(): - # 返回第一个看起来像基因家族名称的部分 - return part - return parts[-1] if parts else prefix - - -def run_cmd(cmd: List[str], cwd: Path = None, env=None, shell: bool = False) -> Tuple[int, str, str]: - """运行命令并返回 (rc, stdout, stderr)""" - print(f"CMD: {' '.join(cmd)}") - if cwd: - print(f"CWD: {cwd}") - if shell: - # shell模式下，cmd可以是字符串 - print(f"SHELL CMD: {cmd}") - p = subprocess.run(cmd, shell=True, capture_output=True, text=True, cwd=cwd, env=env) - else: - p = subprocess.run(cmd, capture_output=True, text=True, cwd=cwd, env=env) - if p.stdout.strip(): - print(p.stdout) - if p.stderr.strip(): - print(p.stderr) - return p.returncode, p.stdout, p.stderr - - - -def sanitize_fasta_for_interproscan(pep_file: Path, clean_fasta: Path) -> Dict[str, Any]: - """Create an InterProScan-safe FASTA by removing terminal/ORF stop markers (*).""" - stats = { - "source": str(pep_file), - "clean_fasta": str(clean_fasta), - "sequence_count": 0, - "modified_sequences": 0, - "removed_asterisks": 0, - } - clean_fasta.parent.mkdir(parents=True, exist_ok=True) - current_had_change = False - with open(pep_file, "r", encoding="utf-8", errors="ignore") as infile, open(clean_fasta, "w", encoding="utf-8") as outfile: - for line in infile: - if line.startswith(">"): - if stats["sequence_count"] and current_had_change: - stats["modified_sequences"] += 1 - stats["sequence_count"] += 1 - current_had_change = False - outfile.write(line) - continue - removed = line.count("*") - if removed: - stats["removed_asterisks"] += removed - current_had_change = True - line = line.replace("*", "") - outfile.write(line) - if stats["sequence_count"] and current_had_change: - stats["modified_sequences"] += 1 - return stats - - -def summarize_interproscan_tsv(tsv_file: Path) -> Dict[str, Any]: - """Return small summary metrics for an InterProScan TSV.""" - summary = {"rows": 0, "proteins_with_hits": 0, "interpro_entries": 0, "signatures": 0} - proteins = set() - interpros = set() - signatures = set() - if not tsv_file.exists(): - return summary - with open(tsv_file, "r", encoding="utf-8", errors="ignore") as fh: - for line in fh: - if not line.strip() or line.startswith("#"): - continue - cols = line.rstrip("\n").split("\t") - summary["rows"] += 1 - if cols: - proteins.add(cols[0]) - if len(cols) > 4 and cols[4]: - signatures.add(cols[4]) - if len(cols) > 11 and cols[11]: - interpros.add(cols[11]) - summary.update({ - "proteins_with_hits": len(proteins), - "interpro_entries": len(interpros), - "signatures": len(signatures), - }) - return summary - - -def write_interproscan_status_file(status_file: Path, result: Dict[str, Any]) -> None: - """Write a small human-readable status file so the frontend always has a visible domain-check artifact.""" - status_file.parent.mkdir(parents=True, exist_ok=True) - lines = [ - "InterProScan domain check", - f"status: {result.get('status', 'unknown')}", - f"input: {result.get('input', '')}", - f"output: {result.get('output', '')}", - f"tool: {result.get('tool', '')}", - f"conda_env: {result.get('conda_env', '')}", - ] - if result.get("clean_input"): - lines.append(f"clean_input: {result.get('clean_input')}") - if result.get("sanitize"): - s = result["sanitize"] - lines.append( - "sanitize: " - f"sequences={s.get('sequence_count', 0)}, " - f"modified={s.get('modified_sequences', 0)}, " - f"removed_asterisks={s.get('removed_asterisks', 0)}" - ) - if result.get("summary"): - s = result["summary"] - lines.append( - "summary: " - f"rows={s.get('rows', 0)}, proteins_with_hits={s.get('proteins_with_hits', 0)}, " - f"interpro_entries={s.get('interpro_entries', 0)}, signatures={s.get('signatures', 0)}" - ) - if result.get("returncode") is not None: - lines.append(f"returncode: {result.get('returncode')}") - if result.get("elapsed_sec") is not None: - lines.append(f"elapsed_sec: {result.get('elapsed_sec')}") - if result.get("reason"): - lines.extend(["", "reason:", str(result.get("reason"))]) - status_file.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def run_interproscan_domain_check(pep_file: Path, output_tsv: Path) -> Dict[str, Any]: - """Run InterProScan on final members as a non-blocking domain check artifact.""" - script = Path(os.environ.get( - "BBA_INTERPROSCAN_SH", - "/home/xia-lab/wutx/02tool/interproscan5.77/interproscan-5.77-108.0/interproscan.sh", - )) - conda_env = os.environ.get("BBA_INTERPROSCAN_ENV", "interproscanpy311") - conda_sh = Path(os.environ.get("BBA_CONDA_SH", "/home/xia-lab/miniconda3/etc/profile.d/conda.sh")) - - result = { - "artifact": "interproscan_domain_check", - "status": "skipped", - "input": str(pep_file), - "output": str(output_tsv), - "tool": str(script), - "conda_env": conda_env, - } - - if not pep_file.exists(): - result["reason"] = f"input FASTA not found: {pep_file}" - print(f"InterProScan domain check skipped: {result['reason']}") - return result - if not script.exists(): - result["reason"] = f"interproscan.sh not found: {script}" - print(f"InterProScan domain check skipped: {result['reason']}") - return result - if not conda_sh.exists(): - result["reason"] = f"conda activation script not found: {conda_sh}" - print(f"InterProScan domain check skipped: {result['reason']}") - return result - - output_tsv.parent.mkdir(parents=True, exist_ok=True) - if output_tsv.exists(): - output_tsv.unlink() - clean_fasta = output_tsv.with_suffix(".input.pep.fa") - sanitize_stats = sanitize_fasta_for_interproscan(pep_file, clean_fasta) - scan_input = clean_fasta - result["clean_input"] = str(scan_input) - result["sanitize"] = sanitize_stats - - cmd = ( - f"source {shlex.quote(str(conda_sh))} && " - f"conda activate {shlex.quote(conda_env)} && " - f"{shlex.quote(str(script))} -i {shlex.quote(str(scan_input))} -f TSV -o {shlex.quote(str(output_tsv))} && " - "conda deactivate" - ) - print("Running InterProScan domain check:") - print(f" {cmd}") - start = time.time() - proc = subprocess.run(["bash", "-lc", cmd], capture_output=True, text=True) - elapsed = round(time.time() - start, 2) - if proc.stdout.strip(): - print(proc.stdout) - if proc.stderr.strip(): - print(proc.stderr) - - result.update({ - "returncode": proc.returncode, - "elapsed_sec": elapsed, - "cmd": cmd, - }) - if proc.returncode == 0 and output_tsv.exists(): - summary = summarize_interproscan_tsv(output_tsv) - result.update({ - "status": "success", - "full_path": str(output_tsv.resolve()), - "relpath_or_hint": f"final/{output_tsv.name}", - "size": output_tsv.stat().st_size, - "summary": summary, - }) - print(f"InterProScan domain check completed: {output_tsv}") - else: - result.update({ - "status": "failed", - "reason": (proc.stderr or proc.stdout or "InterProScan failed without output")[-2000:], - }) - print(f"InterProScan domain check failed, but identify will continue: {result['reason']}") - return result - -def prepare_directories(workspace: Path) -> Dict[str, Path]: - """创建Identify阶段的标准目录结构""" - paths = { - "analysis_dir": workspace / "identify" / "analysis", - "combine_dir": workspace / "identify" / "combine", - "final_dir": workspace / "final", - "logs_dir": workspace / "logs", - } - - print("创建目录结构:") - for name, path in paths.items(): - path.mkdir(parents=True, exist_ok=True) - print(f" {name}: {path}") - - return paths - - -def write_manifest(manifest_path: Path, data: dict): - """写入 manifest.json""" - manifest_path.parent.mkdir(parents=True, exist_ok=True) - with open(manifest_path, 'w', encoding='utf-8') as f: - json.dump(data, f, indent=2, ensure_ascii=False) - print(f"Manifest: {manifest_path}") - - -def _is_user_hmm_name(filename: str) -> bool: - """Return True for user query HMM files, excluding bundled/system Pfam DBs.""" - n = filename.lower() - if 'pfam-a' in n or 'pfam_a' in n: - return False - return n.endswith('.hmm') or n.endswith('.hmm.gz') - - -def _decompress_hmm_gz(hmm_gz: Path, cache_dir: Path) -> Path: - """Decompress a user-uploaded .hmm.gz into a stable cache and return .hmm path.""" - cache_dir.mkdir(parents=True, exist_ok=True) - out_name = hmm_gz.name[:-3] # PFxxxxx.hmm.gz -> PFxxxxx.hmm - out_path = cache_dir / out_name - - if out_path.exists() and out_path.stat().st_mtime >= hmm_gz.stat().st_mtime: - print(f" Using decompressed HMM cache: {out_path}") - return out_path - - print(f" Decompressing user HMM: {hmm_gz.name} -> {out_path}") - with gzip.open(hmm_gz, 'rb') as src, open(out_path, 'wb') as dst: - shutil.copyfileobj(src, dst) - return out_path - - -def find_user_query_hmms(input_dir: Path) -> List[Path]: - """Find user-uploaded .hmm/.hmm.gz files usable by hmmsearch.""" - print(f"Searching user HMM files in: {input_dir}") - - hmms = [] - cache_dir = input_dir / "genome_prepare" / "hmm_cache" - for p in input_dir.iterdir(): - if not p.is_file(): - continue - if not _is_user_hmm_name(p.name): - continue - - n = p.name.lower() - if n.endswith('.hmm.gz'): - hmm_path = _decompress_hmm_gz(p, cache_dir) - hmms.append((hmm_path, p.stat().st_mtime)) - print(f" Found compressed user HMM: {p.name}") - else: - hmms.append((p, p.stat().st_mtime)) - print(f" Found user HMM: {p.name}") - - if not hmms: - print(" No user-uploaded HMM files found") - return [] - - hmms.sort(key=lambda item: item[1], reverse=True) - result = [item[0] for item in hmms] - print(f"Found {len(result)} user HMM file(s)") - return result - -def find_pep_file(input_dir: Path) -> Optional[Path]: - """ - 在 input_dir 中查找蛋白质序列文件 - 支持多种命名格式： - - *.pep.fasta, *.pep.fa - - *.protein.fasta, *.protein.fa - - *.faa - - 或者包含 "pep" 或 "protein" 的fasta文件 - """ - print(f"🔍 查找蛋白质序列文件在: {input_dir}") - - patterns = ["*.pep.fasta", "*.pep.fa", "*.protein.fasta", "*.protein.fa", "*.faa", "*.fasta"] - - for pattern in patterns: - for file in input_dir.rglob(pattern): - if file.is_file(): - # 检查文件名是否包含pep或protein（不区分大小写） - filename = file.name.lower() - if "pep" in filename or "protein" in filename: - print(f"✅ 找到蛋白质序列文件: {file}") - return file - # 对于通用的.fasta或.faa文件，也考虑 - if pattern in ["*.fasta", "*.faa"] and file.stat().st_size > 10000: # 大于10KB - print(f"✅ 找到可能的蛋白质序列文件: {file}") - return file - - print("❌ 未找到蛋白质序列文件") - return None - -def find_pep_files(input_dir: Path) -> List[Path]: - """ - 查找所有目标物种的蛋白质序列文件 - 排除 Ref_combined.* 文件（参考物种） - - 用于多物种基因家族鉴定 - - Args: - input_dir: 输入目录 - - Returns: - 目标物种 pep 文件列表（排序后） - """ - print(f"🔍 查找目标物种蛋白质序列文件在: {input_dir}") - - target_peps = [] - for f in input_dir.glob("*.pep.fasta"): - if not f.name.startswith("Ref_combined"): - target_peps.append(f) - print(f" ✅ 目标物种: {f.name}") - - if not target_peps: - print(" ❌ 未找到目标物种 pep 文件") - else: - print(f" 📊 共找到 {len(target_peps)} 个目标物种") - - return sorted(target_peps) - - -def find_reference_file_simple(input_dir: Path) -> Optional[Path]: - """ - 简化版：查找参考物种合并文件 - 命名规范: Ref_combined.{gene_family}.pep.fasta - - Args: - input_dir: 输入目录 - - Returns: - 参考物种 pep 文件路径，未找到返回 None - """ - print(f"🔍 查找参考物种文件在: {input_dir}") - - ref_files = list(input_dir.glob("Ref_combined.*.pep.fasta")) - if ref_files: - print(f" ✅ 参考物种: {ref_files[0].name}") - return ref_files[0] - - print(" ❌ 未找到参考物种文件 Ref_combined.*.pep.fasta") - return None - - -def extract_gene_family(input_dir: Path) -> str: - """ - 从文件名提取基因家族名称 - 优先从 Ref_combined 文件提取，其次从目标文件 - - 命名规范: {species}.{gene_family}.pep.fasta - 例如: Ath.SPL.pep.fasta -> "SPL" - - Args: - input_dir: 输入目录 - - Returns: - 基因家族名称（大写） - """ - # 优先从参考文件提取 - ref_files = list(input_dir.glob("Ref_combined.*.pep.fasta")) - if ref_files: - # Ref_combined.SPL.pep.fasta -> SPL - parts = ref_files[0].stem.split('.') - if len(parts) >= 2: - # Karpathy: 保持原始大小写，不强制转大写 - gene_family = parts[1] - print(f"🧬 从参考文件提取基因家族: {gene_family}") - return gene_family - - # 从第一个目标文件提取 - target_files = [f for f in input_dir.glob("*.pep.fasta") - if not f.name.startswith("Ref_combined")] - if target_files: - parts = target_files[0].stem.split('.') - if len(parts) >= 2: - # Karpathy: 保持原始大小写，不强制转大写 - gene_family = parts[1] - print(f"🧬 从目标文件提取基因家族: {gene_family}") - return gene_family - - raise RuntimeError("无法从文件名提取基因家族名称") - - -def write_fasta_ids(fasta_file: Path, output_file: Path) -> int: - """Write one sequence ID per FASTA record using the first header token.""" - output_file.parent.mkdir(parents=True, exist_ok=True) - count = 0 - with open(fasta_file, 'r') as infile, open(output_file, 'w') as outfile: - for line in infile: - if line.startswith('>'): - seq_id = line[1:].strip().split()[0] - if seq_id: - outfile.write(seq_id + '\n') - count += 1 - print(f"ID list written: {output_file} ({count} IDs)") - return count - - -def merge_final_members( - final_member_files: List[Path], - output_file: Path -) -> None: - """ - 合并所有目标物种的 final_gfmembers 文件 - - Args: - final_member_files: 各物种 final_gfmembers 文件列表 - output_file: 合并后的输出文件 - """ - output_file.parent.mkdir(parents=True, exist_ok=True) - - with open(output_file, 'w') as outfile: - for member_file in final_member_files: - if member_file.exists(): - with open(member_file, 'r') as infile: - content = infile.read() - outfile.write(content) - # 确保文件末尾有换行 - if not content.endswith('\n'): - outfile.write('\n') - - print(f"✅ 合并完成: {output_file} ({len(final_member_files)} 个文件)") - - - - -# def find_reference_file(input_dir: Path) -> Optional[Path]: -# """ -# 在 input_dir 中查找参考序列文件 -# 用于blast分析 -# """ -# print(f"🔍 查找参考序列文件在: {input_dir}") - -# patterns = ["*.fasta", "*.fa", "*.faa"] - -# for pattern in patterns: -# for file in input_dir.rglob(pattern): -# if file.is_file(): -# # 排除蛋白质序列文件 -# filename = file.name.lower() -# if not ("pep" in filename or "protein" in filename): -# print(f"✅ 找到参考序列文件: {file}") -# return file - -# print("⚠️ 未找到参考序列文件") -# return None -def find_reference_file(input_dir: Path, prefix: str, gene_family: str = None) -> Union[Path, List[Path], None]: - """ - 在 input_dir 中查找参考序列文件（排除用户数据文件） - 支持命名规范: At.pep.fasta (全蛋白), At_NAC.pep.fasta (基因家族蛋白) - - 修订 2026-04-26: 新增对 Ref_combined.*.pep.fasta 的支持 - 优先使用 genome_prepare 生成的参考物种合并文件 - - 对于identify阶段: - - 优先寻找基因家族特定的参考序列 (如 At_NAC.pep.fasta) - - 找到多个时返回列表供用户选择 - - 未找到时返回None并提示用户上传 - - Args: - input_dir: 输入目录 - prefix: 分析前缀，如 "Ft.SPL" 或 "Ft" - gene_family: 基因家族名称，如 "NAC" - """ - print(f"🔍 查找参考序列文件（排除用户文件）在: {input_dir}") - - # 修订 2026-04-26: 优先检查 Ref_combined.*.pep.fasta - ref_combined_files = list(input_dir.glob("Ref_combined.*.pep.fasta")) - if ref_combined_files: - print(f"✅ 找到参考物种合并文件: {ref_combined_files[0].name}") - return ref_combined_files[0] - - if gene_family: - print(f"🧬 目标基因家族: {gene_family}") - - # 从prefix中提取物种前缀和基因家族 - species_prefix = prefix.split('.')[0] if '.' in prefix else prefix - if not gene_family and '.' in prefix: - gene_family = prefix.split('.')[1] if len(prefix.split('.')) > 1 else None - - patterns = ["*.pep.fasta", "*.pep.fa", "*.protein.fasta", "*.protein.fa","*.fasta", "*.fa", "*.faa"] - - # 分类存储 - # 使用 set 去重，避免同一文件被多个 pattern 重复匹配 - family_specific_refs = set() # 基因家族特定的参考序列 (如 At_NAC.pep.fasta) - full_protein_refs = set() # 全蛋白参考序列 (如 At.pep.fasta) - other_refs = set() # 其他参考序列 - - # 参考物种关键词映射 - ref_species_keywords = ["ath", "at.", "osa", "os.", "gma", "gm.", "bra", "br.", "aly", "al.", "tig", "tg.", "sly", "sl."] - - for pattern in patterns: - for file in input_dir.rglob(pattern): - if file.is_file(): - filename = file.name.lower() - original_filename = file.name - - # 排除用户自己的蛋白质序列文件 (如 Zm.pep.fasta, Zm_NAC.pep.fasta) - if filename.startswith(species_prefix.lower() + ".") or filename.startswith(species_prefix.lower() + "_"): - print(f" 排除用户文件: {original_filename}") - continue - - # 排除CDS文件 - if filename.endswith(".cds.fasta") or ".cds." in filename: - print(f" 排除CDS文件: {original_filename}") - continue - - # 判断是否为基因家族特定的参考序列 (如 At_NAC.pep.fasta) - # 格式: 物种_基因家族.pep.fasta 或 物种基因家族.pep.fasta - is_family_specific = False - if gene_family: - # 检查文件名是否包含基因家族名 - gene_family_lower = gene_family.lower() - # 匹配模式: At_NAC, AtNAC, ATH_NAC 等 - if f"_{gene_family_lower}." in filename or f"_{gene_family_lower}_" in filename: - is_family_specific = True - elif gene_family_lower in filename.replace("_", "").replace(".", ""): - # 更宽松的匹配: AtNAC, athnac 等 - is_family_specific = True - - # 判断是否为全蛋白参考序列 (如 At.pep.fasta) - is_full_protein = False - if any(keyword in filename for keyword in ref_species_keywords): - # 检查是否为纯物种名+扩展名 (如 At.pep.fasta, not At_NAC.pep.fasta) - base_name = filename.replace(".pep.fasta", "").replace(".pep.fa", "").replace(".protein.fasta", "").replace(".protein.fa", "").replace(".fa", "").replace(".fasta", "") - if "_" not in base_name and gene_family and gene_family.lower() not in base_name: - is_full_protein = True - - if is_family_specific: - print(f" ✅ 找到基因家族特定参考: {original_filename}") - family_specific_refs.add(file) - elif is_full_protein: - print(f" 📄 找到全蛋白参考: {original_filename}") - full_protein_refs.add(file) - else: - other_refs.add(file) - - # 对于identify阶段: 优先使用基因家族特定的参考序列 - # 将 set 转换为 list 以便索引访问 - family_list = sorted(list(family_specific_refs), key=lambda x: x.name) - - if family_list: - print(f"\n🎯 找到 {len(family_list)} 个基因家族特定参考序列:") - for i, ref in enumerate(family_list, 1): - print(f" {i}. {ref.name}") - - # 如果只有一个，直接返回 - if len(family_list) == 1: - print(f"✅ 使用唯一的基因家族特定参考: {family_list[0].name}") - return family_list[0] - else: - # 多个时返回列表（调用方需要处理选择） - print(f"⚠️ 找到多个基因家族特定参考序列，需要用户选择") - return family_list # 返回列表 - - # 没有找到基因家族特定的参考序列 - full_list = sorted(list(full_protein_refs), key=lambda x: x.name) - if full_list: - print(f"\n⚠️ 警告: 未找到基因家族特定参考序列 (如 At_{gene_family}.pep.fasta)") - print(f" 但找到了全蛋白参考序列:") - for ref in full_list: - print(f" - {ref.name}") - print(f"\n❌ 对于identify阶段，需要使用基因家族特定的参考序列") - return None # 返回None，提示用户上传 - - print("⚠️ 未找到任何参考序列文件") - return None - -def hmmfetch_models(pfam_db_unz: Path, pfams: str, out_hmm: Path): - """ - 用 hmmfetch 从 pfam_db_unz 提取 pfams 到 out_hmm. - 如果 hmmfetch 失败（缺少SSI索引），回退到 python_fetch_models. - """ - print(f"Fetching {pfams} from {pfam_db_unz}") - cmd = [str(hmmfetch_bin()), '-o', str(out_hmm), str(pfam_db_unz), pfams] - rc, out, err = run_cmd(cmd) - if rc != 0: - print(f"hmmfetch failed, trying Python fallback...") - python_fetch_models(pfam_db_unz, pfams, out_hmm) - - -def python_fetch_models(pfam_db: Path, pfams: str, out_hmm: Path): - """ - 手动从 Pfam HMM 文件中提取指定的模型。 - """ - print(f"Python fetching {pfams} from {pfam_db}") - pfam_list = [p.strip() for p in pfams.split(',')] - with open(pfam_db, 'r', encoding='latin-1') as fin: - lines = fin.readlines() - - inside = False - current_name = None - out_lines = [] - found = set() - - i = 0 - while i < len(lines): - line = lines[i] - if line.startswith('HMMER3/'): - # 开始一个新模型 - inside = True - model_lines = [line] - i += 1 - # 读取直到下一个 HMMER3/ 或文件结束 - while i < len(lines) and not lines[i].startswith('HMMER3/'): - model_lines.append(lines[i]) - if line.startswith('NAME '): - current_name = line.split()[1] - i += 1 - # 检查这个模型是否在所需列表中 - if current_name in pfam_list: - out_lines.extend(model_lines) - found.add(current_name) - print(f" ✓ {current_name}") - # i 已经指向下一个 HMMER3/ 或文件末尾，继续外层循环 - continue - i += 1 - - missing = set(pfam_list) - found - if missing: - raise RuntimeError(f"Python fetch models failed, missing: {', '.join(missing)} from {pfam_db}") - - out_hmm.parent.mkdir(parents=True, exist_ok=True) - with open(out_hmm, 'w', encoding='latin-1') as fout: - fout.writelines(out_lines) - print(f"Saved {len(found)} models to {out_hmm}") - - -def hmmsearch_bin() -> Path: - """返回 hmmsearch 二进制路径""" - # 优先使用系统/conda安装的 - for p in [ - 'hmmsearch', - '/home/xia-lab/miniconda3/envs/test4py311/bin/hmmsearch', - '/home/userlab/home/xiazq/00.bin/miniconda3/bin/hmmsearch' - ]: - if subprocess.run(['which', p], capture_output=True).returncode == 0: - return Path(p) - raise RuntimeError("hmmsearch not found in PATH") - - -def hmmfetch_bin() -> Path: - """返回 hmmfetch 二进制路径""" - for p in [ - 'hmmfetch', - '/home/xia-lab/miniconda3/envs/test4py311/bin/hmmfetch', - '/home/userlab/home/xiazq/00.bin/miniconda3/bin/hmmfetch' - ]: - if subprocess.run(['which', p], capture_output=True).returncode == 0: - return Path(p) - raise RuntimeError("hmmfetch not found in PATH") - - -def read_pep_ids(pep_fasta: Path) -> List[str]: - """读取蛋白质fasta中的所有ID""" - ids = [] - with open(pep_fasta, 'r') as f: - for line in f: - if line.startswith('>'): - # 去除 > 和可能的注释 - seq_id = line[1:].split()[0].strip() - ids.append(seq_id) - return ids - - -def read_ids_from_file(ids_file: Path) -> Set[str]: - """读取ID文件，返回集合""" - ids = set() - if ids_file and ids_file.exists(): - with open(ids_file, 'r') as f: - for line in f: - id_str = line.strip() - if id_str: - ids.add(id_str) - return ids - - -def parse_domtblout(domtblout: Path, evalue_max: float, cov_min: float, required_pfams: List[str]) -> Tuple[dict, dict]: - """ - 解析 hmmsearch 的 domtblout 文件。 - 返回: - - per_seq: {seq_id: {pfam: [(evalue, cov)]}} - - per_seq_best: {seq_id: {pfam: (evalue, cov)}} - """ - per_seq = {} - per_seq_best = {} - - # FIX: 防御性处理 - 如果 required_pfams 为 None，设为空集合（不过滤） - required_set = set(required_pfams) if required_pfams is not None else set() - - with open(domtblout, 'r') as f: - for line in f: - if line.startswith('#'): - continue - parts = line.strip().split() - if len(parts) < 23: - continue - target = parts[0] # 序列ID - query = parts[3] # HMM名称 (Pfam ID) - if required_set and query not in required_set: - continue # 只关注我们需要的Pfam（如果设置了过滤集合） - - evalue = float(parts[12]) # domain i-Evalue - # 计算覆盖度 - alen = int(parts[2]) # 序列长度 - hmm_from = int(parts[15]) - hmm_to = int(parts[16]) - hmm_cov = (hmm_to - hmm_from + 1) / alen if alen > 0 else 0 - - # 初步过滤 - if evalue > evalue_max or hmm_cov < cov_min: - continue - - # 记录到 per_seq - if target not in per_seq: - per_seq[target] = {} - per_seq_best[target] = {} - if query not in per_seq[target]: - per_seq[target][query] = [] - per_seq_best[target][query] = (evalue, hmm_cov) - - per_seq[target][query].append((evalue, hmm_cov)) - # 更新最佳值（E-value最小） - if evalue < per_seq_best[target][query][0]: - per_seq_best[target][query] = (evalue, hmm_cov) - - return per_seq, per_seq_best - - - -# ======== 从这里开始插入 ======== -def compute_file_hash(filepath: Path) -> str: - """计算文件的 SHA256 哈希值""" - sha256_hash = hashlib.sha256() - try: - with open(filepath, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - except Exception as e: - return f"error: {e}" -# ======== 插入结束 ======== - -# 添加工具版本打印信息 -def get_tool_versions() -> dict: - """获取关键工具版本信息""" - versions = {} - - # 1. HMMER - try: - # result = subprocess.run(["hmmsearch", "-h"], shell=Ture, apture_output=True, text=True, timeout=5) - # for line in result.stderr.split('\n'): - # if "HMMER" in line: - # versions["hmmer"] = line.strip() - # break - # except: - # versions["hmmer"] = "unknown" - # 先检查命令是否存在 - import shutil - if not shutil.which("hmmsearch"): - versions["hmmer"] = "command not found in PATH" - else: - # 使用更全面的参数尝试 - for arg in ["-h", "--help", "-version", "--version"]: - try: - result = subprocess.run( - ["hmmsearch", arg], - capture_output=True, - text=True, - timeout=5, - encoding='utf-8', - errors='ignore' - ) - - # 合并所有输出 - all_output = result.stdout + result.stderr - - # 使用正则表达式搜索版本 - import re - version_match = re.search(r'HMMER\s+(\d+\.\d+\.?\d*)', all_output) - if version_match: - versions["hmmer"] = f"HMMER {version_match.group(1)}" - break - - # 或者搜索包含 HMMER 的行 - for line in all_output.split('\n'): - if "HMMER" in line.upper(): - versions["hmmer"] = line.strip() - break - if "hmmer" in versions: - break - - except Exception: - continue - - if "hmmer" not in versions: - versions["hmmer"] = "version detection failed" - - except Exception as e: - versions["hmmer"] = f"error: {str(e)[:50]}" - # 2. BLAST - try: - result = subprocess.run(["blastp", "-version"], capture_output=True, text=True, timeout=5) - for line in result.stdout.split('\n'): - if "blastp" in line: - versions["blast"] = line.strip() - break - except: - versions["blast"] = "unknown" - - # 3. Python 版本 - versions["python"] = sys.version.split()[0] - - return versions - - -def select_final_members( - pep_ids: List[str], - per_seq: dict, - per_seq_best: dict, - required_pfams: List[str], - evalue_max: float, - cov_min: float -) -> Tuple[List[str], dict]: - """ - 选择最终的成员。 - 严格模式：必须包含所有 required_pfams。 - 返回 (final_ids, drop_reasons) - """ - final = [] - drop_reasons = {} - - required_set = set(required_pfams) - - for seq_id in pep_ids: - if seq_id not in per_seq: - drop_reasons[seq_id] = 'no_hit' - continue - - # 检查是否包含所有必需的 Pfam - hits = set(per_seq[seq_id].keys()) - missing = required_set - hits - if missing: - drop_reasons[seq_id] = f'missing_required' - continue - - # 检查每个Pfam是否通过阈值 - passed_all = True - for pfam in required_set: - best_e, best_cov = per_seq_best[seq_id][pfam] - if best_e > evalue_max or best_cov < cov_min: - passed_all = False - drop_reasons[seq_id] = f'failed_qc_{pfam}' - break - - if passed_all: - final.append(seq_id) - else: - # 如果已经在上面记录了原因，这里不再记录 - if seq_id not in drop_reasons: - drop_reasons[seq_id] = 'failed_qc' - - return final, drop_reasons - - -def write_fasta_subset(source_fasta: Path, ids: List[str], out_fasta: Path): - """从source_fasta中提取指定的IDs写入out_fasta""" - id_set = set(ids) - writing = False - current_id = None - out_lines = [] - - with open(source_fasta, 'r') as f: - for line in f: - if line.startswith('>'): - # 提取ID - header = line[1:].strip() - seq_id = header.split()[0] - if seq_id in id_set: - writing = True - current_id = seq_id - out_lines.append(line) - else: - writing = False - current_id = None - elif writing: - out_lines.append(line) - - out_fasta.parent.mkdir(parents=True, exist_ok=True) - with open(out_fasta, 'w') as f: - f.writelines(out_lines) - print(f"Saved {len(ids)} sequences to {out_fasta}") - - -def write_report( - out_report: Path, - prefix: str, - required_pfams: List[str], - evalue_max: float, - cov_min: float, - cut_ga: bool, - cpu: int, - pfam_db: Path, - retention_days: int, - total_proteins: int, - candidates: int, - final_members: int, - drop_reasons: dict, - start_time: datetime, - # ====== 新增参数 ====== - tool_versions: dict = None, - cmd: List[str] = None, - inputs: List[dict] = None, - end_time: datetime = None, - elapsed_sec: float = None -): - """写入鉴定报告""" - - # 设置默认值 - tool_versions = tool_versions or {} - cmd = cmd or [] - inputs = inputs or [] - - lines = [ - f"# Identify Report: {prefix}", - "", - "## Parameters", - f"- required_pfams: {', '.join(required_pfams)}", - f"- dom_i_evalue_max: {evalue_max}", - f"- hmm_cov_min: {cov_min}", - f"- cut_ga: {cut_ga}", - f"- cpu: {cpu}", - f"- pfam_db: {pfam_db}", - f"- retention_days: {retention_days} (expires around {start_time + timedelta(days=retention_days)})", - "", - "## Summary", - f"- total input proteins: {total_proteins}", - f"- candidates (any required Pfam hit passing QC): {candidates}", - f"- strict final members (contain ALL required Pfams): {final_members}", - ] - - # 统计 drop 原因 - reason_counts = {} - for reason in drop_reasons.values(): - reason_counts[reason] = reason_counts.get(reason, 0) + 1 - - for reason, cnt in reason_counts.items(): - lines.append(f"- dropped {reason}: {cnt}") - # ====== 新增：Reproducibility Section ====== - lines.append("") - lines.append("## Reproducibility") - lines.append("") - - # 工具版本 - if tool_versions: - lines.append("### Tool Versions") - for tool, version in sorted(tool_versions.items()): - lines.append(f"- **{tool}**: `{version}`") - lines.append("") - - # 执行时间 - if elapsed_sec is not None: - lines.append(f"- **Elapsed time**: {elapsed_sec:.2f} seconds") - if start_time: - lines.append(f"- **Start time**: {start_time.isoformat()}") - if end_time: - lines.append(f"- **End time**: {end_time.isoformat()}") - lines.append("") - - # 执行命令 - if cmd: - lines.append("### Command") - lines.append("```bash") - # 美化命令：每行显示一部分，避免过长 - cmd_str = " ".join(cmd) - if len(cmd_str) > 80: - # 简单换行：每80个字符换行 - import textwrap - wrapped = textwrap.wrap(cmd_str, width=80) - lines.extend(wrapped) - else: - lines.append(cmd_str) - lines.append("```") - lines.append("") - - # 输入文件摘要 - if inputs: - lines.append("### Input Files") - lines.append("| File | Size (MB) | SHA256 (first 12) |") - lines.append("|------|-----------|-------------------|") - for inp in inputs: - name = inp.get("path", "unknown") - size_bytes = inp.get("size", 0) - size_mb = size_bytes / (1024 * 1024) if size_bytes > 0 else 0 - sha_full = inp.get("sha256", "") - sha_short = sha_full[:12] if sha_full else "N/A" - lines.append(f"| `{name}` | {size_mb:.2f} | `{sha_short}` |") - lines.append("") - # ====== 新增部分结束 ====== - - - lines.append("") - lines.append("## Dropped IDs (auto explanation)") - lines.append("") - - # 列出前20个被drop的ID - dropped_items = list(drop_reasons.items()) - for seq_id, reason in dropped_items[:20]: - lines.append(f"- {seq_id:<25} REASON={reason}") - - if len(dropped_items) > 20: - lines.append(f"- ... and {len(dropped_items) - 20} more") - - lines.append("") - lines.append(f"Report generated at {datetime.now().isoformat()}") - - out_report.parent.mkdir(parents=True, exist_ok=True) - with open(out_report, 'w', encoding='utf-8') as f: - f.write('\n'.join(lines)) - - print(f"Report: {out_report}") - - -def run_hmm_analysis( - input_dir: Path, - output_dir: Path, - prefix: str, - hmm_file: Path, - pep_file: Path, - cpu: int = 8, - evalue: float = 1e-5 # 默认E值阈值 -) -> Tuple[Path, Path]: - """ - 运行HMM分析（使用原始蛋白质序列文件） - 返回: (domtblout_file, ids_file) - """ - print("\n" + "=" * 60) - print("🔎 运行HMM分析（使用原始文件）") - print("=" * 60) - - # 创建分析目录 - analysis_dir = output_dir / "analysis" - analysis_dir.mkdir(parents=True, exist_ok=True) - - # 定义输出文件 - domtblout_file = analysis_dir / f"{prefix}.domtblout" - hmmout_file = analysis_dir / f"{prefix}.hmmout" - filtered_file = analysis_dir / f"{prefix}.domtblout.filter" - ids_file = analysis_dir / f"{prefix}.hmm.ids" - - # 1. 运行hmmsearch - print(f"\n🚀 执行hmmsearch命令:") - cmd = [ - str(hmmsearch_bin()), - '-E', str(evalue), # 使用传入的E值阈值 - '--domtblout', str(domtblout_file), - '-o', str(hmmout_file), - '--cpu', str(cpu), - str(hmm_file), - str(pep_file) - ] - print(f" {' '.join(cmd)}") - - rc, out, err = run_cmd(cmd) - if rc != 0: - raise RuntimeError(f"hmmsearch失败: {err}") - - print(f"✅ hmmsearch完成: {domtblout_file}") - - # 2. 过滤结果 (E-value < 0.01) - print(f"\n🚀 过滤结果 (E-value < 0.01):") - awk_cmd = f"awk '$7<0.01 && $1 !~ /^#/ {{print $0 }}' {domtblout_file} > {filtered_file}" - print(f" {awk_cmd}") - - rc, out, err = run_cmd(awk_cmd, shell=True) - if rc != 0: - raise RuntimeError(f"过滤失败: {err}") - - print(f"✅ 过滤完成: {filtered_file}") - - # 3. 提取基因ID - print(f"\n🚀 提取基因ID:") - id_cmd = f"awk '{{print $1}}' {filtered_file} | sort -u > {ids_file}" - print(f" {id_cmd}") - - rc, out, err = run_cmd(id_cmd, shell=True) - if rc != 0: - raise RuntimeError(f"提取ID失败: {err}") - - print(f"✅ 提取ID完成: {ids_file}") - - # 显示结果 - with open(ids_file, 'r') as f: - gene_ids = [line.strip() for line in f if line.strip()] - - print(f"\n📊 HMM分析结果:") - print(f" 鉴定到的基因数: {len(gene_ids)}") - - if gene_ids: - print(f"\n🔝 前10个基因:") - for i, gene_id in enumerate(gene_ids[:10]): - print(f" {i+1:2d}. {gene_id}") - if len(gene_ids) > 10: - print(f" ... 还有 {len(gene_ids)-10} 个基因") - - return domtblout_file, ids_file - - -def run_blast_analysis( - input_dir: Path, - output_dir: Path, - prefix: str, - ref_file: Path, - pep_file: Path, - cpu: int = 8, - evalue: float = 1e-7, -) -> Optional[Path]: - """ - 运行Blast分析（使用原始蛋白质序列文件） - 返回: ids_file (如果参考序列存在) - """ - print("\n" + "=" * 60) - print("🧬 运行Blast分析（使用原始文件）") - print("=" * 60) - - if not ref_file or not ref_file.exists(): - print("⚠️ 未提供参考序列文件，跳过Blast分析") - return None - - # 创建分析目录 - analysis_dir = output_dir / "analysis" - analysis_dir.mkdir(parents=True, exist_ok=True) - - # 定义输出文件 - db_prefix = analysis_dir / "blast_db" - blast_output = analysis_dir / f"{prefix}.blast.out" - ids_file = analysis_dir / f"{prefix}.blast.ids" - - # 1. 创建Blast数据库 - print(f"\n🚀 创建Blast数据库:") - cmd = ['makeblastdb', '-in', str(ref_file), '-dbtype', 'prot', '-out', str(db_prefix)] - print(f" {' '.join(cmd)}") - - rc, out, err = run_cmd(cmd) - if rc != 0: - print(f"⚠️ 创建数据库失败: {err}") - return None - - print(f"✅ 数据库创建完成") - - # 2. 运行blastp - print(f"\n🚀 运行blastp:") - cmd = [ - 'blastp', - '-query', str(pep_file), - '-db', str(db_prefix), - '-out', str(blast_output), - '-evalue', str(evalue), - '-outfmt', '6 std qlen slen', - '-num_threads', str(cpu) - ] - print(f" {' '.join(cmd)}") - - rc, out, err = run_cmd(cmd) - if rc != 0: - print(f"⚠️ blastp失败: {err}") - return None - - print(f"✅ blastp完成: {blast_output}") - - # 3. 过滤结果 (相似度 > 30%) - print(f"\n🚀 过滤结果 (相似度 > 30%):") - awk_cmd = f"awk ' $3 > 30 {{print $1}} ' {blast_output} | sort -u > {ids_file}" - print(f" {awk_cmd}") - - rc, out, err = run_cmd(awk_cmd, shell=True) - if rc != 0: - print(f"⚠️ 过滤失败: {err}") - return None - - print(f"✅ 过滤完成: {ids_file}") - - # 显示结果 - with open(ids_file, 'r') as f: - gene_ids = [line.strip() for line in f if line.strip()] - - print(f"\n📊 Blast分析结果:") - print(f" 鉴定到的基因数: {len(gene_ids)}") - - if gene_ids: - print(f"\n🔝 前10个基因:") - for i, gene_id in enumerate(gene_ids[:10]): - print(f" {i+1:2d}. {gene_id}") - if len(gene_ids) > 10: - print(f" ... 还有 {len(gene_ids)-10} 个基因") - - return ids_file - - -def combine_hmm_blast_results( - hmm_ids_file: Path, - blast_ids_file: Optional[Path], - output_dir: Path, - prefix: str, - merge_mode: str = 'intersection' # 新增参数，默认交集 -) -> Path: - """ - 整合HMM和Blast结果 - 参数: - merge_mode: 'intersection'（交集）或 'union'（并集），默认交集 - 返回: combined_ids_file - """ - print("\n" + "=" * 60) - print(f"🔄 整合HMM和Blast结果 ({merge_mode}模式)") - print("=" * 60) - - # 读取HMM结果 - hmm_ids = read_ids_from_file(hmm_ids_file) - - # 读取Blast结果（如果有） - blast_ids = read_ids_from_file(blast_ids_file) if blast_ids_file else set() - - # 整合结果（根据merge_mode） - if blast_ids: - if merge_mode == 'intersection': - combined_ids = hmm_ids.intersection(blast_ids) - print(f" 合并方式: 交集 (HMM ∩ BLAST)") - else: - combined_ids = hmm_ids.union(blast_ids) - print(f" 合并方式: 并集 (HMM ∪ BLAST)") - else: - combined_ids = hmm_ids - print(f" 无BLAST结果，仅使用HMM结果") - - # 保存整合结果 - combine_dir = output_dir / "combine" - combine_dir.mkdir(parents=True, exist_ok=True) - combined_file = combine_dir / f"{prefix}.combined.ids" - - with open(combined_file, 'w') as f: - for gene_id in sorted(combined_ids): - f.write(f"{gene_id}\n") - - print(f"\n📊 整合结果:") - print(f" HMM鉴定基因数: {len(hmm_ids)}") - if blast_ids_file: - print(f" Blast鉴定基因数: {len(blast_ids)}") - print(f" 整合后基因数: {len(combined_ids)}") - print(f" 结果文件: {combined_file}") - - if combined_ids: - print(f"\n🔝 前10个基因:") - for i, gene_id in enumerate(sorted(combined_ids)[:10]): - print(f" {i+1:2d}. {gene_id}") - if len(combined_ids) > 10: - print(f" ... 还有 {len(combined_ids)-10} 个基因") - - return combined_file - - -# ---------------------------------------------------------------------- -# Main - 修订版 -# ---------------------------------------------------------------------- - - -def main(): - print(f"🐛 DEBUG: Python {sys.version}") - print(f"🐛 DEBUG: 脚本路径: {Path(__file__).resolve()}") - print(f"🐛 DEBUG: 当前目录: {Path.cwd()}") - - try: - test_versions = get_tool_versions() - print(f"🛠️ DEBUG: 工具版本测试: {test_versions}") - except Exception as e: - print(f"❌ DEBUG: 获取工具版本失败: {e}") - - script_start_time = datetime.now() - - parser = argparse.ArgumentParser(description='Identify stage for web pipeline - 多物种修订版') - parser.add_argument('--input-dir', required=True, help='input// (用户上传文件目录)') - parser.add_argument('--workspace', required=True, help='workspace//') - parser.add_argument('--prefix', required=True, help='分析前缀，如 Ft.SPL') - parser.add_argument('--cpu', type=int, default=8) - parser.add_argument('--force', action='store_true', help='overwrite existing results') - parser.add_argument('--required-pfams', help='comma-separated Pfam accessions, e.g. PF03110,PFxxxxx') - parser.add_argument('--dom-i-evalue-max', type=float, default=1e-3, help='domain i-Evalue threshold') - parser.add_argument('--hmm-cov-min', type=float, default=0.3, help='HMM coverage threshold') - parser.add_argument('--blast-evalue', type=float, default=1e-7, help='Blast E-value threshold') - parser.add_argument('--cut-ga', action='store_true', help='use Pfam GA thresholds') - parser.add_argument('--pfam-dir', help='directory containing Pfam-A.hmm (default: env BBA_PFAM_DIR)') - parser.add_argument('--retention-days', type=int, default=7, help='days to keep workspace') - parser.add_argument('--ref-root', help='reference root directory') - parser.add_argument('--ref-species', help='comma-separated reference species names; can also use env BBA_REF_SPECIES') - parser.add_argument('--ref-file', help='指定参考序列文件（用于多个参考序列时选择，如 At_NAC_pep.fasta）') - parser.add_argument('--log-file', help='日志文件路径（从pipeline_backend传入）') - parser.add_argument('--hmm-combine-mode', choices=['intersection', 'union'], default='union', - help='多HMM文件时的整合方式（交集intersection/并集union），默认并集') - parser.add_argument('--hmm-blast-merge-mode', choices=['intersection', 'union'], default=None, - help='HMM和BLAST结果整合方式（交集intersection/并集union）') - parser.add_argument('--merge-mode', choices=['intersection', 'union'], default='intersection', - help='HMM和BLAST结果整合方式（默认交集intersection）') - parser.add_argument('--skip-ref-warning', action='store_true', - help='CLI模式：跳过无参考序列的提醒，直接继续') - parser.add_argument('--countdown-action', choices=['continue', 'cancel'], default='continue', - help='前端倒计时结束后发送的动作') - - args = parser.parse_args() - - in_dir = Path(args.input_dir).resolve() - ws_root = Path(args.workspace).resolve() - prefix = args.prefix - - tool_versions = get_tool_versions() - print(f"🔧 工具版本: {tool_versions}") - - inputs_info = [] - for file_path in in_dir.iterdir(): - if file_path.is_file(): - inputs_info.append({ - "path": str(file_path.relative_to(in_dir)), - "full_path": str(file_path.resolve()), - "sha256": compute_file_hash(file_path), - "size": file_path.stat().st_size, - "mtime": file_path.stat().st_mtime - }) - - if not in_dir.exists(): - raise RuntimeError(f"Input directory not found: {in_dir}") - - log_file_path = None - if args.log_file: - log_file_path = Path(args.log_file).resolve() - print(f"📝 日志文件: {log_file_path}") - - print(f"🌱 Identify stage for {prefix} (多物种修订版)") - print(f" input目录: {in_dir}") - print(f" workspace目录: {ws_root}") - - # 多物种支持 - gene_family = extract_gene_family(in_dir) - print(f"🧬 基因家族: {gene_family}") - - ref_file = find_reference_file_simple(in_dir) - if not ref_file: - gene_family_old = prefix.split('.')[1] if '.' in prefix else None - ref_result = find_reference_file(in_dir, prefix, gene_family_old) - if isinstance(ref_result, list): - ref_file = ref_result[0] - else: - ref_file = ref_result - - if not ref_file: - raise RuntimeError("未找到参考物种文件") - print(f"📚 参考物种: {ref_file.name}") - - target_pep_files = find_pep_files(in_dir) - if not target_pep_files: - raise RuntimeError("未找到目标物种 pep 文件") - print(f"🎯 目标物种数: {len(target_pep_files)}") - for f in target_pep_files: - print(f" - {f.name}") - - print("\n🎯 Pfam获取策略:") - - use_user_hmm = False - query_hmm = None - required_pfams = None - hmm_combine_mode = None - - user_hmms = find_user_query_hmms(in_dir) - - if not user_hmms: - error_data = { - "error_type": "NO_HMM_FILE_ERROR", - "stage": "identify", - "action": "terminate", - "message": "必须上传至少一个HMM文件才能运行Identify阶段" - } - raise RuntimeError(json.dumps(error_data)) - - if False: # 禁用多HMM选择弹窗，改为单HMM模式 - print(f"⚠️ 发现 {len(user_hmms)} 个HMM文件:") - for i, hmm in enumerate(user_hmms, 1): - print(f" {i}. {hmm.name}") - - if args.hmm_combine_mode: - hmm_combine_mode = args.hmm_combine_mode - use_user_hmm = True - print(f"✅ 使用HMM整合模式: {hmm_combine_mode}") - else: - pause_manifest = { - "manifest_version": "1.0", - "job_id": prefix, - "stage": "identify", - "status": "paused", - "pause_reason": "MULTIPLE_HMM_SELECTION", - "timestamp": datetime.now().isoformat(), - "message": f"发现{len(user_hmms)}个HMM文件，需要用户选择整合方式", - "requires_user_input": True, - "input_type": "selection", - "hmm_files": [{"name": h.name, "size": f"{h.stat().st_size//1024}KB"} for h in user_hmms], - "options": [ - {"label": "交集（严格筛选）", "value": "intersection", "description": "序列必须同时匹配所有HMM", "recommended": True}, - {"label": "并集（宽松筛选）", "value": "union", "description": "序列匹配任意HMM即可", "recommended": False} - ], - "selection_param": "--hmm-combine-mode", - "prefix": effective_prefix, - "workspace": str(ws_root), - "input_dir": str(in_dir) - } - - final_dir_temp = ws_root / "final" - final_dir_temp.mkdir(parents=True, exist_ok=True) - manifest_file = final_dir_temp / f"{prefix}.identify.manifest.json" - - with open(manifest_file, 'w', encoding='utf-8') as f: - json.dump(pause_manifest, f, indent=2, ensure_ascii=False) - - print(f"\n⏸️ 任务暂停：发现多个HMM文件，需要用户选择整合方式") - print(f" Manifest已写入: {manifest_file}") - sys.exit(0) - - # 修订 2026-04-26: 改为单HMM模式，只使用第一个HMM文件 - if len(user_hmms) >= 1: - print(f"✅ 使用用户上传的HMM文件: {user_hmms[0].name}") - use_user_hmm = True - query_hmm = user_hmms[0] - hmm_name = user_hmms[0].stem - required_pfams = [hmm_name] - - if len(user_hmms) > 1: - print(f" 注意: 发现 {len(user_hmms)} 个HMM文件，只使用第一个") - for i, hmm in enumerate(user_hmms[1:], 2): - print(f" {i}. {hmm.name} (跳过)") - print(f" HMM名称: {hmm_name}") - - # Karpathy: 设置 HMM/BLAST 合并模式默认值 - # 用户未指定时，默认使用并集（union/宽松筛选） - if not args.hmm_blast_merge_mode: - args.hmm_blast_merge_mode = "union" - print(f"\n🔄 未指定HMM/BLAST合并模式，默认使用: {args.hmm_blast_merge_mode} (并集/宽松筛选)") - else: - print(f"\n🔄 使用HMM/BLAST合并模式: {args.hmm_blast_merge_mode}") - - final_member_files = [] - species_results = [] - - for pep_file in target_pep_files: - file_prefix = pep_file.stem.replace(".pep", "") - species_code = file_prefix.split('.')[0] - - print(f"\n{'='*60}") - print(f"🧬 处理目标物种: {file_prefix}") - print(f"{'='*60}") - - # 修订 2026-04-26: 输出目录改为 identify/{species}/ 结构 - species_workspace = ws_root / "identify" / file_prefix - species_workspace.mkdir(parents=True, exist_ok=True) - - paths = prepare_directories(species_workspace) - - start_time = datetime.now() - - # 修订 2026-04-28: 多HMM支持 - 每个HMM分别运行，根据参数取交集或并集 - hmm_ids_files = [] - - for i, hmm_file in enumerate(user_hmms, 1): - hmm_name = hmm_file.stem - hmm_prefix = f"{file_prefix}.{hmm_name}" - - print(f"\n [{i}/{len(user_hmms)}] 运行HMM: {hmm_file.name}") - - hmm_domtblout, hmm_ids_file = run_hmm_analysis( - input_dir=in_dir, - output_dir=species_workspace, - prefix=hmm_prefix, - hmm_file=hmm_file, - pep_file=pep_file, - cpu=args.cpu, - evalue=args.dom_i_evalue_max - ) - - hmm_ids_files.append(hmm_ids_file) - - # 取所有HMM结果的交集或并集（根据参数） - if len(hmm_ids_files) > 1: - # 读取所有HMM的IDs - hmm_id_sets = [] - for ids_file in hmm_ids_files: - with open(ids_file, 'r') as f: - ids = set(line.strip() for line in f if line.strip()) - hmm_id_sets.append(ids) - - # 根据参数决定取交集还是并集 - if args.hmm_combine_mode == 'intersection': - print(f"\n🔄 取 {len(hmm_ids_files)} 个HMM结果的交集:") - for i, ids in enumerate(hmm_id_sets): - print(f" HMM {i+1}: {len(ids)} 个基因") - final_hmm_ids = set.intersection(*hmm_id_sets) - print(f" 交集结果: {len(final_hmm_ids)} 个基因") - else: # union (默认) - print(f"\n🔄 取 {len(hmm_ids_files)} 个HMM结果的并集:") - for i, ids in enumerate(hmm_id_sets): - print(f" HMM {i+1}: {len(ids)} 个基因") - final_hmm_ids = set.union(*hmm_id_sets) - print(f" 并集结果: {len(final_hmm_ids)} 个基因") - - # 输出结果到 analysis/{species}.{gene_family}.hmm.ids - analysis_dir = species_workspace / "analysis" - hmm_ids_file = analysis_dir / f"{file_prefix}.hmm.ids" - with open(hmm_ids_file, 'w') as f: - for gene_id in sorted(final_hmm_ids): - f.write(f"{gene_id}\n") - - print(f" 输出: {hmm_ids_file.name}") - - blast_ids_file = run_blast_analysis( - input_dir=in_dir, - output_dir=species_workspace, - prefix=file_prefix, - ref_file=ref_file, - pep_file=pep_file, - cpu=args.cpu, - evalue=args.blast_evalue - ) - - # 修订 2026-04-27: 读取HMM和BLAST结果并合并 - with open(hmm_ids_file, 'r') as f: - hmm_ids = set(line.strip() for line in f if line.strip()) - with open(blast_ids_file, 'r') as f: - blast_ids = set(line.strip() for line in f if line.strip()) - - # 根据用户选择的模式合并结果 - if args.hmm_blast_merge_mode == "intersection": - final_ids = hmm_ids & blast_ids - print(f"\n🔄 使用交集模式: {len(final_ids)} 个基因") - else: # union - final_ids = hmm_ids | blast_ids - print(f"\n🔄 使用并集模式: {len(final_ids)} 个基因") - - # 写入合并后的结果文件 - combined_ids_file = species_workspace / "combine" / f"{file_prefix}.combined.ids" - combined_ids_file.parent.mkdir(parents=True, exist_ok=True) - with open(combined_ids_file, 'w') as f: - for gene_id in sorted(final_ids): - f.write(f"{gene_id}\n") - - final_dir = species_workspace / "final" - final_dir.mkdir(parents=True, exist_ok=True) - - final_ids_file = final_dir / f"{file_prefix}.final_gfmembers.id" - with open(final_ids_file, 'w') as f: - for gene_id in final_ids: - f.write(f"{gene_id}\n") - - final_pep_file = final_dir / f"{file_prefix}.final_gfmembers.pep.fa" - write_fasta_subset(pep_file, final_ids, final_pep_file) - - final_member_files.append(final_pep_file) - - species_results.append({ - "species": species_code, - "file_prefix": file_prefix, - "input_file": str(pep_file.name), - "final_members": len(final_ids), - "final_pep_file": str(final_pep_file.relative_to(ws_root)) - }) - - print(f"✅ {file_prefix}: {len(final_ids)} 个成员") - - final_dir = ws_root / "final" - final_dir.mkdir(parents=True, exist_ok=True) - - # Use a stable non-empty prefix for downstream stages when the frontend sends an empty prefix. - merged_prefix = f"Multi.{gene_family}" - effective_prefix = prefix or merged_prefix - merged_pep_file = final_dir / f"{effective_prefix}.final_members.pep.fa" - merge_final_members(final_member_files, merged_pep_file) - merged_id_file = final_dir / f"{effective_prefix}.final_members.id" - merged_id_count = write_fasta_ids(merged_pep_file, merged_id_file) - interproscan_tsv = final_dir / f"{effective_prefix}.final_members.interproscan.tsv" - interproscan_result = run_interproscan_domain_check(merged_pep_file, interproscan_tsv) - interproscan_status_file = final_dir / f"{effective_prefix}.interproscan_domain_check.txt" - write_interproscan_status_file(interproscan_status_file, interproscan_result) - - manifest_data = { - "manifest_version": "1.0", - "stage": "identify", - "prefix": effective_prefix, - "gene_family": gene_family, - "status": "success", - "timestamp": datetime.now().isoformat(), - "start_time": script_start_time.isoformat(), - "end_time": datetime.now().isoformat(), - "elapsed_sec": (datetime.now() - script_start_time).total_seconds(), - "target_species": [r["species"] for r in species_results], - "reference_species": [ref_file.stem.split('.')[0]] if ref_file else [], - "species_count": len(species_results), - "species_results": species_results, - "workspace_root": str(ws_root.resolve()), - "workspace": str(final_dir.resolve()), - "input_dir": str(in_dir.resolve()), - "tool_versions": tool_versions, - "cmd": sys.argv, - "inputs": inputs_info, - "platform": platform.platform(), - "python_version": sys.version.split()[0], - "domain_check": {"interproscan": interproscan_result}, - "warnings": [], - "artifacts": {} - } - - for r in species_results: - species = r["species"] - final_pep_path = ws_root / r["final_pep_file"] - manifest_data["artifacts"][f"{species}_final_members"] = { - "artifact": f"{species}_final_members", - "file": f"{r['file_prefix']}.final_gfmembers.pep.fa", - "full_path": str(final_pep_path.resolve()), - "relpath_or_hint": r["final_pep_file"], - "type": "fasta", - "species": species, - "members": r["final_members"] - } - - manifest_data["artifacts"]["final_members"] = { - "artifact": "final_members", - "file": merged_pep_file.name, - "full_path": str(merged_pep_file.resolve()), - "relpath_or_hint": f"final/{merged_pep_file.name}", - "type": "fasta", - "size": merged_pep_file.stat().st_size if merged_pep_file.exists() else 0, - "merged_from": [r["file_prefix"] for r in species_results] - } - manifest_data["artifacts"]["final_members_id"] = { - "artifact": "final_members_id", - "file": merged_id_file.name, - "full_path": str(merged_id_file.resolve()), - "relpath_or_hint": f"final/{merged_id_file.name}", - "type": "id_list", - "size": merged_id_file.stat().st_size if merged_id_file.exists() else 0, - "members": merged_id_count, - "derived_from": merged_pep_file.name - } - if interproscan_result.get("status") == "success": - manifest_data["artifacts"]["interproscan_domain_check"] = { - "artifact": "interproscan_domain_check", - "file": interproscan_tsv.name, - "full_path": str(interproscan_tsv.resolve()), - "relpath_or_hint": f"final/{interproscan_tsv.name}", - "type": "tsv", - "size": interproscan_tsv.stat().st_size if interproscan_tsv.exists() else 0, - "derived_from": merged_pep_file.name, - "description": "InterProScan TSV for domain-checking final identify members", - "summary": interproscan_result.get("summary", {}), - } - else: - manifest_data["warnings"].append({ - "step": "interproscan_domain_check", - "status": interproscan_result.get("status"), - "reason": interproscan_result.get("reason", "InterProScan did not produce a TSV") - }) - manifest_data["artifacts"]["interproscan_domain_check_status"] = { - "artifact": "interproscan_domain_check_status", - "file": interproscan_status_file.name, - "full_path": str(interproscan_status_file.resolve()), - "relpath_or_hint": f"final/{interproscan_status_file.name}", - "type": "txt", - "size": interproscan_status_file.stat().st_size if interproscan_status_file.exists() else 0, - "status": interproscan_result.get("status"), - "derived_from": merged_pep_file.name, - "description": "Readable InterProScan domain-check status for frontend result review", - } - clean_input_value = interproscan_result.get("clean_input") - clean_input = Path(clean_input_value) if clean_input_value else None - if clean_input and clean_input.exists(): - manifest_data["artifacts"]["interproscan_clean_input"] = { - "artifact": "interproscan_clean_input", - "file": clean_input.name, - "full_path": str(clean_input.resolve()), - "relpath_or_hint": f"final/{clean_input.name}", - "type": "fasta", - "size": clean_input.stat().st_size, - "derived_from": merged_pep_file.name, - "description": "Final member FASTA sanitized for InterProScan by removing '*' stop markers", - "sanitize": interproscan_result.get("sanitize", {}), - } - - # Write a non-hidden manifest name even when the submitted prefix is empty. - manifest_file = final_dir / f"{effective_prefix}.identify.manifest.json" - write_manifest(manifest_file, manifest_data) - - print(f"\n🎉 Identify 阶段完成!") - print(f" 处理物种数: {len(species_results)}") - print(f" 总成员数: {sum(r['final_members'] for r in species_results)}") - print(f" 合并文件: {merged_pep_file}") - print(f" Manifest: {manifest_file}") - - -if __name__ == "__main__": - main() diff --git a/bioflow_kernel/cli/run_interspecies_synteny.py.bak_blast_engine b/bioflow_kernel/cli/run_interspecies_synteny.py.bak_blast_engine deleted file mode 100755 index 17c7a0a..0000000 --- a/bioflow_kernel/cli/run_interspecies_synteny.py.bak_blast_engine +++ /dev/null @@ -1,1715 +0,0 @@ -#!/usr/bin/env python3 -""" -Interspecies Synteny阶段：物种间共线性分析（完整版） - -输入： - - 参考物种蛋白序列 (final/{prefix}.final_members.pep.fa) - - 参考物种GFF3 - - 目标物种蛋白序列 - - 目标物种GFF3 - -输出： - - final/{prefix}.interspecies_tree.nwk (物种间进化树) - - final/{prefix}.interspecies_tree.pdf/png (进化树图) - - final/{prefix}.interspecies_synteny.pdf/png (共线性图) - - final/{prefix}.interspecies_synteny_report.md (分析报告) - - final/{prefix}.interspecies_synteny.manifest.json (机器清单) -""" - -import os -import argparse -import json -import hashlib -import subprocess -import sys -import time -import shutil -import re -from datetime import datetime -from pathlib import Path -from typing import Dict, List, Any, Optional, Tuple - - -# ??conda??bin???PATH -CONDA_BIN = "/home/xia-lab/miniconda3/envs/test4py311/bin" -if CONDA_BIN not in os.environ.get("PATH", ""): - os.environ["PATH"] = CONDA_BIN + ":" + os.environ.get("PATH", "") -def pdf_to_png(pdf_path: Path, png_path: Path, dpi: int = 300) -> bool: - """将PDF转换为PNG""" - try: - # 方法1: 使用pdftoppm (poppler-utils) - cmd = ["pdftoppm", "-png", "-r", str(dpi), "-singlefile", - str(pdf_path), str(png_path.with_suffix(''))] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - if result.returncode == 0: - return True - except Exception: - pass - - try: - # 方法2: 使用convert (ImageMagick) - cmd = ["convert", "-density", str(dpi), str(pdf_path), "-quality", "100", str(png_path)] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - if result.returncode == 0: - return True - except Exception: - pass - - return False - - -def compute_file_hash(filepath: Path) -> str: - """计算文件的SHA256哈希值""" - sha256_hash = hashlib.sha256() - try: - with open(filepath, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - except Exception: - return "" - - -def get_tool_versions() -> Dict[str, str]: - """获取所需工具版本信息""" - versions = {} - - for tool, cmd in [ - ("mafft", ["mafft", "--version"]), - ("iqtree", ["iqtree", "--version"]), - ("blastp", ["blastp", "-version"]), - ("MCScanX", ["MCScanX", "-h"]), - ]: - try: - result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=5) - versions[tool] = "available" if result.returncode in [0, 1] else "not_found" - except Exception: - versions[tool] = "not_found" - - return versions - - -def prepare_directories(workspace: Path) -> Dict[str, Path]: - """创建目录结构""" - paths = { - "analysis_dir": workspace / "interspecies_synteny" / "analysis", - "final_dir": workspace / "final", - "logs_dir": workspace / "logs", - } - for name, path in paths.items(): - path.mkdir(parents=True, exist_ok=True) - return paths - - -def merge_protein_sequences( - target_protein: Path, - ref_protein: Path, - output_fasta: Path -) -> Dict[str, Any]: - """Merge target and reference family proteins for the interspecies tree.""" - print("Merging family protein sequences...") - - try: - with open(output_fasta, 'w') as outfile: - with open(target_protein, 'r') as f: - outfile.write(f.read().rstrip() + "\n") - with open(ref_protein, 'r') as f: - outfile.write(f.read().rstrip() + "\n") - - target_count = sum(1 for line in open(target_protein) if line.startswith('>')) - ref_count = sum(1 for line in open(ref_protein) if line.startswith('>')) - - print(f" merged target/ref family proteins: {target_count} + {ref_count} = {target_count + ref_count}") - - return { - "success": True, - "output_file": output_fasta, - "target_count": target_count, - "ref_count": ref_count - } - except Exception as e: - return {"success": False, "stderr": str(e)} - - -def build_phylogenetic_tree( - merged_fasta: Path, - output_dir: Path, - prefix: str, - cpu: int = 8 -) -> Dict[str, Any]: - """构建系统发育树 (MAFFT + IQ-TREE)""" - print(f"构建进化树...") - - # MAFFT比对 - msa_file = output_dir / f"{prefix}_interspecies.mafft.msa.fa" - mafft_cmd = ["mafft", "--auto", "--thread", str(cpu), str(merged_fasta)] - - try: - with open(msa_file, "w") as f_out: - result = subprocess.run(mafft_cmd, stdout=f_out, stderr=subprocess.PIPE, - text=True, check=False) - - if result.returncode != 0: - return {"success": False, "error": f"MAFFT失败: {result.stderr}"} - - # IQ-TREE建树 - base_name = output_dir / f"{prefix}_interspecies.iqtree" - iqtree_cmd = [ - "iqtree", "-s", str(msa_file), "-m", "MFP", "-bb", "1000", - "-nt", str(cpu), "-pre", str(base_name) - ] - - result = subprocess.run(iqtree_cmd, capture_output=True, text=True, check=False) - - # IQ-TREE 生成的 treefile 是 {prefix}.iqtree.treefile，不是 {prefix}.treefile - tree_file = Path(f"{base_name}.treefile") - - return { - "success": tree_file.exists(), - "tree_file": tree_file if tree_file.exists() else None, - "msa_file": msa_file - } - except Exception as e: - return {"success": False, "error": str(e)} - - -def filter_gff_files( - ref_gff3: Path, - target_gff3: Path, - output_dir: Path, - ref_species: str, - target_species: str -) -> Dict[str, Any]: - """过滤GFF3文件 - 提取基因ID（去掉转录本版本后缀如 .1, .2）""" - print(f"过滤GFF3文件...") - - import re - - results = {} - for gff3_file, species, prefix in [ - (ref_gff3, ref_species, "ref"), - (target_gff3, target_species, "target") - ]: - output_gff = output_dir / f"{species}.gff" - - try: - with open(gff3_file, 'r') as f: - lines = f.readlines() - - count = 0 - seen_genes = set() # 去重 - - with open(output_gff, 'w') as f: - for line in lines: - parts = line.strip().split('\t') - if len(parts) >= 9 and parts[2] == "mRNA": - attr = parts[8] - # 提取ID（如 ID=AT2G47070.1），保留版本号与BLAST结果匹配 - gene_id = attr.split(';')[0].replace('ID=', '') - - # 处理染色体名称：去掉 "Chr" 前缀（Ath用1,2,3而非Chr1,Chr2） - chr_name = parts[0] - if chr_name.startswith('Chr'): - chr_name = chr_name[3:] # Chr1 -> 1 - - # 去重：一个基因可能有多个转录本 - if gene_id not in seen_genes: - seen_genes.add(gene_id) - f.write(f"{chr_name}\t{gene_id}\t{parts[3]}\t{parts[4]}\n") - count += 1 - - print(f" {species}: {count} 个基因（去重后）") - results[prefix] = {"success": True, "output_file": output_gff, "gene_count": count} - except Exception as e: - results[prefix] = {"success": False, "stderr": str(e)} - - if results.get("ref", {}).get("success") and results.get("target", {}).get("success"): - return { - "success": True, - "ref_gff": results["ref"]["output_file"], - "target_gff": results["target"]["output_file"] - } - return {"success": False} - - -def run_reciprocal_blast( - ref_protein: Path, - target_protein: Path, - output_dir: Path, - ref_species: str, - target_species: str, - cpu: int = 8 -) -> Dict[str, Any]: - """双向BLAST比对 - 结果中基因名去掉版本后缀以匹配GFF""" - import re - print(f"双向BLAST比对...") - - # 构建数据库 - for protein_file in [ref_protein, target_protein]: - subprocess.run(["makeblastdb", "-in", str(protein_file), "-dbtype", "prot"], - capture_output=True, check=False) - - # 双向blastp - blast1_out = output_dir / f"tmp.{ref_species}_{target_species}.blast" - subprocess.run([ - "blastp", "-query", str(ref_protein), "-db", str(target_protein), - "-outfmt", "6", "-out", str(blast1_out), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ], capture_output=True, check=False) - - blast2_out = output_dir / f"tmp.{target_species}_{ref_species}.blast" - subprocess.run([ - "blastp", "-query", str(target_protein), "-db", str(ref_protein), - "-outfmt", "6", "-out", str(blast2_out), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ], capture_output=True, check=False) - - # 合并BLAST结果（保留版本号与GFF匹配） - merged_blast = output_dir / f"{ref_species}_{target_species}.blast" - with open(merged_blast, 'w') as outfile: - for blast_file in [blast1_out, blast2_out]: - if blast_file.exists(): - with open(blast_file, 'r') as infile: - for line in infile: - outfile.write(line) # 直接写入，保留版本号 - - return {"success": True, "output_file": merged_blast} - - -def run_mcscanx_interspecies( - work_dir: Path, - ref_species: str, - target_species: str -) -> Dict[str, Any]: - """MCScanX种间共线性 - 需要合并的GFF文件""" - print(f"MCScanX种间共线性...") - - prefix = f"{ref_species}_{target_species}" - - # MCScanX 需要合并的 GFF 文件 (prefix.gff) - merged_gff = work_dir / f"{prefix}.gff" - ref_gff = work_dir / f"{ref_species}.gff" - target_gff = work_dir / f"{target_species}.gff" - - if not merged_gff.exists() and ref_gff.exists() and target_gff.exists(): - print(f" 合并 GFF 文件: {ref_gff.name} + {target_gff.name} -> {merged_gff.name}") - with open(merged_gff, 'w') as outfile: - for gff_file in [target_gff, ref_gff]: # 目标物种在前，参考物种在后 - with open(gff_file, 'r') as infile: - outfile.write(infile.read()) - - cmd = ["MCScanX", f"./{prefix}"] - - result = subprocess.run(cmd, capture_output=True, text=True, - cwd=str(work_dir), check=False) - - collinearity_file = work_dir / f"{prefix}.collinearity" - - return { - "success": collinearity_file.exists(), - "collinearity_file": collinearity_file if collinearity_file.exists() else None - } - - - - -def fasta_ids(fasta_file: Path) -> List[str]: - ids = [] - with open(fasta_file, "r") as f: - for line in f: - if line.startswith(">"): - ids.append(line[1:].strip().split()[0]) - return ids - - -def display_species_code(species: str, gene_id: str = "") -> str: - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return species - - -def species_for_gene(gene_id: str, species_list: List[str], default_species: str) -> str: - for species in sorted(species_list, key=len, reverse=True): - if gene_id.lower().startswith(species.lower()): - return display_species_code(species, gene_id) - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return display_species_code(default_species, gene_id) - - -def write_gene_species_map(target_fasta: Path, ref_fasta: Path, target_species: List[str], ref_species: List[str], output_file: Path) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - default_target = target_species[0] if target_species else "Target" - default_ref = ref_species[0] if ref_species else "Ref" - with open(output_file, "w") as out: - out.write("gene_id\tspecies\tgroup\n") - for gene_id in fasta_ids(target_fasta): - out.write(f"{gene_id}\t{species_for_gene(gene_id, target_species, default_target)}\ttarget\n") - for gene_id in fasta_ids(ref_fasta): - out.write(f"{gene_id}\t{species_for_gene(gene_id, ref_species, default_ref)}\tref\n") - return output_file -def normalize_seq_id(text: str) -> str: - """Normalize FASTA/GFF IDs while preserving biologically meaningful versions.""" - value = (text or "").strip() - if value.startswith(">"): - value = value[1:] - value = value.split()[0] if value.split() else value - if "|" in value: - value = value.split("|", 1)[0] - for prefix in ("gene:", "transcript:", "CDS:", "rna-", "gene-", "cds-"): - if value.startswith(prefix): - value = value[len(prefix):] - return value.strip().strip('"') - - -def id_variants(gene_id: str) -> set: - norm = normalize_seq_id(gene_id) - variants = {norm} - for prefix in ("gene:", "transcript:", "CDS:", "rna-", "gene-", "cds-"): - variants.add(normalize_seq_id(prefix + norm)) - if "|" in gene_id: - variants.add(normalize_seq_id(gene_id.split("|", 1)[0])) - return {v for v in variants if v} - - -def fasta_ids(fasta_file: Path) -> List[str]: - ids = [] - with open(fasta_file, "r") as f: - for line in f: - if line.startswith(">"): - seq_id = normalize_seq_id(line[1:]) - if seq_id: - ids.append(seq_id) - return ids - - -def fasta_id_set(fasta_file: Path) -> set: - ids = set() - for seq_id in fasta_ids(fasta_file): - ids.update(id_variants(seq_id)) - return ids - - -def display_species_code(species: str, gene_id: str = "") -> str: - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return species - - -def species_for_gene(gene_id: str, species_list: List[str], default_species: str, species_id_sets: Optional[Dict[str, set]] = None) -> str: - variants = id_variants(gene_id) - if species_id_sets: - for species, ids in species_id_sets.items(): - if variants & ids: - return display_species_code(species, gene_id) - for species in sorted(species_list, key=len, reverse=True): - if normalize_seq_id(gene_id).lower().startswith(species.lower()): - return display_species_code(species, gene_id) - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return display_species_code(default_species, gene_id) - - -def collect_species_id_sets(groups: Dict[str, Any]) -> Dict[str, set]: - species_id_sets: Dict[str, set] = {} - for group_name in ("target", "ref"): - for species, item in groups.get(group_name, {}).items(): - pep = item.get("resolved", {}).get("pep") - if pep and Path(pep).exists(): - species_id_sets[species] = fasta_id_set(Path(pep)) - return species_id_sets - - -def write_gene_species_map( - target_fasta: Path, - ref_fasta: Path, - target_species: List[str], - ref_species: List[str], - output_file: Path, - species_id_sets: Optional[Dict[str, set]] = None, -) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - default_target = target_species[0] if target_species else "Target" - default_ref = ref_species[0] if ref_species else "Ref" - with open(output_file, "w") as out: - out.write("gene_id\tspecies\tgroup\n") - for gene_id in fasta_ids(target_fasta): - species = species_for_gene(gene_id, target_species, default_target, species_id_sets) - out.write(f"{gene_id}\t{species}\ttarget\n") - for gene_id in fasta_ids(ref_fasta): - species = species_for_gene(gene_id, ref_species, default_ref, species_id_sets) - out.write(f"{gene_id}\t{species}\tref\n") - return output_file - - -def run_reciprocal_blast( - target_protein: Path, - ref_protein: Path, - output_dir: Path, - target_species: str, - ref_species: str, - cpu: int = 8 -) -> Dict[str, Any]: - """Run pairwise reciprocal BLASTP for one target-ref species pair.""" - print(f"Reciprocal BLASTP: {target_species} vs {ref_species}") - - pair_prefix = f"{target_species}.{ref_species}" - target_db = output_dir / f"{target_species}.blastdb" - ref_db = output_dir / f"{ref_species}.blastdb" - commands = [] - - for protein_file, db_prefix in [(target_protein, target_db), (ref_protein, ref_db)]: - cmd = ["makeblastdb", "-in", str(protein_file), "-dbtype", "prot", "-out", str(db_prefix)] - commands.append(" ".join(cmd)) - result = subprocess.run(cmd, capture_output=True, text=True, check=False) - if result.returncode != 0: - return {"success": False, "stderr": result.stderr, "commands": commands} - - target_to_ref = output_dir / f"tmp.{target_species}_{ref_species}.blast" - ref_to_target = output_dir / f"tmp.{ref_species}_{target_species}.blast" - for query, db_prefix, out_file in [(target_protein, ref_db, target_to_ref), (ref_protein, target_db, ref_to_target)]: - cmd = [ - "blastp", "-query", str(query), "-db", str(db_prefix), - "-outfmt", "6", "-out", str(out_file), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ] - commands.append(" ".join(cmd)) - result = subprocess.run(cmd, capture_output=True, text=True, check=False) - if result.returncode != 0: - return {"success": False, "stderr": result.stderr, "commands": commands} - - merged_blast = output_dir / f"{pair_prefix}.blast" - with open(merged_blast, 'w') as outfile: - for blast_file in [target_to_ref, ref_to_target]: - if blast_file.exists(): - with open(blast_file, 'r') as infile: - for line in infile: - if line.strip(): - outfile.write(line) - - return {"success": merged_blast.exists() and merged_blast.stat().st_size > 0, "output_file": merged_blast, "commands": commands} - - -def run_mcscanx_interspecies(work_dir: Path, target_species: str, ref_species: str) -> Dict[str, Any]: - """Run MCScanX for one target-ref pair.""" - pair_prefix = f"{target_species}.{ref_species}" - print(f"MCScanX pair: {pair_prefix}") - - merged_gff = work_dir / f"{pair_prefix}.gff" - target_gff = work_dir / f"{target_species}.gff" - ref_gff = work_dir / f"{ref_species}.gff" - - if target_gff.exists() and ref_gff.exists(): - with open(merged_gff, 'w') as outfile: - for gff_file in [target_gff, ref_gff]: - with open(gff_file, 'r') as infile: - outfile.write(infile.read().rstrip() + "\n") - else: - return {"success": False, "error": f"Missing simplified GFF for {pair_prefix}"} - - result = subprocess.run(["MCScanX", f"./{pair_prefix}"], capture_output=True, text=True, cwd=str(work_dir), check=False) - collinearity_file = work_dir / f"{pair_prefix}.collinearity" - return { - "success": collinearity_file.exists() and collinearity_file.stat().st_size > 0, - "collinearity_file": collinearity_file if collinearity_file.exists() else None, - "stdout": result.stdout, - "stderr": result.stderr, - "returncode": result.returncode, - } - - -def plot_interspecies_tree( - tree_file: Path, - output_dir: Path, - prefix: str, - target_species: str, - ref_species: str, - target_subfamily_file: Optional[Path] = None, - ref_subfamily_file: Optional[Path] = None, - gene_species_map: Optional[Path] = None -) -> Dict[str, Any]: - """绘制物种间进化树图 - 使用专用的 interspecies_tree.R 脚本""" - print(f"绘制进化树图...") - - # 查找专用的种间进化树R脚本 - r_script = Path("scripts/interspecies_tree.R") - if not r_script.exists(): - alt_script = Path("/home/xia-lab/wutx/00test/bbav2/scripts/interspecies_tree.R") - if alt_script.exists(): - r_script = alt_script - else: - return {"success": False, "error": f"R脚本不存在: {r_script}"} - - try: - # 使用专用的 interspecies_tree.R 脚本，传递物种参数和亚家族文件 - # 使用conda环境的Rscript - conda_rscript = "/home/xia-lab/miniconda3/envs/test4py311/bin/Rscript" - cmd = [ - conda_rscript, str(r_script), - str(tree_file), - str(output_dir), - target_species, - ref_species - ] - - # 添加亚家族文件参数（如果存在） - if target_subfamily_file and target_subfamily_file.exists(): - cmd.append(str(target_subfamily_file)) - print(f" 目标物种亚家族文件: {target_subfamily_file}") - else: - cmd.append("") # 空字符串表示无文件 - - if ref_subfamily_file and ref_subfamily_file.exists(): - cmd.append(str(ref_subfamily_file)) - print(f" 参考物种亚家族文件: {ref_subfamily_file}") - else: - cmd.append("") # 空字符串表示无文件 - if gene_species_map and gene_species_map.exists(): - cmd.append(str(gene_species_map)) - print(f" Gene-to-species map: {gene_species_map}") - else: - cmd.append("") - result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=120) - - # 调试输出 - print(f" R脚本返回码: {result.returncode}") - if result.stdout: - print(f" R脚本输出: {result.stdout[:500]}") - if result.stderr: - print(f" R脚本错误: {result.stderr[:500]}") - - # 专用脚本输出固定为 interspecies_tree.pdf/png - output_pdf = output_dir / "interspecies_tree.pdf" - output_png = output_dir / "interspecies_tree.png" - - # 重命名为规范名称 - final_pdf = output_dir / f"{prefix}.interspecies_tree.pdf" - final_png = output_dir / f"{prefix}.interspecies_tree.png" - - if output_pdf.exists(): - shutil.move(output_pdf, final_pdf) - print(f" 进化树PDF: {final_pdf}") - if output_png.exists(): - shutil.move(output_png, final_png) - print(f" 进化树PNG: {final_png}") - - # PDF转PNG（如果没有生成PNG） - if final_pdf.exists() and not final_png.exists(): - png_from_pdf = output_dir / "interspecies_tree.png" - if pdf_to_png(final_pdf, png_from_pdf): - final_png = output_dir / f"{prefix}.interspecies_tree.png" - shutil.move(png_from_pdf, final_png) - print(f" PDF转PNG: {final_png}") - - return { - "success": final_pdf.exists() or final_png.exists(), - "pdf_file": final_pdf if final_pdf.exists() else None, - "png_file": final_png if final_png.exists() else None, - "stdout": result.stdout, - "stderr": result.stderr - } - except Exception as e: - return {"success": False, "error": str(e)} - - -def convert_blast_to_family_simple( - work_dir: Path, - blast_file: Path, - target_species: str, - ref_species: str, -) -> Dict[str, Any]: - """Fallback: convert pairwise BLAST hits directly to JCVI anchors/simple. - - This is not a strict MCScanX collinearity block. It is used when the ref side - only has family proteins, so MCScanX cannot satisfy whole-genome block rules. - """ - pair_prefix = f"{target_species}.{ref_species}" - target_bed = work_dir / f"{target_species}.bed" - ref_bed = work_dir / f"{ref_species}.bed" - anchors_file = work_dir / f"{pair_prefix}.anchors" - simple_file = work_dir / f"{pair_prefix}.anchors.simple" - - valid_target = {line.rstrip("\n").split("\t")[3] for line in open(target_bed) if len(line.rstrip("\n").split("\t")) >= 4} - valid_ref = {line.rstrip("\n").split("\t")[3] for line in open(ref_bed) if len(line.rstrip("\n").split("\t")) >= 4} - seen = set() - links = [] - if not blast_file.exists(): - return {"success": False, "error": "BLAST file missing for family-only fallback"} - with open(blast_file, "r") as f: - for line in f: - if not line.strip() or line.startswith("#"): - continue - parts = line.rstrip("\n").split("\t") - if len(parts) < 2: - continue - a = normalize_seq_id(parts[0]) - b = normalize_seq_id(parts[1]) - target_gene = ref_gene = None - if a in valid_target and b in valid_ref: - target_gene, ref_gene = a, b - elif b in valid_target and a in valid_ref: - target_gene, ref_gene = b, a - if not target_gene or not ref_gene: - continue - key = (target_gene, ref_gene) - if key in seen: - continue - seen.add(key) - score_text = parts[11] if len(parts) > 11 else "1000" - try: - score = str(int(float(score_text))) - except Exception: - score = "1000" - links.append((target_gene, ref_gene, score)) - if not links: - return {"success": False, "error": "No BLAST hits matched pair BED IDs for family-only fallback"} - with open(anchors_file, "w") as out: - out.write("##\n") - for target_gene, ref_gene, score in links: - out.write(f"{target_gene}\t{ref_gene}\t{score}\n") - with open(simple_file, "w") as out: - for target_gene, ref_gene, score in links: - out.write(f"{target_gene}\t{target_gene}\t{ref_gene}\t{ref_gene}\t{score}\t+\n") - return { - "success": True, - "anchors_file": anchors_file, - "simple_file": simple_file, - "anchor_count": len(links), - "fallback": "family_only_blast_links", - } - - -def convert_collinearity_to_simple( - work_dir: Path, - collinearity_file: Path, - target_species: str, - ref_species: str, -) -> Dict[str, Any]: - """Convert one MCScanX .collinearity file into JCVI anchors/simple files.""" - conda_python = "/home/xia-lab/miniconda3/envs/test4py311/bin/python" - pair_prefix = f"{target_species}.{ref_species}" - target_bed = work_dir / f"{target_species}.bed" - ref_bed = work_dir / f"{ref_species}.bed" - anchors_file = work_dir / f"{pair_prefix}.anchors" - anchors_new_file = work_dir / f"{pair_prefix}.anchors.new" - simple_file = work_dir / f"{pair_prefix}.anchors.simple" - - valid_target = {line.rstrip("\n").split("\t")[3] for line in open(target_bed) if len(line.rstrip("\n").split("\t")) >= 4} - valid_ref = {line.rstrip("\n").split("\t")[3] for line in open(ref_bed) if len(line.rstrip("\n").split("\t")) >= 4} - - current_block = [] - anchors = 0 - filtered = 0 - with open(collinearity_file, "r") as f_in, open(anchors_file, "w") as f_out: - for line in f_in: - if not line.strip(): - continue - if line.startswith("## Alignment"): - if current_block: - f_out.write("##\n") - f_out.writelines(current_block) - current_block = [] - continue - if line.startswith("#"): - continue - parts = line.strip().split("\t") - if len(parts) < 3: - continue - gene_a = normalize_seq_id(parts[1]) - gene_b = normalize_seq_id(parts[2]) - target_gene = ref_gene = None - if gene_a in valid_target and gene_b in valid_ref: - target_gene, ref_gene = gene_a, gene_b - elif gene_b in valid_target and gene_a in valid_ref: - target_gene, ref_gene = gene_b, gene_a - if target_gene and ref_gene: - current_block.append(f"{target_gene} {ref_gene} 1000\n") - anchors += 1 - else: - filtered += 1 - if current_block: - f_out.write("##\n") - f_out.writelines(current_block) - - if anchors == 0: - return {"success": False, "error": "No anchors matched pair BED IDs", "anchors_file": anchors_file, "filtered": filtered} - - cmd_screen = [ - conda_python, "-m", "jcvi.compara.synteny", "screen", - "--minspan=10", "--simple", str(anchors_file), str(anchors_new_file), - f"--qbed={target_bed.name}", f"--sbed={ref_bed.name}", - ] - result = subprocess.run(cmd_screen, capture_output=True, text=True, cwd=str(work_dir), check=False, timeout=180) - for candidate in (work_dir / f"{pair_prefix}.anchors.new.simple", work_dir / f"{pair_prefix}.new.simple"): - if candidate.exists(): - shutil.copy(candidate, simple_file) - break - if not simple_file.exists(): - shutil.copy(anchors_file, simple_file) - - return { - "success": simple_file.exists() and simple_file.stat().st_size > 0, - "anchors_file": anchors_file, - "anchors_new_file": anchors_new_file, - "simple_file": simple_file, - "anchor_count": anchors, - "filtered_count": filtered, - "screen_returncode": result.returncode, - "screen_stderr": result.stderr, - } - - -def _chr_sort_key(chrom: str): - cleaned = re.sub(r"^(chr|Chr|CHR)", "", chrom) - if cleaned.isdigit(): - return (0, int(cleaned)) - return (1, cleaned) - - -def _read_bed_chroms(bed_file: Path) -> List[str]: - skip = {"C", "M", "ChrC", "ChrM", "chrc", "chrm"} - chroms = set() - with open(bed_file, "r") as f: - for line in f: - parts = line.strip().split("\t") - if len(parts) >= 4 and parts[0] not in skip: - chroms.add(parts[0]) - return sorted(chroms, key=_chr_sort_key) - - -def _bed_gene_ids(bed_file: Path) -> set: - ids = set() - if not bed_file.exists(): - return ids - with open(bed_file, "r") as f: - for line in f: - parts = line.rstrip("\n").split("\t") - if len(parts) >= 4: - ids.add(normalize_seq_id(parts[3])) - return ids - - -def _is_family_gene(gene_id: str, family_ids: set) -> bool: - return bool(set(id_variants(gene_id)) & family_ids) - - -def _family_links_from_blast( - blast_file: Path, - family_ids: set, - target_bed_ids: set, - ref_bed_ids: set, - max_links: int = 300, -) -> List[Tuple[str, str, int]]: - if not blast_file or not blast_file.exists() or not family_ids: - return [] - best: Dict[Tuple[str, str], float] = {} - with open(blast_file, "r") as f: - for line in f: - parts = line.rstrip("\n").split("\t") - if len(parts) < 12: - continue - q, s = normalize_seq_id(parts[0]), normalize_seq_id(parts[1]) - if q not in target_bed_ids or s not in ref_bed_ids: - continue - if not (_is_family_gene(q, family_ids) and _is_family_gene(s, family_ids)): - continue - try: - score = float(parts[11]) - except Exception: - score = 1.0 - key = (q, s) - if score > best.get(key, -1): - best[key] = score - ranked = sorted(best.items(), key=lambda item: item[1], reverse=True)[:max_links] - return [(q, s, max(1, int(round(score)))) for (q, s), score in ranked] - - -def _write_marked_simple_file( - simple_file: Path, - marked_file: Path, - family_ids: set, - family_links: List[Tuple[str, str, int]], -) -> int: - if not simple_file.exists() or not family_ids: - return 0 - marked_count = 0 - seen_rows = set() - with open(simple_file, "r") as f_in, open(marked_file, "w") as f_out: - for line in f_in: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - f_out.write(line) - continue - parts = stripped.split() - genes = [normalize_seq_id(p.split("*", 1)[-1]) for p in parts[:4]] - row_key = tuple(parts[:6]) if len(parts) >= 6 else tuple(parts) - seen_rows.add(row_key) - if any(_is_family_gene(g, family_ids) for g in genes) and "*" not in parts[0]: - f_out.write("#e31a1c*" + "\t".join(parts) + "\n") - marked_count += 1 - else: - f_out.write(line) - for q, s, score in family_links: - row = (q, q, s, s, str(score), "+") - if row in seen_rows: - continue - f_out.write("#e31a1c*" + "\t".join(row) + "\n") - seen_rows.add(row) - marked_count += 1 - return marked_count - - -def plot_multi_species_synteny( - work_dir: Path, - output_dir: Path, - prefix: str, - species_order: List[str], - pair_results: List[Dict[str, Any]], - family_ids: set, -) -> Dict[str, Any]: - """Draw one multi-track JCVI karyotype figure from all successful target-ref pairs.""" - print("Plotting multi-species JCVI synteny...") - conda_python = "/home/xia-lab/miniconda3/envs/test4py311/bin/python" - usable_pairs = [p for p in pair_results if p.get("simple_file") and Path(p["simple_file"]).exists()] - if not usable_pairs: - return {"success": False, "error": "No pairwise anchors.simple files available for JCVI"} - - seqids_file = work_dir / f"{prefix}.seqids" - layout_file = work_dir / f"{prefix}.layout" - layout_mark_file = work_dir / f"{prefix}.layout_mark" - - species_with_bed = [] - with open(seqids_file, "w") as f: - for species in species_order: - bed = work_dir / f"{species}.bed" - if not bed.exists(): - continue - chroms = _read_bed_chroms(bed) - if not chroms: - continue - species_with_bed.append(species) - f.write(",".join(chroms) + "\n") - - if len(species_with_bed) < 2: - return {"success": False, "error": "Fewer than two species BED tracks are available"} - - index = {species: i for i, species in enumerate(species_with_bed)} - n = len(species_with_bed) - y_values = [round(0.85 - (0.7 * i / max(n - 1, 1)), 3) for i in range(n)] - palette = ["#d73027", "#1a9850", "#4575b4", "#984ea3", "#ff7f00", "#4daf4a", "#377eb8", "#a65628"] - - def write_layout(layout: Path, use_marked: bool) -> None: - with open(layout, "w") as f: - f.write("# y, xstart, xend, rotation, color, label, va, bed\n") - for i, species in enumerate(species_with_bed): - va = "top" if i == 0 else "bottom" - f.write(f"{y_values[i]}, .08, .92, 0, {palette[i % len(palette)]}, {species}, {va}, {species}.bed\n") - f.write("# edges\n") - for pair in usable_pairs: - target = pair["target_species"] - ref = pair["ref_species"] - if target not in index or ref not in index: - continue - edge = Path(pair.get("marked_simple_file") if use_marked and pair.get("marked_simple_file") else pair["simple_file"]).name - f.write(f"e, {index[target]}, {index[ref]}, {edge}\n") - - for pair in usable_pairs: - simple = Path(pair["simple_file"]) - marked = work_dir / f"{pair['target_species']}.{pair['ref_species']}.anchors.simple.mark" - target_bed_ids = _bed_gene_ids(work_dir / f"{pair['target_species']}.bed") - ref_bed_ids = _bed_gene_ids(work_dir / f"{pair['ref_species']}.bed") - family_links = _family_links_from_blast(Path(pair.get("blast_file", "")), family_ids, target_bed_ids, ref_bed_ids) - marked_count = _write_marked_simple_file(simple, marked, family_ids, family_links) - pair["family_highlight_links"] = len(family_links) - pair["marked_block_count"] = marked_count - if marked_count > 0: - pair["marked_simple_file"] = str(marked) - - write_layout(layout_file, use_marked=False) - write_layout(layout_mark_file, use_marked=True) - - def run_plot(layout: Path, final_pdf: Path, final_png: Path) -> Dict[str, Any]: - for stale in [work_dir / "karyotype.pdf", work_dir / "karyotype.png", final_pdf, final_png]: - if stale.exists(): - stale.unlink() - cmd = [conda_python, "-m", "jcvi.graphics.karyotype", str(seqids_file.resolve()), str(layout.resolve()), "--notex"] - result = subprocess.run(cmd, capture_output=True, text=True, cwd=str(work_dir), check=False, timeout=180) - if (work_dir / "karyotype.pdf").exists(): - shutil.copy(work_dir / "karyotype.pdf", final_pdf.resolve()) - if (work_dir / "karyotype.png").exists(): - shutil.copy(work_dir / "karyotype.png", final_png.resolve()) - if final_pdf.exists() and (not final_png.exists() or final_png.stat().st_mtime < final_pdf.stat().st_mtime): - pdf_to_png(final_pdf, final_png) - return {"returncode": result.returncode, "stdout": result.stdout, "stderr": result.stderr} - - final_pdf = output_dir / f"{prefix}.interspecies_synteny.pdf" - final_png = output_dir / f"{prefix}.interspecies_synteny.png" - final_mark_pdf = output_dir / f"{prefix}.interspecies_synteny.marked.pdf" - final_mark_png = output_dir / f"{prefix}.interspecies_synteny.marked.png" - - base_result = run_plot(layout_file, final_pdf, final_png) - mark_result = run_plot(layout_mark_file, final_mark_pdf, final_mark_png) - - return { - "success": final_pdf.exists() or final_png.exists(), - "pdf_file": final_pdf if final_pdf.exists() else None, - "png_file": final_png if final_png.exists() else None, - "marked_pdf_file": final_mark_pdf if final_mark_pdf.exists() else None, - "marked_png_file": final_mark_png if final_mark_png.exists() else None, - "seqids_file": seqids_file, - "layout_file": layout_file, - "layout_mark_file": layout_mark_file, - "base_plot": base_result, - "marked_plot": mark_result, - } - - -def write_report(workspace: Path, prefix: str, metrics: Dict, parameters: Dict, - tool_versions: Dict, start_time: datetime, end_time: datetime, - elapsed_sec: float, cmd: List[str]) -> Path: - """写入报告""" - report_path = workspace / "final" / f"{prefix}.interspecies_synteny_report.md" - - # 收集可复现性信息 - reproducibility_info = [] - reproducibility_info.append("## 可复现性信息\n\n") - reproducibility_info.append("### 执行环境\n") - reproducibility_info.append(f"- **分析时间**: {start_time.isoformat()}\n") - reproducibility_info.append(f"- **完成时间**: {end_time.isoformat()}\n") - reproducibility_info.append(f"- **运行时长**: {elapsed_sec:.2f} 秒\n") - reproducibility_info.append(f"- **工作目录**: {workspace}\n") - reproducibility_info.append(f"- **输入目录**: {parameters.get('input_dir', 'N/A')}\n\n") - - reproducibility_info.append("### 工具版本\n") - for tool, version in tool_versions.items(): - reproducibility_info.append(f"- **{tool}**: {version}\n") - reproducibility_info.append("\n") - - reproducibility_info.append("### 执行命令\n") - reproducibility_info.append(f"```bash\n{' '.join(cmd)}\n```\n\n") - - reproducibility_info.append("### 输入参数\n") - for key, value in parameters.items(): - if key != 'input_dir': - reproducibility_info.append(f"- **{key}**: {value}\n") - reproducibility_info.append("\n") - - with open(report_path, "w") as f: - f.write(f"# 物种间共线性分析 - {prefix}\n\n") - f.write(f"**时间**: {datetime.now().isoformat()}\n\n") - f.write(f"- 目标物种 (研究对象): {parameters.get('target_species')}\n") - f.write(f"- 参考物种 (对照): {parameters.get('ref_species')}\n") - f.write(f"- 目标蛋白文件: {parameters.get('target_protein_spl')}\n") - f.write(f"- 参考蛋白文件: {parameters.get('ref_protein_spl')}\n") - f.write(f"- 目标全蛋白组: {parameters.get('target_protein_all')}\n") - f.write(f"- 参考全蛋白组: {parameters.get('ref_protein_all')}\n") - f.write("\n") - f.writelines(reproducibility_info) - return report_path - - -def get_file_info(file_path: Path) -> Dict[str, Any]: - """获取文件的详细信息，用于artifact规范""" - info = { - "size": 0, - "sha256": "", - "mtime": 0 - } - if file_path.exists(): - info["size"] = file_path.stat().st_size - info["mtime"] = file_path.stat().st_mtime - info["sha256"] = compute_file_hash(file_path) - return info - - -def write_manifest(workspace: Path, job_id: str, run_id: str, prefix: str, - start_time: datetime, end_time: datetime, artifacts: Dict, - metrics: Dict, parameters: Dict, tool_versions: Dict, - workspace_root: Path, input_dir: Path, cmd: List[str], - status: str = "success") -> Path: - """写入manifest - 符合schema v1标准""" - elapsed_sec = (end_time - start_time).total_seconds() - - # 构建符合规范的artifacts结构 - formatted_artifacts = {} - for key, value in artifacts.items(): - if isinstance(value, dict): - # 已经是字典格式，确保包含必需字段 - formatted_artifacts[key] = value - elif isinstance(value, str): - # 字符串格式，转换为artifact对象 - file_path = workspace / value - file_info = get_file_info(file_path) - formatted_artifacts[key] = { - "artifact": key, - "path": value, - "full_path": str(file_path.absolute()), - "type": "file", - "size": file_info["size"], - "sha256": file_info["sha256"], - "mtime": file_info["mtime"] - } - - manifest_data = { - "manifest_version": "1.0", - "job_id": job_id, - "stage": "interspecies_synteny", - "run_id": run_id, - "prefix": prefix, - "timestamp": end_time.isoformat(), - "status": status, - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - "elapsed_sec": round(elapsed_sec, 2), - "cmd": cmd, - "workspace_root": str(workspace_root.absolute()), - "workspace": str(workspace.absolute()), - "input_dir": str(input_dir.absolute()), - "tool_versions": tool_versions, - "artifacts": formatted_artifacts, - "parameters": parameters, - "metrics": metrics - } - - manifest_path = workspace / "final" / f"{prefix}.interspecies_synteny.manifest.json" - with open(manifest_path, "w", encoding='utf-8') as f: - json.dump(manifest_data, f, indent=2, ensure_ascii=False) - - return manifest_path - - - -def _clean_bool(value: Any) -> bool: - return str(value).strip().lower() in {"true", "1", "yes", "y"} - - -def _gene_family_from_prefix(prefix: str) -> str: - return prefix.split('.', 1)[1] if '.' in prefix else prefix - - -def _resolve_path(path_text: str, input_dir: Path) -> Optional[Path]: - if not path_text: - return None - path = Path(path_text) - if path.is_absolute(): - return path if path.exists() else None - for candidate in (input_dir / path, input_dir / path.name): - if candidate.exists(): - return candidate - return None - - -def _read_config_groups(input_dir: Path, default_family: str) -> Dict[str, Any]: - config_path = input_dir / "config.xlsx" - groups = {"target": {}, "ref": {}, "gene_family": default_family} - if not config_path.exists(): - return groups - try: - import pandas as pd - df = pd.read_excel(config_path) - except Exception as exc: - print(f"[WARN] Failed to read config.xlsx: {exc}") - return groups - for _, row in df.iterrows(): - species = str(row.get("Species_Code", "")).strip() - family = str(row.get("Gene_Family", default_family)).strip() or default_family - file_type = str(row.get("File_Type", "")).strip().lower() - original_name = str(row.get("Original_File_Name", "")).strip() - if not species: - continue - group_name = "ref" if _clean_bool(row.get("Ref", False)) else "target" - item = groups[group_name].setdefault(species, {"species": species, "gene_family": family, "files": {}}) - item["gene_family"] = family - if file_type: - item["files"][file_type] = original_name - groups["gene_family"] = family or groups["gene_family"] - return groups - - -def _read_genome_prepare_artifacts(workspace: Path, input_dir: Path, prefix: str) -> Dict[str, Dict[str, Path]]: - result: Dict[str, Dict[str, Path]] = {} - for manifest_path in (workspace / "final" / f"{prefix}.genome_prepare.manifest.json", input_dir / "final" / f"{prefix}.genome_prepare.manifest.json"): - if not manifest_path.exists(): - continue - try: - data = json.loads(manifest_path.read_text(encoding="utf-8")) - except Exception as exc: - print(f"[WARN] Failed to read genome_prepare manifest {manifest_path}: {exc}") - continue - for key, artifact in (data.get("artifacts") or {}).items(): - kind = artifact.get("artifact") - species = key.rsplit("_", 1)[0] - found = _resolve_path(artifact.get("full_path") or artifact.get("path") or "", input_dir) - if found and kind: - result.setdefault(species, {})[kind] = found - return result - - -def _find_species_file(input_dir: Path, species: str, family: str, file_type: str, original_name: str = "") -> Optional[Path]: - if file_type == "all_pep": - patterns = [ - f"{species}.{family}.all_pep.fasta", f"{species}.{family}.all_pep.fa", - f"{species}.{family}.protein.fasta", f"{species}.{family}.faa", - f"{species}.{family}.pep.fasta", f"{species}.{family}.pep.fa", - f"{species}.all_pep.fasta", f"{species}.pep.fasta", f"{species}.protein.fasta", f"{species}.faa", - ] - elif file_type == "family_pep": - patterns = [ - f"{species}.{family}.family_pep.fasta", f"{species}.{family}.family_pep.fa", - f"{species}.{family}.pep.fasta", f"{species}.{family}.pep.fa", - ] - elif file_type == "pep": - patterns = [ - f"{species}.{family}.pep.fasta", f"{species}.{family}.pep.fa", - f"{species}.{family}.protein.fasta", f"{species}.{family}.faa", - f"{species}.pep.fasta", f"{species}.pep.fa", f"{species}.protein.fasta", f"{species}.faa", - ] - elif file_type == "gff": - patterns = [ - f"{species}.{family}_final.gff3", f"{species}.{family}.gff3", f"{species}.{family}.gff", - f"{species}_final.gff3", f"{species}.gff3", f"{species}.gff", - ] - else: - patterns = [] - if original_name: - patterns.append(Path(original_name).name) - for pattern in patterns: - for candidate in (input_dir / pattern, input_dir / species / pattern): - if candidate.exists(): - return candidate - return None - -def _parse_species_selection(value: Optional[str]) -> List[str]: - if not value: - return [] - return [item.strip() for item in str(value).split(',') if item.strip()] - - -def _filter_group_by_selection(group: Dict[str, Any], selected: List[str]) -> Dict[str, Any]: - if not selected: - return group - selected_set = set(selected) - return {species: item for species, item in group.items() if species in selected_set} - - -def _species_default_selection(available: List[str], preferred_ref: bool = False) -> List[str]: - if not available: - return [] - if preferred_ref: - for preferred in ("at", "ath"): - for species in available: - if species.lower() == preferred: - return [species] - return [available[0]] - return list(available) - - -def _gene_matches_species(gene_id: str, species: str, species_id_sets: Dict[str, set]) -> bool: - variants = id_variants(gene_id) - if species in species_id_sets and variants & species_id_sets[species]: - return True - return normalize_seq_id(gene_id).lower().startswith(species.lower()) - - -def _filter_fasta_by_species(source: Path, output_file: Path, selected_species: List[str], species_id_sets: Dict[str, set]) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - if not selected_species: - shutil.copy(source, output_file) - return output_file - keep = False - written = 0 - with open(source, "r") as f_in, open(output_file, "w") as f_out: - for line in f_in: - if line.startswith(">"): - gene_id = normalize_seq_id(line[1:]) - keep = any(_gene_matches_species(gene_id, species, species_id_sets) for species in selected_species) - if keep: - written += 1 - f_out.write(line) - elif keep: - f_out.write(line) - if written == 0: - raise RuntimeError(f"Selected species {selected_species} produced an empty filtered FASTA from {source}") - return output_file - - -def discover_group_inputs(workspace: Path, input_dir: Path, prefix: str, args) -> Dict[str, Any]: - family = _gene_family_from_prefix(prefix) - groups = _read_config_groups(input_dir, family) - artifacts = _read_genome_prepare_artifacts(workspace, input_dir, prefix) - family = groups.get("gene_family") or family - - available_target_species = list(groups["target"].keys()) - available_ref_species = list(groups["ref"].keys()) - requested_targets = _parse_species_selection(getattr(args, "target_species", None)) - requested_refs = _parse_species_selection(getattr(args, "ref_species", None)) - selected_targets = requested_targets or _species_default_selection(available_target_species, preferred_ref=False) - selected_refs = requested_refs or _species_default_selection(available_ref_species, preferred_ref=True) - - unknown_targets = [sp for sp in selected_targets if sp not in groups["target"]] - unknown_refs = [sp for sp in selected_refs if sp not in groups["ref"]] - if unknown_targets: - raise RuntimeError(f"Selected target species not found in config Ref=False group: {', '.join(unknown_targets)}") - if unknown_refs: - raise RuntimeError(f"Selected reference species not found in config Ref=True group: {', '.join(unknown_refs)}") - if not selected_targets: - raise RuntimeError("No target species selected for interspecies analysis") - if not selected_refs: - raise RuntimeError("No reference species selected for interspecies analysis") - - groups["target"] = _filter_group_by_selection(groups["target"], selected_targets) - groups["ref"] = _filter_group_by_selection(groups["ref"], selected_refs) - - for group_name in ("target", "ref"): - for species, item in groups[group_name].items(): - files = item.setdefault("resolved", {}) - artifact_paths = artifacts.get(species, {}) - original_files = item.get("files", {}) - item_family = item.get("gene_family", family) - all_original = original_files.get("all_pep", "") or original_files.get("protein", "") - family_original = original_files.get("family_pep", "") - legacy_pep_original = original_files.get("pep", "") - if group_name == "ref" and legacy_pep_original and not family_original: - family_original = legacy_pep_original - if group_name == "target" and legacy_pep_original and not all_original: - all_original = legacy_pep_original - files["all_pep"] = ( - _find_species_file(input_dir, species, item_family, "all_pep", all_original) or - artifact_paths.get("all_pep") or artifact_paths.get("ref_all_pep") or artifact_paths.get("pep") - ) - files["family_pep"] = ( - _find_species_file(input_dir, species, item_family, "family_pep", family_original) or - artifact_paths.get("family_pep") or artifact_paths.get("ref_family_pep") or artifact_paths.get("ref_pep") - ) - files["pep"] = files.get("all_pep") or files.get("family_pep") - files["gff"] = ( - artifact_paths.get("gff") or artifact_paths.get("ref_gff") or - _find_species_file(input_dir, species, item_family, "gff", original_files.get("gff", "")) - ) - - analysis_dir = workspace / "interspecies_synteny" / "analysis" - analysis_dir.mkdir(parents=True, exist_ok=True) - species_id_sets = collect_species_id_sets(groups) - - raw_target_family_protein = Path(args.target_protein) if args.target_protein else workspace / "final" / f"{prefix}.final_members.pep.fa" - target_family_protein = raw_target_family_protein - if raw_target_family_protein and raw_target_family_protein.exists(): - target_family_protein = _filter_fasta_by_species( - raw_target_family_protein, - analysis_dir / f"{prefix}.selected_target.family.pep.fa", - selected_targets, - species_id_sets, - ) - - ref_family_protein = Path(args.ref_protein) if args.ref_protein else None - if ref_family_protein is None: - ref_peps = [item.get("resolved", {}).get("family_pep") for item in groups["ref"].values() if item.get("resolved", {}).get("family_pep")] - if ref_peps: - ref_family_protein = analysis_dir / f"Ref_selected.{family}.pep.fasta" - merge_many_fastas(ref_peps, ref_family_protein) - else: - for candidate in [input_dir / f"Ref_combined.{family}.pep.fasta", input_dir / f"Ref_combined.{family}.pep.fa", input_dir / "Ref_combined.pep.fasta"]: - if candidate.exists(): - ref_family_protein = candidate - break - if ref_family_protein and Path(ref_family_protein).exists(): - ref_family_protein = _filter_fasta_by_species( - Path(ref_family_protein), - analysis_dir / f"{prefix}.selected_ref.family.pep.fa", - selected_refs, - species_id_sets, - ) - - target_all_peps = {sp: item.get("resolved", {}).get("all_pep") for sp, item in groups["target"].items() if item.get("resolved", {}).get("all_pep")} - ref_all_peps = {sp: item.get("resolved", {}).get("all_pep") for sp, item in groups["ref"].items() if item.get("resolved", {}).get("all_pep")} - target_gffs = {sp: item.get("resolved", {}).get("gff") for sp, item in groups["target"].items() if item.get("resolved", {}).get("gff")} - ref_gffs = {sp: item.get("resolved", {}).get("gff") for sp, item in groups["ref"].items() if item.get("resolved", {}).get("gff")} - - return { - "gene_family": family, - "target_species": selected_targets, - "ref_species": selected_refs, - "available_target_species": available_target_species, - "available_ref_species": available_ref_species, - "selected_target_species": selected_targets, - "selected_ref_species": selected_refs, - "target_family_protein": target_family_protein, - "ref_family_protein": ref_family_protein, - "target_all_peps": target_all_peps, - "ref_all_peps": ref_all_peps, - "target_gffs": target_gffs, - "ref_gffs": ref_gffs, - "groups": groups, - } - - -def merge_many_fastas(files: List[Path], output_file: Path) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as out: - for fasta in files: - with open(fasta, "r") as f: - out.write(f.read().rstrip() + "\n") - return output_file - - -def _parse_gff_attr(attr: str) -> Dict[str, str]: - result = {} - for item in attr.strip().strip(';').split(';'): - item = item.strip() - if not item: - continue - if '=' in item: - k, v = item.split('=', 1) - elif ' ' in item: - k, v = item.split(' ', 1) - else: - continue - result[k.strip()] = v.strip().strip('"') - return result - - -def _match_gff_id(attrs: Dict[str, str], protein_ids: set) -> Optional[str]: - candidates = [attrs.get("ID"), attrs.get("transcript_id"), attrs.get("Name"), attrs.get("Parent")] - for candidate in candidates: - if not candidate: - continue - for item in str(candidate).split(','): - norm = normalize_seq_id(item) - if norm in protein_ids: - return norm - for variant in id_variants(norm): - if variant in protein_ids: - return variant - return None - - -def build_species_position_files(species: str, gff3: Path, protein_fasta: Path, work_dir: Path) -> Dict[str, Any]: - """Create MCScanX simplified GFF and JCVI BED for one species.""" - simple_gff = work_dir / f"{species}.gff" - bed_file = work_dir / f"{species}.bed" - protein_ids = fasta_id_set(protein_fasta) - seen = set() - count = 0 - features_seen = 0 - work_dir.mkdir(parents=True, exist_ok=True) - with open(gff3, "r") as f_in, open(simple_gff, "w") as gff_out, open(bed_file, "w") as bed_out: - for line in f_in: - if not line.strip() or line.startswith('#'): - continue - parts = line.rstrip('\n').split(' ') - if len(parts) < 9 or parts[2] not in {"mRNA", "transcript", "gene"}: - continue - features_seen += 1 - attrs = _parse_gff_attr(parts[8]) - gene_id = _match_gff_id(attrs, protein_ids) - if not gene_id or gene_id in seen: - continue - seen.add(gene_id) - chrom = parts[0] - start, end = parts[3], parts[4] - gff_out.write(f"{chrom} {gene_id} {start} {end}\n") - bed_out.write(f"{chrom} {start} {end} {gene_id}\n") - count += 1 - return { - "success": count > 0, - "species": species, - "gff": simple_gff, - "bed": bed_file, - "gene_count": count, - "features_seen": features_seen, - "protein_id_count": len(protein_ids), - "error": "No GFF IDs matched protein FASTA IDs" if count == 0 else "", - } - - -def collect_family_ids(target_family_fasta: Path, ref_family_fasta: Path, final_id_file: Optional[Path] = None) -> set: - ids = set() - for fasta in [target_family_fasta, ref_family_fasta]: - if fasta and Path(fasta).exists(): - ids.update(fasta_id_set(Path(fasta))) - if final_id_file and final_id_file.exists(): - with open(final_id_file, "r") as f: - for line in f: - if line.strip() and not line.startswith('#'): - ids.update(id_variants(line.strip().split()[0])) - return ids - - -def add_artifact(artifacts: Dict[str, Dict], key: str, file_path: Path, rel_path: str, artifact: str, file_type: str) -> None: - if not file_path or not file_path.exists(): - return - info = get_file_info(file_path) - artifacts[key] = { - "artifact": artifact, - "path": rel_path, - "full_path": str(file_path.absolute()), - "type": file_type, - "size": info["size"], - "sha256": info["sha256"], - "mtime": info["mtime"], - } - - -def write_failure_manifest(workspace: Path, job_id: str, run_id: str, prefix: str, start_time: datetime, - workspace_root: Path, input_dir: Path, cmd: List[str], error: str) -> Path: - end_time = datetime.now() - return write_manifest( - workspace, job_id, run_id, prefix, start_time, end_time, - artifacts={}, metrics={"error": error}, parameters={"prefix": prefix, "error": error}, - tool_versions={}, workspace_root=workspace_root, input_dir=input_dir, cmd=cmd, status="failed" - ) - - -def main(): - parser = argparse.ArgumentParser(description="????????") - parser.add_argument("--prefix", required=True, help="?????? Multi.SPL") - parser.add_argument("--workspace", required=True) - parser.add_argument("--input-dir") - parser.add_argument("--ref-species", default=None, help="???????? config.xlsx ??") - parser.add_argument("--target-species", default=None, help="???????? config.xlsx ??") - parser.add_argument("--ref-protein", help="?????????") - parser.add_argument("--target-protein", help="?????????") - parser.add_argument("--cpu", type=int, default=8) - parser.add_argument("--log-file", help="??????????????") - parser.add_argument("--workspace-root", help="??????????manifest?") - args = parser.parse_args() - - start_time = datetime.now() - run_id = f"interspecies_synteny_{start_time.strftime('%Y%m%d_%H%M%S')}" - workspace = Path(args.workspace) - job_id = workspace.name - input_dir = Path(args.input_dir) if args.input_dir else workspace.parent.parent / "input" / job_id - workspace_root = Path(args.workspace_root) if args.workspace_root else workspace.parent - cmd = sys.argv - - print("=" * 60) - print("Interspecies synteny analysis") - print("=" * 60) - - try: - paths = prepare_directories(workspace) - tool_versions = get_tool_versions() - discovery = discover_group_inputs(workspace, input_dir, args.prefix, args) - target_group = "Target_combined" - ref_group = "Ref_combined" - print(f"Target species: {', '.join(discovery['target_species']) or 'NONE'}") - print(f"Reference species: {', '.join(discovery['ref_species']) or 'NONE'}") - print(f"Target family protein: {discovery['target_family_protein']}") - print(f"Ref family protein: {discovery['ref_family_protein']}") - - if not discovery["target_family_protein"] or not Path(discovery["target_family_protein"]).exists(): - raise RuntimeError(f"Target family protein missing: {discovery['target_family_protein']}") - if not discovery["ref_family_protein"] or not Path(discovery["ref_family_protein"]).exists(): - raise RuntimeError("Reference family protein missing; expected Ref_combined..pep.fasta or Ref species pep files") - - artifacts = {} - metrics = { - "target_species": discovery["target_species"], - "ref_species": discovery["ref_species"], - "available_target_species": discovery.get("available_target_species", discovery["target_species"]), - "available_ref_species": discovery.get("available_ref_species", discovery["ref_species"]), - "selected_target_species": discovery.get("selected_target_species", discovery["target_species"]), - "selected_ref_species": discovery.get("selected_ref_species", discovery["ref_species"]), - "tree_success": False, - "synteny_success": False, - "synteny_skipped_reason": "", - } - - print("\nMerging family proteins for tree...") - merged_fasta = paths["analysis_dir"] / f"{args.prefix}.target_ref.family.pep.fa" - merge_result = merge_protein_sequences(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), merged_fasta) - if not merge_result.get("success"): - raise RuntimeError(f"Failed to merge family proteins: {merge_result.get('stderr')}") - metrics["target_family_count"] = merge_result.get("target_count") - metrics["ref_family_count"] = merge_result.get("ref_count") - - print("\nBuilding interspecies tree...") - tree_result = build_phylogenetic_tree(merged_fasta, paths["analysis_dir"], args.prefix, args.cpu) - if not tree_result.get("tree_file"): - raise RuntimeError(f"Tree construction failed: {tree_result.get('error', 'unknown error')}") - final_tree = paths["final_dir"] / f"{args.prefix}.interspecies_tree.nwk" - shutil.copy(Path(tree_result["tree_file"]), final_tree) - metrics["tree_success"] = True - add_artifact(artifacts, "tree_nwk", final_tree, f"final/{args.prefix}.interspecies_tree.nwk", "tree_nwk", "tree") - - gene_species_map = paths["analysis_dir"] / f"{args.prefix}.gene_to_species.tsv" - write_gene_species_map(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), discovery["target_species"], discovery["ref_species"], gene_species_map, collect_species_id_sets(discovery["groups"])) - add_artifact(artifacts, "gene_to_species", gene_species_map, f"interspecies_synteny/analysis/{args.prefix}.gene_to_species.tsv", "gene_to_species", "tsv") - - print("\nPlotting interspecies tree...") - target_subfamily_file = input_dir / f"{args.prefix}.geneID.subfamily" - if not target_subfamily_file.exists(): - target_subfamily_file = workspace / "final" / f"{args.prefix}.geneID.subfamily" - ref_subfamily_file = input_dir / f"SPL_{discovery['ref_species'][0]}.subfamily" if discovery["ref_species"] else input_dir / "SPL_ref.subfamily" - if not ref_subfamily_file.exists(): - candidates = sorted(input_dir.glob("SPL_*.subfamily")) + sorted(input_dir.glob("*.SPL.subfamily")) - ref_subfamily_file = candidates[0] if candidates else ref_subfamily_file - tree_plot_result = plot_interspecies_tree( - final_tree, - paths["final_dir"], - args.prefix, - target_group, - ref_group, - target_subfamily_file if target_subfamily_file.exists() else None, - ref_subfamily_file if ref_subfamily_file.exists() else None, - gene_species_map, - ) - if tree_plot_result.get("pdf_file"): - add_artifact(artifacts, "tree_pdf", tree_plot_result["pdf_file"], f"final/{args.prefix}.interspecies_tree.pdf", "tree_pdf", "plot") - if tree_plot_result.get("png_file"): - add_artifact(artifacts, "tree_png", tree_plot_result["png_file"], f"final/{args.prefix}.interspecies_tree.png", "tree_png", "plot") - - synteny_plot_result = {"success": False} - pair_results: List[Dict[str, Any]] = [] - species_position_results: Dict[str, Dict[str, Any]] = {} - - missing = [] - if not discovery["target_all_peps"]: - missing.append("target all protein") - if not discovery["ref_all_peps"]: - missing.append("reference all protein") - if not discovery["target_gffs"]: - missing.append("target GFF") - if not discovery["ref_gffs"]: - missing.append("reference GFF") - - if not missing: - print("\nPreparing per-species GFF/BED files for pairwise MCScanX...") - all_species = discovery["target_species"] + discovery["ref_species"] - for species in all_species: - pep = discovery["target_all_peps"].get(species) or discovery["ref_all_peps"].get(species) - gff = discovery["target_gffs"].get(species) or discovery["ref_gffs"].get(species) - if not pep or not gff: - species_position_results[species] = {"success": False, "error": "missing pep or gff"} - continue - result = build_species_position_files(species, Path(gff), Path(pep), paths["analysis_dir"]) - species_position_results[species] = result - if result.get("success"): - add_artifact(artifacts, f"{species}_mcscanx_gff", result["gff"], f"interspecies_synteny/analysis/{species}.gff", "mcscanx_gff", "gff") - add_artifact(artifacts, f"{species}_bed", result["bed"], f"interspecies_synteny/analysis/{species}.bed", "jcvi_bed", "bed") - print(f" {species}: {result.get('gene_count', 0)} matched genes") - - print("\nRunning pairwise target-ref MCScanX...") - for target_species in discovery["target_species"]: - for ref_species in discovery["ref_species"]: - pair_key = f"{target_species}.{ref_species}" - pair = {"pair": pair_key, "target_species": target_species, "ref_species": ref_species, "success": False} - target_ready = species_position_results.get(target_species, {}).get("success") - ref_ready = species_position_results.get(ref_species, {}).get("success") - target_pep = discovery["target_all_peps"].get(target_species) - ref_pep = discovery["ref_all_peps"].get(ref_species) - if not (target_ready and ref_ready and target_pep and ref_pep): - pair["error"] = "missing matched GFF/BED or protein FASTA" - pair_results.append(pair) - print(f" {pair_key}: skipped ({pair['error']})") - continue - blast_result = run_reciprocal_blast(Path(target_pep), Path(ref_pep), paths["analysis_dir"], target_species, ref_species, args.cpu) - pair["blast_success"] = bool(blast_result.get("success")) - if blast_result.get("output_file"): - pair["blast_file"] = str(blast_result["output_file"]) - add_artifact(artifacts, f"{pair_key}_blast", blast_result["output_file"], f"interspecies_synteny/analysis/{pair_key}.blast", "blast", "data") - if not blast_result.get("success"): - pair["error"] = "BLAST failed or produced no hits" - pair_results.append(pair) - print(f" {pair_key}: {pair['error']}") - continue - mcscanx_result = run_mcscanx_interspecies(paths["analysis_dir"], target_species, ref_species) - pair_gff = paths["analysis_dir"] / f"{pair_key}.gff" - if pair_gff.exists(): - pair["pair_gff"] = str(pair_gff) - add_artifact(artifacts, f"{pair_key}_gff", pair_gff, f"interspecies_synteny/analysis/{pair_key}.gff", "pair_mcscanx_gff", "gff") - pair["mcscanx_success"] = bool(mcscanx_result.get("success")) - if mcscanx_result.get("collinearity_file"): - pair["collinearity_file"] = str(mcscanx_result["collinearity_file"]) - add_artifact(artifacts, f"{pair_key}_collinearity", mcscanx_result["collinearity_file"], f"interspecies_synteny/analysis/{pair_key}.collinearity", "collinearity", "data") - if not mcscanx_result.get("success"): - pair["error"] = "MCScanX did not produce collinearity" - pair_results.append(pair) - print(f" {pair_key}: {pair['error']}") - continue - simple_result = convert_collinearity_to_simple(paths["analysis_dir"], Path(mcscanx_result["collinearity_file"]), target_species, ref_species) - if not simple_result.get("success") and blast_result.get("output_file"): - fallback_result = convert_blast_to_family_simple(paths["analysis_dir"], Path(blast_result["output_file"]), target_species, ref_species) - if fallback_result.get("success"): - fallback_result["note"] = "MCScanX produced no whole-genome blocks; using family-only BLAST links for JCVI visualization" - simple_result = fallback_result - pair.update({k: str(v) if isinstance(v, Path) else v for k, v in simple_result.items()}) - pair["success"] = bool(simple_result.get("success")) - if simple_result.get("anchors_file"): - add_artifact(artifacts, f"{pair_key}_anchors", simple_result["anchors_file"], f"interspecies_synteny/analysis/{pair_key}.anchors", "anchors", "data") - if simple_result.get("simple_file"): - add_artifact(artifacts, f"{pair_key}_anchors_simple", simple_result["simple_file"], f"interspecies_synteny/analysis/{pair_key}.anchors.simple", "anchors_simple", "data") - pair_results.append(pair) - print(f" {pair_key}: {'success' if pair['success'] else 'anchors conversion failed'}") - - family_ids = collect_family_ids(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), workspace / "final" / f"{args.prefix}.final_members.id") - synteny_plot_result = plot_multi_species_synteny( - paths["analysis_dir"], paths["final_dir"], args.prefix, - discovery["target_species"] + discovery["ref_species"], pair_results, family_ids, - ) - successful_pairs = [p for p in pair_results if p.get("success")] - metrics["synteny_success"] = bool(synteny_plot_result.get("success")) and len(successful_pairs) == len(pair_results) and bool(pair_results) - if not successful_pairs: - metrics["synteny_skipped_reason"] = synteny_plot_result.get("error") or "No target-ref pair produced JCVI anchors" - else: - metrics["synteny_skipped_reason"] = "Missing full data for MCScanX: " + ", ".join(missing) - print(metrics["synteny_skipped_reason"]) - - metrics["species_position_results"] = { - sp: {k: str(v) if isinstance(v, Path) else v for k, v in result.items()} - for sp, result in species_position_results.items() - } - metrics["pairwise_synteny"] = pair_results - metrics["pair_success_count"] = sum(1 for p in pair_results if p.get("success")) - metrics["pair_total_count"] = len(pair_results) - - if synteny_plot_result.get("pdf_file"): - add_artifact(artifacts, "synteny_pdf", synteny_plot_result["pdf_file"], f"final/{args.prefix}.interspecies_synteny.pdf", "synteny_pdf", "plot") - if synteny_plot_result.get("png_file"): - add_artifact(artifacts, "synteny_png", synteny_plot_result["png_file"], f"final/{args.prefix}.interspecies_synteny.png", "synteny_png", "plot") - if synteny_plot_result.get("marked_pdf_file"): - add_artifact(artifacts, "synteny_marked_pdf", synteny_plot_result["marked_pdf_file"], f"final/{args.prefix}.interspecies_synteny.marked.pdf", "synteny_marked_pdf", "plot") - if synteny_plot_result.get("marked_png_file"): - add_artifact(artifacts, "synteny_marked_png", synteny_plot_result["marked_png_file"], f"final/{args.prefix}.interspecies_synteny.marked.png", "synteny_marked_png", "plot") - if synteny_plot_result.get("seqids_file"): - add_artifact(artifacts, "jcvi_seqids", synteny_plot_result["seqids_file"], f"interspecies_synteny/analysis/{args.prefix}.seqids", "jcvi_seqids", "data") - if synteny_plot_result.get("layout_file"): - add_artifact(artifacts, "jcvi_layout", synteny_plot_result["layout_file"], f"interspecies_synteny/analysis/{args.prefix}.layout", "jcvi_layout", "data") - - end_time = datetime.now() - status = "success" if metrics["tree_success"] and metrics["synteny_success"] else "partial_success" - parameters = { - "prefix": args.prefix, - "gene_family": discovery["gene_family"], - "target_group": target_group, - "ref_group": ref_group, - "target_species": discovery["target_species"], - "ref_species": discovery["ref_species"], - "available_target_species": discovery.get("available_target_species", discovery["target_species"]), - "available_ref_species": discovery.get("available_ref_species", discovery["ref_species"]), - "selected_target_species": discovery.get("selected_target_species", discovery["target_species"]), - "selected_ref_species": discovery.get("selected_ref_species", discovery["ref_species"]), - "target_protein_spl": str(discovery["target_family_protein"]), - "ref_protein_spl": str(discovery["ref_family_protein"]), - "input_dir": str(input_dir), - "cpu": args.cpu, - } - report_path = write_report(workspace, args.prefix, metrics, parameters, tool_versions, start_time, end_time, (end_time - start_time).total_seconds(), cmd) - add_artifact(artifacts, "report", report_path, f"final/{args.prefix}.interspecies_synteny_report.md", "report", "report") - manifest_path = write_manifest(workspace, job_id, run_id, args.prefix, start_time, end_time, artifacts, metrics, parameters, tool_versions, workspace_root, input_dir, cmd, status=status) - add_artifact(artifacts, "manifest", manifest_path, f"final/{args.prefix}.interspecies_synteny.manifest.json", "manifest", "json") - write_manifest(workspace, job_id, run_id, args.prefix, start_time, end_time, artifacts, metrics, parameters, tool_versions, workspace_root, input_dir, cmd, status=status) - print(f"Interspecies synteny finished with status: {status}") - print(f"Manifest: {manifest_path}") - return 0 if metrics["tree_success"] else 1 - except Exception as exc: - print(f"Interspecies synteny failed: {exc}") - try: - write_failure_manifest(workspace, job_id, run_id, args.prefix, start_time, workspace_root, input_dir, cmd, str(exc)) - except Exception as manifest_exc: - print(f"Failed to write failure manifest: {manifest_exc}") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bioflow_kernel/cli/run_interspecies_synteny.py.bak_family_allpep_20260504_193257 b/bioflow_kernel/cli/run_interspecies_synteny.py.bak_family_allpep_20260504_193257 deleted file mode 100755 index 1e5cc23..0000000 --- a/bioflow_kernel/cli/run_interspecies_synteny.py.bak_family_allpep_20260504_193257 +++ /dev/null @@ -1,1518 +0,0 @@ -#!/usr/bin/env python3 -""" -Interspecies Synteny阶段：物种间共线性分析（完整版） - -输入： - - 参考物种蛋白序列 (final/{prefix}.final_members.pep.fa) - - 参考物种GFF3 - - 目标物种蛋白序列 - - 目标物种GFF3 - -输出： - - final/{prefix}.interspecies_tree.nwk (物种间进化树) - - final/{prefix}.interspecies_tree.pdf/png (进化树图) - - final/{prefix}.interspecies_synteny.pdf/png (共线性图) - - final/{prefix}.interspecies_synteny_report.md (分析报告) - - final/{prefix}.interspecies_synteny.manifest.json (机器清单) -""" - -import os -import argparse -import json -import hashlib -import subprocess -import sys -import time -import shutil -import re -from datetime import datetime -from pathlib import Path -from typing import Dict, List, Any, Optional, Tuple - - -# ??conda??bin???PATH -CONDA_BIN = "/home/xia-lab/miniconda3/envs/test4py311/bin" -if CONDA_BIN not in os.environ.get("PATH", ""): - os.environ["PATH"] = CONDA_BIN + ":" + os.environ.get("PATH", "") -def pdf_to_png(pdf_path: Path, png_path: Path, dpi: int = 300) -> bool: - """将PDF转换为PNG""" - try: - # 方法1: 使用pdftoppm (poppler-utils) - cmd = ["pdftoppm", "-png", "-r", str(dpi), "-singlefile", - str(pdf_path), str(png_path.with_suffix(''))] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - if result.returncode == 0: - return True - except Exception: - pass - - try: - # 方法2: 使用convert (ImageMagick) - cmd = ["convert", "-density", str(dpi), str(pdf_path), "-quality", "100", str(png_path)] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - if result.returncode == 0: - return True - except Exception: - pass - - return False - - -def compute_file_hash(filepath: Path) -> str: - """计算文件的SHA256哈希值""" - sha256_hash = hashlib.sha256() - try: - with open(filepath, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - except Exception: - return "" - - -def get_tool_versions() -> Dict[str, str]: - """获取所需工具版本信息""" - versions = {} - - for tool, cmd in [ - ("mafft", ["mafft", "--version"]), - ("iqtree", ["iqtree", "--version"]), - ("blastp", ["blastp", "-version"]), - ("MCScanX", ["MCScanX", "-h"]), - ]: - try: - result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=5) - versions[tool] = "available" if result.returncode in [0, 1] else "not_found" - except Exception: - versions[tool] = "not_found" - - return versions - - -def prepare_directories(workspace: Path) -> Dict[str, Path]: - """创建目录结构""" - paths = { - "analysis_dir": workspace / "interspecies_synteny" / "analysis", - "final_dir": workspace / "final", - "logs_dir": workspace / "logs", - } - for name, path in paths.items(): - path.mkdir(parents=True, exist_ok=True) - return paths - - -def merge_protein_sequences( - target_protein: Path, - ref_protein: Path, - output_fasta: Path -) -> Dict[str, Any]: - """Merge target and reference family proteins for the interspecies tree.""" - print("Merging family protein sequences...") - - try: - with open(output_fasta, 'w') as outfile: - with open(target_protein, 'r') as f: - outfile.write(f.read().rstrip() + "\n") - with open(ref_protein, 'r') as f: - outfile.write(f.read().rstrip() + "\n") - - target_count = sum(1 for line in open(target_protein) if line.startswith('>')) - ref_count = sum(1 for line in open(ref_protein) if line.startswith('>')) - - print(f" merged target/ref family proteins: {target_count} + {ref_count} = {target_count + ref_count}") - - return { - "success": True, - "output_file": output_fasta, - "target_count": target_count, - "ref_count": ref_count - } - except Exception as e: - return {"success": False, "stderr": str(e)} - - -def build_phylogenetic_tree( - merged_fasta: Path, - output_dir: Path, - prefix: str, - cpu: int = 8 -) -> Dict[str, Any]: - """构建系统发育树 (MAFFT + IQ-TREE)""" - print(f"构建进化树...") - - # MAFFT比对 - msa_file = output_dir / f"{prefix}_interspecies.mafft.msa.fa" - mafft_cmd = ["mafft", "--auto", "--thread", str(cpu), str(merged_fasta)] - - try: - with open(msa_file, "w") as f_out: - result = subprocess.run(mafft_cmd, stdout=f_out, stderr=subprocess.PIPE, - text=True, check=False) - - if result.returncode != 0: - return {"success": False, "error": f"MAFFT失败: {result.stderr}"} - - # IQ-TREE建树 - base_name = output_dir / f"{prefix}_interspecies.iqtree" - iqtree_cmd = [ - "iqtree", "-s", str(msa_file), "-m", "MFP", "-bb", "1000", - "-nt", str(cpu), "-pre", str(base_name) - ] - - result = subprocess.run(iqtree_cmd, capture_output=True, text=True, check=False) - - # IQ-TREE 生成的 treefile 是 {prefix}.iqtree.treefile，不是 {prefix}.treefile - tree_file = Path(f"{base_name}.treefile") - - return { - "success": tree_file.exists(), - "tree_file": tree_file if tree_file.exists() else None, - "msa_file": msa_file - } - except Exception as e: - return {"success": False, "error": str(e)} - - -def filter_gff_files( - ref_gff3: Path, - target_gff3: Path, - output_dir: Path, - ref_species: str, - target_species: str -) -> Dict[str, Any]: - """过滤GFF3文件 - 提取基因ID（去掉转录本版本后缀如 .1, .2）""" - print(f"过滤GFF3文件...") - - import re - - results = {} - for gff3_file, species, prefix in [ - (ref_gff3, ref_species, "ref"), - (target_gff3, target_species, "target") - ]: - output_gff = output_dir / f"{species}.gff" - - try: - with open(gff3_file, 'r') as f: - lines = f.readlines() - - count = 0 - seen_genes = set() # 去重 - - with open(output_gff, 'w') as f: - for line in lines: - parts = line.strip().split('\t') - if len(parts) >= 9 and parts[2] == "mRNA": - attr = parts[8] - # 提取ID（如 ID=AT2G47070.1），保留版本号与BLAST结果匹配 - gene_id = attr.split(';')[0].replace('ID=', '') - - # 处理染色体名称：去掉 "Chr" 前缀（Ath用1,2,3而非Chr1,Chr2） - chr_name = parts[0] - if chr_name.startswith('Chr'): - chr_name = chr_name[3:] # Chr1 -> 1 - - # 去重：一个基因可能有多个转录本 - if gene_id not in seen_genes: - seen_genes.add(gene_id) - f.write(f"{chr_name}\t{gene_id}\t{parts[3]}\t{parts[4]}\n") - count += 1 - - print(f" {species}: {count} 个基因（去重后）") - results[prefix] = {"success": True, "output_file": output_gff, "gene_count": count} - except Exception as e: - results[prefix] = {"success": False, "stderr": str(e)} - - if results.get("ref", {}).get("success") and results.get("target", {}).get("success"): - return { - "success": True, - "ref_gff": results["ref"]["output_file"], - "target_gff": results["target"]["output_file"] - } - return {"success": False} - - -def run_reciprocal_blast( - ref_protein: Path, - target_protein: Path, - output_dir: Path, - ref_species: str, - target_species: str, - cpu: int = 8 -) -> Dict[str, Any]: - """双向BLAST比对 - 结果中基因名去掉版本后缀以匹配GFF""" - import re - print(f"双向BLAST比对...") - - # 构建数据库 - for protein_file in [ref_protein, target_protein]: - subprocess.run(["makeblastdb", "-in", str(protein_file), "-dbtype", "prot"], - capture_output=True, check=False) - - # 双向blastp - blast1_out = output_dir / f"tmp.{ref_species}_{target_species}.blast" - subprocess.run([ - "blastp", "-query", str(ref_protein), "-db", str(target_protein), - "-outfmt", "6", "-out", str(blast1_out), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ], capture_output=True, check=False) - - blast2_out = output_dir / f"tmp.{target_species}_{ref_species}.blast" - subprocess.run([ - "blastp", "-query", str(target_protein), "-db", str(ref_protein), - "-outfmt", "6", "-out", str(blast2_out), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ], capture_output=True, check=False) - - # 合并BLAST结果（保留版本号与GFF匹配） - merged_blast = output_dir / f"{ref_species}_{target_species}.blast" - with open(merged_blast, 'w') as outfile: - for blast_file in [blast1_out, blast2_out]: - if blast_file.exists(): - with open(blast_file, 'r') as infile: - for line in infile: - outfile.write(line) # 直接写入，保留版本号 - - return {"success": True, "output_file": merged_blast} - - -def run_mcscanx_interspecies( - work_dir: Path, - ref_species: str, - target_species: str -) -> Dict[str, Any]: - """MCScanX种间共线性 - 需要合并的GFF文件""" - print(f"MCScanX种间共线性...") - - prefix = f"{ref_species}_{target_species}" - - # MCScanX 需要合并的 GFF 文件 (prefix.gff) - merged_gff = work_dir / f"{prefix}.gff" - ref_gff = work_dir / f"{ref_species}.gff" - target_gff = work_dir / f"{target_species}.gff" - - if not merged_gff.exists() and ref_gff.exists() and target_gff.exists(): - print(f" 合并 GFF 文件: {ref_gff.name} + {target_gff.name} -> {merged_gff.name}") - with open(merged_gff, 'w') as outfile: - for gff_file in [target_gff, ref_gff]: # 目标物种在前，参考物种在后 - with open(gff_file, 'r') as infile: - outfile.write(infile.read()) - - cmd = ["MCScanX", f"./{prefix}"] - - result = subprocess.run(cmd, capture_output=True, text=True, - cwd=str(work_dir), check=False) - - collinearity_file = work_dir / f"{prefix}.collinearity" - - return { - "success": collinearity_file.exists(), - "collinearity_file": collinearity_file if collinearity_file.exists() else None - } - - - - -def fasta_ids(fasta_file: Path) -> List[str]: - ids = [] - with open(fasta_file, "r") as f: - for line in f: - if line.startswith(">"): - ids.append(line[1:].strip().split()[0]) - return ids - - -def display_species_code(species: str, gene_id: str = "") -> str: - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return species - - -def species_for_gene(gene_id: str, species_list: List[str], default_species: str) -> str: - for species in sorted(species_list, key=len, reverse=True): - if gene_id.lower().startswith(species.lower()): - return display_species_code(species, gene_id) - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return display_species_code(default_species, gene_id) - - -def write_gene_species_map(target_fasta: Path, ref_fasta: Path, target_species: List[str], ref_species: List[str], output_file: Path) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - default_target = target_species[0] if target_species else "Target" - default_ref = ref_species[0] if ref_species else "Ref" - with open(output_file, "w") as out: - out.write("gene_id\tspecies\tgroup\n") - for gene_id in fasta_ids(target_fasta): - out.write(f"{gene_id}\t{species_for_gene(gene_id, target_species, default_target)}\ttarget\n") - for gene_id in fasta_ids(ref_fasta): - out.write(f"{gene_id}\t{species_for_gene(gene_id, ref_species, default_ref)}\tref\n") - return output_file -def normalize_seq_id(text: str) -> str: - """Normalize FASTA/GFF IDs while preserving biologically meaningful versions.""" - value = (text or "").strip() - if value.startswith(">"): - value = value[1:] - value = value.split()[0] if value.split() else value - if "|" in value: - value = value.split("|", 1)[0] - for prefix in ("gene:", "transcript:", "CDS:", "rna-", "gene-", "cds-"): - if value.startswith(prefix): - value = value[len(prefix):] - return value.strip().strip('"') - - -def id_variants(gene_id: str) -> set: - norm = normalize_seq_id(gene_id) - variants = {norm} - for prefix in ("gene:", "transcript:", "CDS:", "rna-", "gene-", "cds-"): - variants.add(normalize_seq_id(prefix + norm)) - if "|" in gene_id: - variants.add(normalize_seq_id(gene_id.split("|", 1)[0])) - return {v for v in variants if v} - - -def fasta_ids(fasta_file: Path) -> List[str]: - ids = [] - with open(fasta_file, "r") as f: - for line in f: - if line.startswith(">"): - seq_id = normalize_seq_id(line[1:]) - if seq_id: - ids.append(seq_id) - return ids - - -def fasta_id_set(fasta_file: Path) -> set: - ids = set() - for seq_id in fasta_ids(fasta_file): - ids.update(id_variants(seq_id)) - return ids - - -def display_species_code(species: str, gene_id: str = "") -> str: - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return species - - -def species_for_gene(gene_id: str, species_list: List[str], default_species: str, species_id_sets: Optional[Dict[str, set]] = None) -> str: - variants = id_variants(gene_id) - if species_id_sets: - for species, ids in species_id_sets.items(): - if variants & ids: - return display_species_code(species, gene_id) - for species in sorted(species_list, key=len, reverse=True): - if normalize_seq_id(gene_id).lower().startswith(species.lower()): - return display_species_code(species, gene_id) - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return display_species_code(default_species, gene_id) - - -def collect_species_id_sets(groups: Dict[str, Any]) -> Dict[str, set]: - species_id_sets: Dict[str, set] = {} - for group_name in ("target", "ref"): - for species, item in groups.get(group_name, {}).items(): - pep = item.get("resolved", {}).get("pep") - if pep and Path(pep).exists(): - species_id_sets[species] = fasta_id_set(Path(pep)) - return species_id_sets - - -def write_gene_species_map( - target_fasta: Path, - ref_fasta: Path, - target_species: List[str], - ref_species: List[str], - output_file: Path, - species_id_sets: Optional[Dict[str, set]] = None, -) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - default_target = target_species[0] if target_species else "Target" - default_ref = ref_species[0] if ref_species else "Ref" - with open(output_file, "w") as out: - out.write("gene_id\tspecies\tgroup\n") - for gene_id in fasta_ids(target_fasta): - species = species_for_gene(gene_id, target_species, default_target, species_id_sets) - out.write(f"{gene_id}\t{species}\ttarget\n") - for gene_id in fasta_ids(ref_fasta): - species = species_for_gene(gene_id, ref_species, default_ref, species_id_sets) - out.write(f"{gene_id}\t{species}\tref\n") - return output_file - - -def run_reciprocal_blast( - target_protein: Path, - ref_protein: Path, - output_dir: Path, - target_species: str, - ref_species: str, - cpu: int = 8 -) -> Dict[str, Any]: - """Run pairwise reciprocal BLASTP for one target-ref species pair.""" - print(f"Reciprocal BLASTP: {target_species} vs {ref_species}") - - pair_prefix = f"{target_species}.{ref_species}" - target_db = output_dir / f"{target_species}.blastdb" - ref_db = output_dir / f"{ref_species}.blastdb" - commands = [] - - for protein_file, db_prefix in [(target_protein, target_db), (ref_protein, ref_db)]: - cmd = ["makeblastdb", "-in", str(protein_file), "-dbtype", "prot", "-out", str(db_prefix)] - commands.append(" ".join(cmd)) - result = subprocess.run(cmd, capture_output=True, text=True, check=False) - if result.returncode != 0: - return {"success": False, "stderr": result.stderr, "commands": commands} - - target_to_ref = output_dir / f"tmp.{target_species}_{ref_species}.blast" - ref_to_target = output_dir / f"tmp.{ref_species}_{target_species}.blast" - for query, db_prefix, out_file in [(target_protein, ref_db, target_to_ref), (ref_protein, target_db, ref_to_target)]: - cmd = [ - "blastp", "-query", str(query), "-db", str(db_prefix), - "-outfmt", "6", "-out", str(out_file), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ] - commands.append(" ".join(cmd)) - result = subprocess.run(cmd, capture_output=True, text=True, check=False) - if result.returncode != 0: - return {"success": False, "stderr": result.stderr, "commands": commands} - - merged_blast = output_dir / f"{pair_prefix}.blast" - with open(merged_blast, 'w') as outfile: - for blast_file in [target_to_ref, ref_to_target]: - if blast_file.exists(): - with open(blast_file, 'r') as infile: - for line in infile: - if line.strip(): - outfile.write(line) - - return {"success": merged_blast.exists() and merged_blast.stat().st_size > 0, "output_file": merged_blast, "commands": commands} - - -def run_mcscanx_interspecies(work_dir: Path, target_species: str, ref_species: str) -> Dict[str, Any]: - """Run MCScanX for one target-ref pair.""" - pair_prefix = f"{target_species}.{ref_species}" - print(f"MCScanX pair: {pair_prefix}") - - merged_gff = work_dir / f"{pair_prefix}.gff" - target_gff = work_dir / f"{target_species}.gff" - ref_gff = work_dir / f"{ref_species}.gff" - - if target_gff.exists() and ref_gff.exists(): - with open(merged_gff, 'w') as outfile: - for gff_file in [target_gff, ref_gff]: - with open(gff_file, 'r') as infile: - outfile.write(infile.read().rstrip() + "\n") - else: - return {"success": False, "error": f"Missing simplified GFF for {pair_prefix}"} - - result = subprocess.run(["MCScanX", f"./{pair_prefix}"], capture_output=True, text=True, cwd=str(work_dir), check=False) - collinearity_file = work_dir / f"{pair_prefix}.collinearity" - return { - "success": collinearity_file.exists() and collinearity_file.stat().st_size > 0, - "collinearity_file": collinearity_file if collinearity_file.exists() else None, - "stdout": result.stdout, - "stderr": result.stderr, - "returncode": result.returncode, - } - - -def plot_interspecies_tree( - tree_file: Path, - output_dir: Path, - prefix: str, - target_species: str, - ref_species: str, - target_subfamily_file: Optional[Path] = None, - ref_subfamily_file: Optional[Path] = None, - gene_species_map: Optional[Path] = None -) -> Dict[str, Any]: - """绘制物种间进化树图 - 使用专用的 interspecies_tree.R 脚本""" - print(f"绘制进化树图...") - - # 查找专用的种间进化树R脚本 - r_script = Path("scripts/interspecies_tree.R") - if not r_script.exists(): - alt_script = Path("/home/xia-lab/wutx/00test/bbav2/scripts/interspecies_tree.R") - if alt_script.exists(): - r_script = alt_script - else: - return {"success": False, "error": f"R脚本不存在: {r_script}"} - - try: - # 使用专用的 interspecies_tree.R 脚本，传递物种参数和亚家族文件 - # 使用conda环境的Rscript - conda_rscript = "/home/xia-lab/miniconda3/envs/test4py311/bin/Rscript" - cmd = [ - conda_rscript, str(r_script), - str(tree_file), - str(output_dir), - target_species, - ref_species - ] - - # 添加亚家族文件参数（如果存在） - if target_subfamily_file and target_subfamily_file.exists(): - cmd.append(str(target_subfamily_file)) - print(f" 目标物种亚家族文件: {target_subfamily_file}") - else: - cmd.append("") # 空字符串表示无文件 - - if ref_subfamily_file and ref_subfamily_file.exists(): - cmd.append(str(ref_subfamily_file)) - print(f" 参考物种亚家族文件: {ref_subfamily_file}") - else: - cmd.append("") # 空字符串表示无文件 - if gene_species_map and gene_species_map.exists(): - cmd.append(str(gene_species_map)) - print(f" Gene-to-species map: {gene_species_map}") - else: - cmd.append("") - result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=120) - - # 调试输出 - print(f" R脚本返回码: {result.returncode}") - if result.stdout: - print(f" R脚本输出: {result.stdout[:500]}") - if result.stderr: - print(f" R脚本错误: {result.stderr[:500]}") - - # 专用脚本输出固定为 interspecies_tree.pdf/png - output_pdf = output_dir / "interspecies_tree.pdf" - output_png = output_dir / "interspecies_tree.png" - - # 重命名为规范名称 - final_pdf = output_dir / f"{prefix}.interspecies_tree.pdf" - final_png = output_dir / f"{prefix}.interspecies_tree.png" - - if output_pdf.exists(): - shutil.move(output_pdf, final_pdf) - print(f" 进化树PDF: {final_pdf}") - if output_png.exists(): - shutil.move(output_png, final_png) - print(f" 进化树PNG: {final_png}") - - # PDF转PNG（如果没有生成PNG） - if final_pdf.exists() and not final_png.exists(): - png_from_pdf = output_dir / "interspecies_tree.png" - if pdf_to_png(final_pdf, png_from_pdf): - final_png = output_dir / f"{prefix}.interspecies_tree.png" - shutil.move(png_from_pdf, final_png) - print(f" PDF转PNG: {final_png}") - - return { - "success": final_pdf.exists() or final_png.exists(), - "pdf_file": final_pdf if final_pdf.exists() else None, - "png_file": final_png if final_png.exists() else None, - "stdout": result.stdout, - "stderr": result.stderr - } - except Exception as e: - return {"success": False, "error": str(e)} - - -def convert_blast_to_family_simple( - work_dir: Path, - blast_file: Path, - target_species: str, - ref_species: str, -) -> Dict[str, Any]: - """Fallback: convert pairwise BLAST hits directly to JCVI anchors/simple. - - This is not a strict MCScanX collinearity block. It is used when the ref side - only has family proteins, so MCScanX cannot satisfy whole-genome block rules. - """ - pair_prefix = f"{target_species}.{ref_species}" - target_bed = work_dir / f"{target_species}.bed" - ref_bed = work_dir / f"{ref_species}.bed" - anchors_file = work_dir / f"{pair_prefix}.anchors" - simple_file = work_dir / f"{pair_prefix}.anchors.simple" - - valid_target = {line.rstrip("\n").split("\t")[3] for line in open(target_bed) if len(line.rstrip("\n").split("\t")) >= 4} - valid_ref = {line.rstrip("\n").split("\t")[3] for line in open(ref_bed) if len(line.rstrip("\n").split("\t")) >= 4} - seen = set() - links = [] - if not blast_file.exists(): - return {"success": False, "error": "BLAST file missing for family-only fallback"} - with open(blast_file, "r") as f: - for line in f: - if not line.strip() or line.startswith("#"): - continue - parts = line.rstrip("\n").split("\t") - if len(parts) < 2: - continue - a = normalize_seq_id(parts[0]) - b = normalize_seq_id(parts[1]) - target_gene = ref_gene = None - if a in valid_target and b in valid_ref: - target_gene, ref_gene = a, b - elif b in valid_target and a in valid_ref: - target_gene, ref_gene = b, a - if not target_gene or not ref_gene: - continue - key = (target_gene, ref_gene) - if key in seen: - continue - seen.add(key) - score_text = parts[11] if len(parts) > 11 else "1000" - try: - score = str(int(float(score_text))) - except Exception: - score = "1000" - links.append((target_gene, ref_gene, score)) - if not links: - return {"success": False, "error": "No BLAST hits matched pair BED IDs for family-only fallback"} - with open(anchors_file, "w") as out: - out.write("##\n") - for target_gene, ref_gene, score in links: - out.write(f"{target_gene}\t{ref_gene}\t{score}\n") - with open(simple_file, "w") as out: - for target_gene, ref_gene, score in links: - out.write(f"{target_gene}\t{target_gene}\t{ref_gene}\t{ref_gene}\t{score}\t+\n") - return { - "success": True, - "anchors_file": anchors_file, - "simple_file": simple_file, - "anchor_count": len(links), - "fallback": "family_only_blast_links", - } - - -def convert_collinearity_to_simple( - work_dir: Path, - collinearity_file: Path, - target_species: str, - ref_species: str, -) -> Dict[str, Any]: - """Convert one MCScanX .collinearity file into JCVI anchors/simple files.""" - conda_python = "/home/xia-lab/miniconda3/envs/test4py311/bin/python" - pair_prefix = f"{target_species}.{ref_species}" - target_bed = work_dir / f"{target_species}.bed" - ref_bed = work_dir / f"{ref_species}.bed" - anchors_file = work_dir / f"{pair_prefix}.anchors" - anchors_new_file = work_dir / f"{pair_prefix}.anchors.new" - simple_file = work_dir / f"{pair_prefix}.anchors.simple" - - valid_target = {line.rstrip("\n").split("\t")[3] for line in open(target_bed) if len(line.rstrip("\n").split("\t")) >= 4} - valid_ref = {line.rstrip("\n").split("\t")[3] for line in open(ref_bed) if len(line.rstrip("\n").split("\t")) >= 4} - - current_block = [] - anchors = 0 - filtered = 0 - with open(collinearity_file, "r") as f_in, open(anchors_file, "w") as f_out: - for line in f_in: - if not line.strip(): - continue - if line.startswith("## Alignment"): - if current_block: - f_out.write("##\n") - f_out.writelines(current_block) - current_block = [] - continue - if line.startswith("#"): - continue - parts = line.strip().split("\t") - if len(parts) < 3: - continue - gene_a = normalize_seq_id(parts[1]) - gene_b = normalize_seq_id(parts[2]) - target_gene = ref_gene = None - if gene_a in valid_target and gene_b in valid_ref: - target_gene, ref_gene = gene_a, gene_b - elif gene_b in valid_target and gene_a in valid_ref: - target_gene, ref_gene = gene_b, gene_a - if target_gene and ref_gene: - current_block.append(f"{target_gene} {ref_gene} 1000\n") - anchors += 1 - else: - filtered += 1 - if current_block: - f_out.write("##\n") - f_out.writelines(current_block) - - if anchors == 0: - return {"success": False, "error": "No anchors matched pair BED IDs", "anchors_file": anchors_file, "filtered": filtered} - - cmd_screen = [ - conda_python, "-m", "jcvi.compara.synteny", "screen", - "--minspan=10", "--simple", str(anchors_file), str(anchors_new_file), - f"--qbed={target_bed.name}", f"--sbed={ref_bed.name}", - ] - result = subprocess.run(cmd_screen, capture_output=True, text=True, cwd=str(work_dir), check=False, timeout=180) - for candidate in (work_dir / f"{pair_prefix}.anchors.new.simple", work_dir / f"{pair_prefix}.new.simple"): - if candidate.exists(): - shutil.copy(candidate, simple_file) - break - if not simple_file.exists(): - shutil.copy(anchors_file, simple_file) - - return { - "success": simple_file.exists() and simple_file.stat().st_size > 0, - "anchors_file": anchors_file, - "anchors_new_file": anchors_new_file, - "simple_file": simple_file, - "anchor_count": anchors, - "filtered_count": filtered, - "screen_returncode": result.returncode, - "screen_stderr": result.stderr, - } - - -def _chr_sort_key(chrom: str): - cleaned = re.sub(r"^(chr|Chr|CHR)", "", chrom) - if cleaned.isdigit(): - return (0, int(cleaned)) - return (1, cleaned) - - -def _read_bed_chroms(bed_file: Path) -> List[str]: - skip = {"C", "M", "ChrC", "ChrM", "chrc", "chrm"} - chroms = set() - with open(bed_file, "r") as f: - for line in f: - parts = line.strip().split("\t") - if len(parts) >= 4 and parts[0] not in skip: - chroms.add(parts[0]) - return sorted(chroms, key=_chr_sort_key) - - -def _mark_simple_file(simple_file: Path, marked_file: Path, family_ids: set) -> bool: - if not simple_file.exists() or not family_ids: - return False - wrote = False - with open(simple_file, "r") as f_in, open(marked_file, "w") as f_out: - for line in f_in: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - f_out.write(line) - continue - parts = stripped.split() - normalized = {normalize_seq_id(p.lstrip("r*")) for p in parts[:4]} - if normalized & family_ids and not parts[0].startswith("r*"): - parts[0] = "r*" + parts[0] - wrote = True - f_out.write("\t".join(parts) + "\n") - else: - f_out.write(line) - return wrote - - -def plot_multi_species_synteny( - work_dir: Path, - output_dir: Path, - prefix: str, - species_order: List[str], - pair_results: List[Dict[str, Any]], - family_ids: set, -) -> Dict[str, Any]: - """Draw one multi-track JCVI karyotype figure from all successful target-ref pairs.""" - print("Plotting multi-species JCVI synteny...") - conda_python = "/home/xia-lab/miniconda3/envs/test4py311/bin/python" - usable_pairs = [p for p in pair_results if p.get("simple_file") and Path(p["simple_file"]).exists()] - if not usable_pairs: - return {"success": False, "error": "No pairwise anchors.simple files available for JCVI"} - - seqids_file = work_dir / f"{prefix}.seqids" - layout_file = work_dir / f"{prefix}.layout" - layout_mark_file = work_dir / f"{prefix}.layout_mark" - - species_with_bed = [] - with open(seqids_file, "w") as f: - for species in species_order: - bed = work_dir / f"{species}.bed" - if not bed.exists(): - continue - chroms = _read_bed_chroms(bed) - if not chroms: - continue - species_with_bed.append(species) - f.write(",".join(chroms) + "\n") - - if len(species_with_bed) < 2: - return {"success": False, "error": "Fewer than two species BED tracks are available"} - - index = {species: i for i, species in enumerate(species_with_bed)} - n = len(species_with_bed) - y_values = [round(0.85 - (0.7 * i / max(n - 1, 1)), 3) for i in range(n)] - palette = ["#d73027", "#1a9850", "#4575b4", "#984ea3", "#ff7f00", "#4daf4a", "#377eb8", "#a65628"] - - def write_layout(layout: Path, use_marked: bool) -> None: - with open(layout, "w") as f: - f.write("# y, xstart, xend, rotation, color, label, va, bed\n") - for i, species in enumerate(species_with_bed): - va = "top" if i == 0 else "bottom" - f.write(f"{y_values[i]}, .08, .92, 0, {palette[i % len(palette)]}, {species}, {va}, {species}.bed\n") - f.write("# edges\n") - for pair in usable_pairs: - target = pair["target_species"] - ref = pair["ref_species"] - if target not in index or ref not in index: - continue - edge = Path(pair.get("marked_simple_file") if use_marked and pair.get("marked_simple_file") else pair["simple_file"]).name - f.write(f"e, {index[target]}, {index[ref]}, {edge}\n") - - for pair in usable_pairs: - simple = Path(pair["simple_file"]) - marked = work_dir / f"{pair['target_species']}.{pair['ref_species']}.anchors.simple.mark" - if _mark_simple_file(simple, marked, family_ids): - pair["marked_simple_file"] = str(marked) - - write_layout(layout_file, use_marked=False) - write_layout(layout_mark_file, use_marked=True) - - def run_plot(layout: Path, final_pdf: Path, final_png: Path) -> Dict[str, Any]: - for stale in [work_dir / "karyotype.pdf", work_dir / "karyotype.png"]: - if stale.exists(): - stale.unlink() - cmd = [conda_python, "-m", "jcvi.graphics.karyotype", str(seqids_file.resolve()), str(layout.resolve()), "--notex"] - result = subprocess.run(cmd, capture_output=True, text=True, cwd=str(work_dir), check=False, timeout=180) - if (work_dir / "karyotype.pdf").exists(): - shutil.copy(work_dir / "karyotype.pdf", final_pdf.resolve()) - if (work_dir / "karyotype.png").exists(): - shutil.copy(work_dir / "karyotype.png", final_png.resolve()) - if final_pdf.exists() and not final_png.exists(): - pdf_to_png(final_pdf, final_png) - return {"returncode": result.returncode, "stdout": result.stdout, "stderr": result.stderr} - - final_pdf = output_dir / f"{prefix}.interspecies_synteny.pdf" - final_png = output_dir / f"{prefix}.interspecies_synteny.png" - final_mark_pdf = output_dir / f"{prefix}.interspecies_synteny.marked.pdf" - final_mark_png = output_dir / f"{prefix}.interspecies_synteny.marked.png" - - base_result = run_plot(layout_file, final_pdf, final_png) - mark_result = run_plot(layout_mark_file, final_mark_pdf, final_mark_png) - - return { - "success": final_pdf.exists() or final_png.exists(), - "pdf_file": final_pdf if final_pdf.exists() else None, - "png_file": final_png if final_png.exists() else None, - "marked_pdf_file": final_mark_pdf if final_mark_pdf.exists() else None, - "marked_png_file": final_mark_png if final_mark_png.exists() else None, - "seqids_file": seqids_file, - "layout_file": layout_file, - "layout_mark_file": layout_mark_file, - "base_plot": base_result, - "marked_plot": mark_result, - } - - -def write_report(workspace: Path, prefix: str, metrics: Dict, parameters: Dict, - tool_versions: Dict, start_time: datetime, end_time: datetime, - elapsed_sec: float, cmd: List[str]) -> Path: - """写入报告""" - report_path = workspace / "final" / f"{prefix}.interspecies_synteny_report.md" - - # 收集可复现性信息 - reproducibility_info = [] - reproducibility_info.append("## 可复现性信息\n\n") - reproducibility_info.append("### 执行环境\n") - reproducibility_info.append(f"- **分析时间**: {start_time.isoformat()}\n") - reproducibility_info.append(f"- **完成时间**: {end_time.isoformat()}\n") - reproducibility_info.append(f"- **运行时长**: {elapsed_sec:.2f} 秒\n") - reproducibility_info.append(f"- **工作目录**: {workspace}\n") - reproducibility_info.append(f"- **输入目录**: {parameters.get('input_dir', 'N/A')}\n\n") - - reproducibility_info.append("### 工具版本\n") - for tool, version in tool_versions.items(): - reproducibility_info.append(f"- **{tool}**: {version}\n") - reproducibility_info.append("\n") - - reproducibility_info.append("### 执行命令\n") - reproducibility_info.append(f"```bash\n{' '.join(cmd)}\n```\n\n") - - reproducibility_info.append("### 输入参数\n") - for key, value in parameters.items(): - if key != 'input_dir': - reproducibility_info.append(f"- **{key}**: {value}\n") - reproducibility_info.append("\n") - - with open(report_path, "w") as f: - f.write(f"# 物种间共线性分析 - {prefix}\n\n") - f.write(f"**时间**: {datetime.now().isoformat()}\n\n") - f.write(f"- 目标物种 (研究对象): {parameters.get('target_species')}\n") - f.write(f"- 参考物种 (对照): {parameters.get('ref_species')}\n") - f.write(f"- 目标蛋白文件: {parameters.get('target_protein_spl')}\n") - f.write(f"- 参考蛋白文件: {parameters.get('ref_protein_spl')}\n") - f.write(f"- 目标全蛋白组: {parameters.get('target_protein_all')}\n") - f.write(f"- 参考全蛋白组: {parameters.get('ref_protein_all')}\n") - f.write("\n") - f.writelines(reproducibility_info) - return report_path - - -def get_file_info(file_path: Path) -> Dict[str, Any]: - """获取文件的详细信息，用于artifact规范""" - info = { - "size": 0, - "sha256": "", - "mtime": 0 - } - if file_path.exists(): - info["size"] = file_path.stat().st_size - info["mtime"] = file_path.stat().st_mtime - info["sha256"] = compute_file_hash(file_path) - return info - - -def write_manifest(workspace: Path, job_id: str, run_id: str, prefix: str, - start_time: datetime, end_time: datetime, artifacts: Dict, - metrics: Dict, parameters: Dict, tool_versions: Dict, - workspace_root: Path, input_dir: Path, cmd: List[str], - status: str = "success") -> Path: - """写入manifest - 符合schema v1标准""" - elapsed_sec = (end_time - start_time).total_seconds() - - # 构建符合规范的artifacts结构 - formatted_artifacts = {} - for key, value in artifacts.items(): - if isinstance(value, dict): - # 已经是字典格式，确保包含必需字段 - formatted_artifacts[key] = value - elif isinstance(value, str): - # 字符串格式，转换为artifact对象 - file_path = workspace / value - file_info = get_file_info(file_path) - formatted_artifacts[key] = { - "artifact": key, - "path": value, - "full_path": str(file_path.absolute()), - "type": "file", - "size": file_info["size"], - "sha256": file_info["sha256"], - "mtime": file_info["mtime"] - } - - manifest_data = { - "manifest_version": "1.0", - "job_id": job_id, - "stage": "interspecies_synteny", - "run_id": run_id, - "prefix": prefix, - "timestamp": end_time.isoformat(), - "status": status, - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - "elapsed_sec": round(elapsed_sec, 2), - "cmd": cmd, - "workspace_root": str(workspace_root.absolute()), - "workspace": str(workspace.absolute()), - "input_dir": str(input_dir.absolute()), - "tool_versions": tool_versions, - "artifacts": formatted_artifacts, - "parameters": parameters, - "metrics": metrics - } - - manifest_path = workspace / "final" / f"{prefix}.interspecies_synteny.manifest.json" - with open(manifest_path, "w", encoding='utf-8') as f: - json.dump(manifest_data, f, indent=2, ensure_ascii=False) - - return manifest_path - - - -def _clean_bool(value: Any) -> bool: - return str(value).strip().lower() in {"true", "1", "yes", "y"} - - -def _gene_family_from_prefix(prefix: str) -> str: - return prefix.split('.', 1)[1] if '.' in prefix else prefix - - -def _resolve_path(path_text: str, input_dir: Path) -> Optional[Path]: - if not path_text: - return None - path = Path(path_text) - if path.is_absolute(): - return path if path.exists() else None - for candidate in (input_dir / path, input_dir / path.name): - if candidate.exists(): - return candidate - return None - - -def _read_config_groups(input_dir: Path, default_family: str) -> Dict[str, Any]: - config_path = input_dir / "config.xlsx" - groups = {"target": {}, "ref": {}, "gene_family": default_family} - if not config_path.exists(): - return groups - try: - import pandas as pd - df = pd.read_excel(config_path) - except Exception as exc: - print(f"[WARN] Failed to read config.xlsx: {exc}") - return groups - for _, row in df.iterrows(): - species = str(row.get("Species_Code", "")).strip() - family = str(row.get("Gene_Family", default_family)).strip() or default_family - file_type = str(row.get("File_Type", "")).strip().lower() - original_name = str(row.get("Original_File_Name", "")).strip() - if not species: - continue - group_name = "ref" if _clean_bool(row.get("Ref", False)) else "target" - item = groups[group_name].setdefault(species, {"species": species, "gene_family": family, "files": {}}) - item["gene_family"] = family - if file_type: - item["files"][file_type] = original_name - groups["gene_family"] = family or groups["gene_family"] - return groups - - -def _read_genome_prepare_artifacts(workspace: Path, input_dir: Path, prefix: str) -> Dict[str, Dict[str, Path]]: - result: Dict[str, Dict[str, Path]] = {} - for manifest_path in (workspace / "final" / f"{prefix}.genome_prepare.manifest.json", input_dir / "final" / f"{prefix}.genome_prepare.manifest.json"): - if not manifest_path.exists(): - continue - try: - data = json.loads(manifest_path.read_text(encoding="utf-8")) - except Exception as exc: - print(f"[WARN] Failed to read genome_prepare manifest {manifest_path}: {exc}") - continue - for key, artifact in (data.get("artifacts") or {}).items(): - kind = artifact.get("artifact") - species = key.rsplit("_", 1)[0] - found = _resolve_path(artifact.get("full_path") or artifact.get("path") or "", input_dir) - if found and kind: - result.setdefault(species, {})[kind] = found - return result - - -def _find_species_file(input_dir: Path, species: str, family: str, file_type: str, original_name: str = "") -> Optional[Path]: - if file_type == "pep": - patterns = [ - f"{species}.{family}.pep.fasta", f"{species}.{family}.pep.fa", - f"{species}.{family}.protein.fasta", f"{species}.{family}.faa", - f"{species}.pep.fasta", f"{species}.pep.fa", f"{species}.protein.fasta", f"{species}.faa", - ] - elif file_type == "gff": - patterns = [ - f"{species}.{family}_final.gff3", f"{species}.{family}.gff3", f"{species}.{family}.gff", - f"{species}_final.gff3", f"{species}.gff3", f"{species}.gff", - ] - else: - patterns = [] - if original_name: - patterns.append(Path(original_name).name) - for pattern in patterns: - for candidate in (input_dir / pattern, input_dir / species / pattern): - if candidate.exists(): - return candidate - return None - - -def discover_group_inputs(workspace: Path, input_dir: Path, prefix: str, args) -> Dict[str, Any]: - family = _gene_family_from_prefix(prefix) - groups = _read_config_groups(input_dir, family) - artifacts = _read_genome_prepare_artifacts(workspace, input_dir, prefix) - family = groups.get("gene_family") or family - - for group_name in ("target", "ref"): - for species, item in groups[group_name].items(): - files = item.setdefault("resolved", {}) - artifact_paths = artifacts.get(species, {}) - original_files = item.get("files", {}) - item_family = item.get("gene_family", family) - files["pep"] = ( - artifact_paths.get("pep") or artifact_paths.get("ref_pep") or - _find_species_file(input_dir, species, item_family, "pep", original_files.get("pep", "")) - ) - files["gff"] = ( - artifact_paths.get("gff") or artifact_paths.get("ref_gff") or - _find_species_file(input_dir, species, item_family, "gff", original_files.get("gff", "")) - ) - - target_family_protein = Path(args.target_protein) if args.target_protein else workspace / "final" / f"{prefix}.final_members.pep.fa" - ref_family_protein = Path(args.ref_protein) if args.ref_protein else None - if ref_family_protein is None: - for candidate in [input_dir / f"Ref_combined.{family}.pep.fasta", input_dir / f"Ref_combined.{family}.pep.fa", input_dir / "Ref_combined.pep.fasta"]: - if candidate.exists(): - ref_family_protein = candidate - break - if ref_family_protein is None: - ref_peps = [item.get("resolved", {}).get("pep") for item in groups["ref"].values() if item.get("resolved", {}).get("pep")] - if ref_peps: - ref_family_protein = workspace / "interspecies_synteny" / "analysis" / f"Ref_combined.{family}.pep.fasta" - merge_many_fastas(ref_peps, ref_family_protein) - - target_all_peps = {sp: item.get("resolved", {}).get("pep") for sp, item in groups["target"].items() if item.get("resolved", {}).get("pep")} - ref_all_peps = {sp: item.get("resolved", {}).get("pep") for sp, item in groups["ref"].items() if item.get("resolved", {}).get("pep")} - target_gffs = {sp: item.get("resolved", {}).get("gff") for sp, item in groups["target"].items() if item.get("resolved", {}).get("gff")} - ref_gffs = {sp: item.get("resolved", {}).get("gff") for sp, item in groups["ref"].items() if item.get("resolved", {}).get("gff")} - - return { - "gene_family": family, - "target_species": sorted(groups["target"].keys()), - "ref_species": sorted(groups["ref"].keys()), - "target_family_protein": target_family_protein, - "ref_family_protein": ref_family_protein, - "target_all_peps": target_all_peps, - "ref_all_peps": ref_all_peps, - "target_gffs": target_gffs, - "ref_gffs": ref_gffs, - "groups": groups, - } - - -def merge_many_fastas(files: List[Path], output_file: Path) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as out: - for fasta in files: - with open(fasta, "r") as f: - out.write(f.read().rstrip() + "\n") - return output_file - - -def _parse_gff_attr(attr: str) -> Dict[str, str]: - result = {} - for item in attr.strip().strip(';').split(';'): - item = item.strip() - if not item: - continue - if '=' in item: - k, v = item.split('=', 1) - elif ' ' in item: - k, v = item.split(' ', 1) - else: - continue - result[k.strip()] = v.strip().strip('"') - return result - - -def _match_gff_id(attrs: Dict[str, str], protein_ids: set) -> Optional[str]: - candidates = [attrs.get("ID"), attrs.get("transcript_id"), attrs.get("Name"), attrs.get("Parent")] - for candidate in candidates: - if not candidate: - continue - for item in str(candidate).split(','): - norm = normalize_seq_id(item) - if norm in protein_ids: - return norm - for variant in id_variants(norm): - if variant in protein_ids: - return variant - return None - - -def build_species_position_files(species: str, gff3: Path, protein_fasta: Path, work_dir: Path) -> Dict[str, Any]: - """Create MCScanX simplified GFF and JCVI BED for one species.""" - simple_gff = work_dir / f"{species}.gff" - bed_file = work_dir / f"{species}.bed" - protein_ids = fasta_id_set(protein_fasta) - seen = set() - count = 0 - features_seen = 0 - work_dir.mkdir(parents=True, exist_ok=True) - with open(gff3, "r") as f_in, open(simple_gff, "w") as gff_out, open(bed_file, "w") as bed_out: - for line in f_in: - if not line.strip() or line.startswith('#'): - continue - parts = line.rstrip('\n').split(' ') - if len(parts) < 9 or parts[2] not in {"mRNA", "transcript", "gene"}: - continue - features_seen += 1 - attrs = _parse_gff_attr(parts[8]) - gene_id = _match_gff_id(attrs, protein_ids) - if not gene_id or gene_id in seen: - continue - seen.add(gene_id) - chrom = parts[0] - start, end = parts[3], parts[4] - gff_out.write(f"{chrom} {gene_id} {start} {end}\n") - bed_out.write(f"{chrom} {start} {end} {gene_id}\n") - count += 1 - return { - "success": count > 0, - "species": species, - "gff": simple_gff, - "bed": bed_file, - "gene_count": count, - "features_seen": features_seen, - "protein_id_count": len(protein_ids), - "error": "No GFF IDs matched protein FASTA IDs" if count == 0 else "", - } - - -def collect_family_ids(target_family_fasta: Path, ref_family_fasta: Path, final_id_file: Optional[Path] = None) -> set: - ids = set() - for fasta in [target_family_fasta, ref_family_fasta]: - if fasta and Path(fasta).exists(): - ids.update(fasta_id_set(Path(fasta))) - if final_id_file and final_id_file.exists(): - with open(final_id_file, "r") as f: - for line in f: - if line.strip() and not line.startswith('#'): - ids.update(id_variants(line.strip().split()[0])) - return ids - - -def add_artifact(artifacts: Dict[str, Dict], key: str, file_path: Path, rel_path: str, artifact: str, file_type: str) -> None: - if not file_path or not file_path.exists(): - return - info = get_file_info(file_path) - artifacts[key] = { - "artifact": artifact, - "path": rel_path, - "full_path": str(file_path.absolute()), - "type": file_type, - "size": info["size"], - "sha256": info["sha256"], - "mtime": info["mtime"], - } - - -def write_failure_manifest(workspace: Path, job_id: str, run_id: str, prefix: str, start_time: datetime, - workspace_root: Path, input_dir: Path, cmd: List[str], error: str) -> Path: - end_time = datetime.now() - return write_manifest( - workspace, job_id, run_id, prefix, start_time, end_time, - artifacts={}, metrics={"error": error}, parameters={"prefix": prefix, "error": error}, - tool_versions={}, workspace_root=workspace_root, input_dir=input_dir, cmd=cmd, status="failed" - ) - - -def main(): - parser = argparse.ArgumentParser(description="????????") - parser.add_argument("--prefix", required=True, help="?????? Multi.SPL") - parser.add_argument("--workspace", required=True) - parser.add_argument("--input-dir") - parser.add_argument("--ref-species", default=None, help="???????? config.xlsx ??") - parser.add_argument("--target-species", default=None, help="???????? config.xlsx ??") - parser.add_argument("--ref-protein", help="?????????") - parser.add_argument("--target-protein", help="?????????") - parser.add_argument("--cpu", type=int, default=8) - parser.add_argument("--log-file", help="??????????????") - parser.add_argument("--workspace-root", help="??????????manifest?") - args = parser.parse_args() - - start_time = datetime.now() - run_id = f"interspecies_synteny_{start_time.strftime('%Y%m%d_%H%M%S')}" - workspace = Path(args.workspace) - job_id = workspace.name - input_dir = Path(args.input_dir) if args.input_dir else workspace.parent.parent / "input" / job_id - workspace_root = Path(args.workspace_root) if args.workspace_root else workspace.parent - cmd = sys.argv - - print("=" * 60) - print("Interspecies synteny analysis") - print("=" * 60) - - try: - paths = prepare_directories(workspace) - tool_versions = get_tool_versions() - discovery = discover_group_inputs(workspace, input_dir, args.prefix, args) - target_group = "Target_combined" - ref_group = "Ref_combined" - print(f"Target species: {', '.join(discovery['target_species']) or 'NONE'}") - print(f"Reference species: {', '.join(discovery['ref_species']) or 'NONE'}") - print(f"Target family protein: {discovery['target_family_protein']}") - print(f"Ref family protein: {discovery['ref_family_protein']}") - - if not discovery["target_family_protein"] or not Path(discovery["target_family_protein"]).exists(): - raise RuntimeError(f"Target family protein missing: {discovery['target_family_protein']}") - if not discovery["ref_family_protein"] or not Path(discovery["ref_family_protein"]).exists(): - raise RuntimeError("Reference family protein missing; expected Ref_combined..pep.fasta or Ref species pep files") - - artifacts = {} - metrics = { - "target_species": discovery["target_species"], - "ref_species": discovery["ref_species"], - "tree_success": False, - "synteny_success": False, - "synteny_skipped_reason": "", - } - - print("\nMerging family proteins for tree...") - merged_fasta = paths["analysis_dir"] / f"{args.prefix}.target_ref.family.pep.fa" - merge_result = merge_protein_sequences(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), merged_fasta) - if not merge_result.get("success"): - raise RuntimeError(f"Failed to merge family proteins: {merge_result.get('stderr')}") - metrics["target_family_count"] = merge_result.get("target_count") - metrics["ref_family_count"] = merge_result.get("ref_count") - - print("\nBuilding interspecies tree...") - tree_result = build_phylogenetic_tree(merged_fasta, paths["analysis_dir"], args.prefix, args.cpu) - if not tree_result.get("tree_file"): - raise RuntimeError(f"Tree construction failed: {tree_result.get('error', 'unknown error')}") - final_tree = paths["final_dir"] / f"{args.prefix}.interspecies_tree.nwk" - shutil.copy(Path(tree_result["tree_file"]), final_tree) - metrics["tree_success"] = True - add_artifact(artifacts, "tree_nwk", final_tree, f"final/{args.prefix}.interspecies_tree.nwk", "tree_nwk", "tree") - - gene_species_map = paths["analysis_dir"] / f"{args.prefix}.gene_to_species.tsv" - write_gene_species_map(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), discovery["target_species"], discovery["ref_species"], gene_species_map, collect_species_id_sets(discovery["groups"])) - add_artifact(artifacts, "gene_to_species", gene_species_map, f"interspecies_synteny/analysis/{args.prefix}.gene_to_species.tsv", "gene_to_species", "tsv") - - print("\nPlotting interspecies tree...") - target_subfamily_file = input_dir / f"{args.prefix}.geneID.subfamily" - if not target_subfamily_file.exists(): - target_subfamily_file = workspace / "final" / f"{args.prefix}.geneID.subfamily" - ref_subfamily_file = input_dir / f"SPL_{discovery['ref_species'][0]}.subfamily" if discovery["ref_species"] else input_dir / "SPL_ref.subfamily" - if not ref_subfamily_file.exists(): - candidates = sorted(input_dir.glob("SPL_*.subfamily")) + sorted(input_dir.glob("*.SPL.subfamily")) - ref_subfamily_file = candidates[0] if candidates else ref_subfamily_file - tree_plot_result = plot_interspecies_tree( - final_tree, - paths["final_dir"], - args.prefix, - target_group, - ref_group, - target_subfamily_file if target_subfamily_file.exists() else None, - ref_subfamily_file if ref_subfamily_file.exists() else None, - gene_species_map, - ) - if tree_plot_result.get("pdf_file"): - add_artifact(artifacts, "tree_pdf", tree_plot_result["pdf_file"], f"final/{args.prefix}.interspecies_tree.pdf", "tree_pdf", "plot") - if tree_plot_result.get("png_file"): - add_artifact(artifacts, "tree_png", tree_plot_result["png_file"], f"final/{args.prefix}.interspecies_tree.png", "tree_png", "plot") - - synteny_plot_result = {"success": False} - pair_results: List[Dict[str, Any]] = [] - species_position_results: Dict[str, Dict[str, Any]] = {} - - missing = [] - if not discovery["target_all_peps"]: - missing.append("target all protein") - if not discovery["ref_all_peps"]: - missing.append("reference all protein") - if not discovery["target_gffs"]: - missing.append("target GFF") - if not discovery["ref_gffs"]: - missing.append("reference GFF") - - if not missing: - print("\nPreparing per-species GFF/BED files for pairwise MCScanX...") - all_species = discovery["target_species"] + discovery["ref_species"] - for species in all_species: - pep = discovery["target_all_peps"].get(species) or discovery["ref_all_peps"].get(species) - gff = discovery["target_gffs"].get(species) or discovery["ref_gffs"].get(species) - if not pep or not gff: - species_position_results[species] = {"success": False, "error": "missing pep or gff"} - continue - result = build_species_position_files(species, Path(gff), Path(pep), paths["analysis_dir"]) - species_position_results[species] = result - if result.get("success"): - add_artifact(artifacts, f"{species}_mcscanx_gff", result["gff"], f"interspecies_synteny/analysis/{species}.gff", "mcscanx_gff", "gff") - add_artifact(artifacts, f"{species}_bed", result["bed"], f"interspecies_synteny/analysis/{species}.bed", "jcvi_bed", "bed") - print(f" {species}: {result.get('gene_count', 0)} matched genes") - - print("\nRunning pairwise target-ref MCScanX...") - for target_species in discovery["target_species"]: - for ref_species in discovery["ref_species"]: - pair_key = f"{target_species}.{ref_species}" - pair = {"pair": pair_key, "target_species": target_species, "ref_species": ref_species, "success": False} - target_ready = species_position_results.get(target_species, {}).get("success") - ref_ready = species_position_results.get(ref_species, {}).get("success") - target_pep = discovery["target_all_peps"].get(target_species) - ref_pep = discovery["ref_all_peps"].get(ref_species) - if not (target_ready and ref_ready and target_pep and ref_pep): - pair["error"] = "missing matched GFF/BED or protein FASTA" - pair_results.append(pair) - print(f" {pair_key}: skipped ({pair['error']})") - continue - blast_result = run_reciprocal_blast(Path(target_pep), Path(ref_pep), paths["analysis_dir"], target_species, ref_species, args.cpu) - pair["blast_success"] = bool(blast_result.get("success")) - if blast_result.get("output_file"): - pair["blast_file"] = str(blast_result["output_file"]) - add_artifact(artifacts, f"{pair_key}_blast", blast_result["output_file"], f"interspecies_synteny/analysis/{pair_key}.blast", "blast", "data") - if not blast_result.get("success"): - pair["error"] = "BLAST failed or produced no hits" - pair_results.append(pair) - print(f" {pair_key}: {pair['error']}") - continue - mcscanx_result = run_mcscanx_interspecies(paths["analysis_dir"], target_species, ref_species) - pair_gff = paths["analysis_dir"] / f"{pair_key}.gff" - if pair_gff.exists(): - pair["pair_gff"] = str(pair_gff) - add_artifact(artifacts, f"{pair_key}_gff", pair_gff, f"interspecies_synteny/analysis/{pair_key}.gff", "pair_mcscanx_gff", "gff") - pair["mcscanx_success"] = bool(mcscanx_result.get("success")) - if mcscanx_result.get("collinearity_file"): - pair["collinearity_file"] = str(mcscanx_result["collinearity_file"]) - add_artifact(artifacts, f"{pair_key}_collinearity", mcscanx_result["collinearity_file"], f"interspecies_synteny/analysis/{pair_key}.collinearity", "collinearity", "data") - if not mcscanx_result.get("success"): - pair["error"] = "MCScanX did not produce collinearity" - pair_results.append(pair) - print(f" {pair_key}: {pair['error']}") - continue - simple_result = convert_collinearity_to_simple(paths["analysis_dir"], Path(mcscanx_result["collinearity_file"]), target_species, ref_species) - if not simple_result.get("success") and blast_result.get("output_file"): - fallback_result = convert_blast_to_family_simple(paths["analysis_dir"], Path(blast_result["output_file"]), target_species, ref_species) - if fallback_result.get("success"): - fallback_result["note"] = "MCScanX produced no whole-genome blocks; using family-only BLAST links for JCVI visualization" - simple_result = fallback_result - pair.update({k: str(v) if isinstance(v, Path) else v for k, v in simple_result.items()}) - pair["success"] = bool(simple_result.get("success")) - if simple_result.get("anchors_file"): - add_artifact(artifacts, f"{pair_key}_anchors", simple_result["anchors_file"], f"interspecies_synteny/analysis/{pair_key}.anchors", "anchors", "data") - if simple_result.get("simple_file"): - add_artifact(artifacts, f"{pair_key}_anchors_simple", simple_result["simple_file"], f"interspecies_synteny/analysis/{pair_key}.anchors.simple", "anchors_simple", "data") - pair_results.append(pair) - print(f" {pair_key}: {'success' if pair['success'] else 'anchors conversion failed'}") - - family_ids = collect_family_ids(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), workspace / "final" / f"{args.prefix}.final_members.id") - synteny_plot_result = plot_multi_species_synteny( - paths["analysis_dir"], paths["final_dir"], args.prefix, - discovery["target_species"] + discovery["ref_species"], pair_results, family_ids, - ) - successful_pairs = [p for p in pair_results if p.get("success")] - metrics["synteny_success"] = bool(synteny_plot_result.get("success")) and len(successful_pairs) == len(pair_results) and bool(pair_results) - if not successful_pairs: - metrics["synteny_skipped_reason"] = synteny_plot_result.get("error") or "No target-ref pair produced JCVI anchors" - else: - metrics["synteny_skipped_reason"] = "Missing full data for MCScanX: " + ", ".join(missing) - print(metrics["synteny_skipped_reason"]) - - metrics["species_position_results"] = { - sp: {k: str(v) if isinstance(v, Path) else v for k, v in result.items()} - for sp, result in species_position_results.items() - } - metrics["pairwise_synteny"] = pair_results - metrics["pair_success_count"] = sum(1 for p in pair_results if p.get("success")) - metrics["pair_total_count"] = len(pair_results) - - if synteny_plot_result.get("pdf_file"): - add_artifact(artifacts, "synteny_pdf", synteny_plot_result["pdf_file"], f"final/{args.prefix}.interspecies_synteny.pdf", "synteny_pdf", "plot") - if synteny_plot_result.get("png_file"): - add_artifact(artifacts, "synteny_png", synteny_plot_result["png_file"], f"final/{args.prefix}.interspecies_synteny.png", "synteny_png", "plot") - if synteny_plot_result.get("marked_pdf_file"): - add_artifact(artifacts, "synteny_marked_pdf", synteny_plot_result["marked_pdf_file"], f"final/{args.prefix}.interspecies_synteny.marked.pdf", "synteny_marked_pdf", "plot") - if synteny_plot_result.get("marked_png_file"): - add_artifact(artifacts, "synteny_marked_png", synteny_plot_result["marked_png_file"], f"final/{args.prefix}.interspecies_synteny.marked.png", "synteny_marked_png", "plot") - if synteny_plot_result.get("seqids_file"): - add_artifact(artifacts, "jcvi_seqids", synteny_plot_result["seqids_file"], f"interspecies_synteny/analysis/{args.prefix}.seqids", "jcvi_seqids", "data") - if synteny_plot_result.get("layout_file"): - add_artifact(artifacts, "jcvi_layout", synteny_plot_result["layout_file"], f"interspecies_synteny/analysis/{args.prefix}.layout", "jcvi_layout", "data") - - end_time = datetime.now() - status = "success" if metrics["tree_success"] and metrics["synteny_success"] else "partial_success" - parameters = { - "prefix": args.prefix, - "gene_family": discovery["gene_family"], - "target_group": target_group, - "ref_group": ref_group, - "target_species": discovery["target_species"], - "ref_species": discovery["ref_species"], - "target_protein_spl": str(discovery["target_family_protein"]), - "ref_protein_spl": str(discovery["ref_family_protein"]), - "input_dir": str(input_dir), - "cpu": args.cpu, - } - report_path = write_report(workspace, args.prefix, metrics, parameters, tool_versions, start_time, end_time, (end_time - start_time).total_seconds(), cmd) - add_artifact(artifacts, "report", report_path, f"final/{args.prefix}.interspecies_synteny_report.md", "report", "report") - manifest_path = write_manifest(workspace, job_id, run_id, args.prefix, start_time, end_time, artifacts, metrics, parameters, tool_versions, workspace_root, input_dir, cmd, status=status) - add_artifact(artifacts, "manifest", manifest_path, f"final/{args.prefix}.interspecies_synteny.manifest.json", "manifest", "json") - write_manifest(workspace, job_id, run_id, args.prefix, start_time, end_time, artifacts, metrics, parameters, tool_versions, workspace_root, input_dir, cmd, status=status) - print(f"Interspecies synteny finished with status: {status}") - print(f"Manifest: {manifest_path}") - return 0 if metrics["tree_success"] else 1 - except Exception as exc: - print(f"Interspecies synteny failed: {exc}") - try: - write_failure_manifest(workspace, job_id, run_id, args.prefix, start_time, workspace_root, input_dir, cmd, str(exc)) - except Exception as manifest_exc: - print(f"Failed to write failure manifest: {manifest_exc}") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bioflow_kernel/cli/run_interspecies_synteny.py.bak_highlight_local b/bioflow_kernel/cli/run_interspecies_synteny.py.bak_highlight_local deleted file mode 100755 index e592d21..0000000 --- a/bioflow_kernel/cli/run_interspecies_synteny.py.bak_highlight_local +++ /dev/null @@ -1,1542 +0,0 @@ -#!/usr/bin/env python3 -""" -Interspecies Synteny阶段：物种间共线性分析（完整版） - -输入： - - 参考物种蛋白序列 (final/{prefix}.final_members.pep.fa) - - 参考物种GFF3 - - 目标物种蛋白序列 - - 目标物种GFF3 - -输出： - - final/{prefix}.interspecies_tree.nwk (物种间进化树) - - final/{prefix}.interspecies_tree.pdf/png (进化树图) - - final/{prefix}.interspecies_synteny.pdf/png (共线性图) - - final/{prefix}.interspecies_synteny_report.md (分析报告) - - final/{prefix}.interspecies_synteny.manifest.json (机器清单) -""" - -import os -import argparse -import json -import hashlib -import subprocess -import sys -import time -import shutil -import re -from datetime import datetime -from pathlib import Path -from typing import Dict, List, Any, Optional, Tuple - - -# ??conda??bin???PATH -CONDA_BIN = "/home/xia-lab/miniconda3/envs/test4py311/bin" -if CONDA_BIN not in os.environ.get("PATH", ""): - os.environ["PATH"] = CONDA_BIN + ":" + os.environ.get("PATH", "") -def pdf_to_png(pdf_path: Path, png_path: Path, dpi: int = 300) -> bool: - """将PDF转换为PNG""" - try: - # 方法1: 使用pdftoppm (poppler-utils) - cmd = ["pdftoppm", "-png", "-r", str(dpi), "-singlefile", - str(pdf_path), str(png_path.with_suffix(''))] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - if result.returncode == 0: - return True - except Exception: - pass - - try: - # 方法2: 使用convert (ImageMagick) - cmd = ["convert", "-density", str(dpi), str(pdf_path), "-quality", "100", str(png_path)] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - if result.returncode == 0: - return True - except Exception: - pass - - return False - - -def compute_file_hash(filepath: Path) -> str: - """计算文件的SHA256哈希值""" - sha256_hash = hashlib.sha256() - try: - with open(filepath, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - except Exception: - return "" - - -def get_tool_versions() -> Dict[str, str]: - """获取所需工具版本信息""" - versions = {} - - for tool, cmd in [ - ("mafft", ["mafft", "--version"]), - ("iqtree", ["iqtree", "--version"]), - ("blastp", ["blastp", "-version"]), - ("MCScanX", ["MCScanX", "-h"]), - ]: - try: - result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=5) - versions[tool] = "available" if result.returncode in [0, 1] else "not_found" - except Exception: - versions[tool] = "not_found" - - return versions - - -def prepare_directories(workspace: Path) -> Dict[str, Path]: - """创建目录结构""" - paths = { - "analysis_dir": workspace / "interspecies_synteny" / "analysis", - "final_dir": workspace / "final", - "logs_dir": workspace / "logs", - } - for name, path in paths.items(): - path.mkdir(parents=True, exist_ok=True) - return paths - - -def merge_protein_sequences( - target_protein: Path, - ref_protein: Path, - output_fasta: Path -) -> Dict[str, Any]: - """Merge target and reference family proteins for the interspecies tree.""" - print("Merging family protein sequences...") - - try: - with open(output_fasta, 'w') as outfile: - with open(target_protein, 'r') as f: - outfile.write(f.read().rstrip() + "\n") - with open(ref_protein, 'r') as f: - outfile.write(f.read().rstrip() + "\n") - - target_count = sum(1 for line in open(target_protein) if line.startswith('>')) - ref_count = sum(1 for line in open(ref_protein) if line.startswith('>')) - - print(f" merged target/ref family proteins: {target_count} + {ref_count} = {target_count + ref_count}") - - return { - "success": True, - "output_file": output_fasta, - "target_count": target_count, - "ref_count": ref_count - } - except Exception as e: - return {"success": False, "stderr": str(e)} - - -def build_phylogenetic_tree( - merged_fasta: Path, - output_dir: Path, - prefix: str, - cpu: int = 8 -) -> Dict[str, Any]: - """构建系统发育树 (MAFFT + IQ-TREE)""" - print(f"构建进化树...") - - # MAFFT比对 - msa_file = output_dir / f"{prefix}_interspecies.mafft.msa.fa" - mafft_cmd = ["mafft", "--auto", "--thread", str(cpu), str(merged_fasta)] - - try: - with open(msa_file, "w") as f_out: - result = subprocess.run(mafft_cmd, stdout=f_out, stderr=subprocess.PIPE, - text=True, check=False) - - if result.returncode != 0: - return {"success": False, "error": f"MAFFT失败: {result.stderr}"} - - # IQ-TREE建树 - base_name = output_dir / f"{prefix}_interspecies.iqtree" - iqtree_cmd = [ - "iqtree", "-s", str(msa_file), "-m", "MFP", "-bb", "1000", - "-nt", str(cpu), "-pre", str(base_name) - ] - - result = subprocess.run(iqtree_cmd, capture_output=True, text=True, check=False) - - # IQ-TREE 生成的 treefile 是 {prefix}.iqtree.treefile，不是 {prefix}.treefile - tree_file = Path(f"{base_name}.treefile") - - return { - "success": tree_file.exists(), - "tree_file": tree_file if tree_file.exists() else None, - "msa_file": msa_file - } - except Exception as e: - return {"success": False, "error": str(e)} - - -def filter_gff_files( - ref_gff3: Path, - target_gff3: Path, - output_dir: Path, - ref_species: str, - target_species: str -) -> Dict[str, Any]: - """过滤GFF3文件 - 提取基因ID（去掉转录本版本后缀如 .1, .2）""" - print(f"过滤GFF3文件...") - - import re - - results = {} - for gff3_file, species, prefix in [ - (ref_gff3, ref_species, "ref"), - (target_gff3, target_species, "target") - ]: - output_gff = output_dir / f"{species}.gff" - - try: - with open(gff3_file, 'r') as f: - lines = f.readlines() - - count = 0 - seen_genes = set() # 去重 - - with open(output_gff, 'w') as f: - for line in lines: - parts = line.strip().split('\t') - if len(parts) >= 9 and parts[2] == "mRNA": - attr = parts[8] - # 提取ID（如 ID=AT2G47070.1），保留版本号与BLAST结果匹配 - gene_id = attr.split(';')[0].replace('ID=', '') - - # 处理染色体名称：去掉 "Chr" 前缀（Ath用1,2,3而非Chr1,Chr2） - chr_name = parts[0] - if chr_name.startswith('Chr'): - chr_name = chr_name[3:] # Chr1 -> 1 - - # 去重：一个基因可能有多个转录本 - if gene_id not in seen_genes: - seen_genes.add(gene_id) - f.write(f"{chr_name}\t{gene_id}\t{parts[3]}\t{parts[4]}\n") - count += 1 - - print(f" {species}: {count} 个基因（去重后）") - results[prefix] = {"success": True, "output_file": output_gff, "gene_count": count} - except Exception as e: - results[prefix] = {"success": False, "stderr": str(e)} - - if results.get("ref", {}).get("success") and results.get("target", {}).get("success"): - return { - "success": True, - "ref_gff": results["ref"]["output_file"], - "target_gff": results["target"]["output_file"] - } - return {"success": False} - - -def run_reciprocal_blast( - ref_protein: Path, - target_protein: Path, - output_dir: Path, - ref_species: str, - target_species: str, - cpu: int = 8 -) -> Dict[str, Any]: - """双向BLAST比对 - 结果中基因名去掉版本后缀以匹配GFF""" - import re - print(f"双向BLAST比对...") - - # 构建数据库 - for protein_file in [ref_protein, target_protein]: - subprocess.run(["makeblastdb", "-in", str(protein_file), "-dbtype", "prot"], - capture_output=True, check=False) - - # 双向blastp - blast1_out = output_dir / f"tmp.{ref_species}_{target_species}.blast" - subprocess.run([ - "blastp", "-query", str(ref_protein), "-db", str(target_protein), - "-outfmt", "6", "-out", str(blast1_out), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ], capture_output=True, check=False) - - blast2_out = output_dir / f"tmp.{target_species}_{ref_species}.blast" - subprocess.run([ - "blastp", "-query", str(target_protein), "-db", str(ref_protein), - "-outfmt", "6", "-out", str(blast2_out), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ], capture_output=True, check=False) - - # 合并BLAST结果（保留版本号与GFF匹配） - merged_blast = output_dir / f"{ref_species}_{target_species}.blast" - with open(merged_blast, 'w') as outfile: - for blast_file in [blast1_out, blast2_out]: - if blast_file.exists(): - with open(blast_file, 'r') as infile: - for line in infile: - outfile.write(line) # 直接写入，保留版本号 - - return {"success": True, "output_file": merged_blast} - - -def run_mcscanx_interspecies( - work_dir: Path, - ref_species: str, - target_species: str -) -> Dict[str, Any]: - """MCScanX种间共线性 - 需要合并的GFF文件""" - print(f"MCScanX种间共线性...") - - prefix = f"{ref_species}_{target_species}" - - # MCScanX 需要合并的 GFF 文件 (prefix.gff) - merged_gff = work_dir / f"{prefix}.gff" - ref_gff = work_dir / f"{ref_species}.gff" - target_gff = work_dir / f"{target_species}.gff" - - if not merged_gff.exists() and ref_gff.exists() and target_gff.exists(): - print(f" 合并 GFF 文件: {ref_gff.name} + {target_gff.name} -> {merged_gff.name}") - with open(merged_gff, 'w') as outfile: - for gff_file in [target_gff, ref_gff]: # 目标物种在前，参考物种在后 - with open(gff_file, 'r') as infile: - outfile.write(infile.read()) - - cmd = ["MCScanX", f"./{prefix}"] - - result = subprocess.run(cmd, capture_output=True, text=True, - cwd=str(work_dir), check=False) - - collinearity_file = work_dir / f"{prefix}.collinearity" - - return { - "success": collinearity_file.exists(), - "collinearity_file": collinearity_file if collinearity_file.exists() else None - } - - - - -def fasta_ids(fasta_file: Path) -> List[str]: - ids = [] - with open(fasta_file, "r") as f: - for line in f: - if line.startswith(">"): - ids.append(line[1:].strip().split()[0]) - return ids - - -def display_species_code(species: str, gene_id: str = "") -> str: - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return species - - -def species_for_gene(gene_id: str, species_list: List[str], default_species: str) -> str: - for species in sorted(species_list, key=len, reverse=True): - if gene_id.lower().startswith(species.lower()): - return display_species_code(species, gene_id) - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return display_species_code(default_species, gene_id) - - -def write_gene_species_map(target_fasta: Path, ref_fasta: Path, target_species: List[str], ref_species: List[str], output_file: Path) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - default_target = target_species[0] if target_species else "Target" - default_ref = ref_species[0] if ref_species else "Ref" - with open(output_file, "w") as out: - out.write("gene_id\tspecies\tgroup\n") - for gene_id in fasta_ids(target_fasta): - out.write(f"{gene_id}\t{species_for_gene(gene_id, target_species, default_target)}\ttarget\n") - for gene_id in fasta_ids(ref_fasta): - out.write(f"{gene_id}\t{species_for_gene(gene_id, ref_species, default_ref)}\tref\n") - return output_file -def normalize_seq_id(text: str) -> str: - """Normalize FASTA/GFF IDs while preserving biologically meaningful versions.""" - value = (text or "").strip() - if value.startswith(">"): - value = value[1:] - value = value.split()[0] if value.split() else value - if "|" in value: - value = value.split("|", 1)[0] - for prefix in ("gene:", "transcript:", "CDS:", "rna-", "gene-", "cds-"): - if value.startswith(prefix): - value = value[len(prefix):] - return value.strip().strip('"') - - -def id_variants(gene_id: str) -> set: - norm = normalize_seq_id(gene_id) - variants = {norm} - for prefix in ("gene:", "transcript:", "CDS:", "rna-", "gene-", "cds-"): - variants.add(normalize_seq_id(prefix + norm)) - if "|" in gene_id: - variants.add(normalize_seq_id(gene_id.split("|", 1)[0])) - return {v for v in variants if v} - - -def fasta_ids(fasta_file: Path) -> List[str]: - ids = [] - with open(fasta_file, "r") as f: - for line in f: - if line.startswith(">"): - seq_id = normalize_seq_id(line[1:]) - if seq_id: - ids.append(seq_id) - return ids - - -def fasta_id_set(fasta_file: Path) -> set: - ids = set() - for seq_id in fasta_ids(fasta_file): - ids.update(id_variants(seq_id)) - return ids - - -def display_species_code(species: str, gene_id: str = "") -> str: - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return species - - -def species_for_gene(gene_id: str, species_list: List[str], default_species: str, species_id_sets: Optional[Dict[str, set]] = None) -> str: - variants = id_variants(gene_id) - if species_id_sets: - for species, ids in species_id_sets.items(): - if variants & ids: - return display_species_code(species, gene_id) - for species in sorted(species_list, key=len, reverse=True): - if normalize_seq_id(gene_id).lower().startswith(species.lower()): - return display_species_code(species, gene_id) - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return display_species_code(default_species, gene_id) - - -def collect_species_id_sets(groups: Dict[str, Any]) -> Dict[str, set]: - species_id_sets: Dict[str, set] = {} - for group_name in ("target", "ref"): - for species, item in groups.get(group_name, {}).items(): - pep = item.get("resolved", {}).get("pep") - if pep and Path(pep).exists(): - species_id_sets[species] = fasta_id_set(Path(pep)) - return species_id_sets - - -def write_gene_species_map( - target_fasta: Path, - ref_fasta: Path, - target_species: List[str], - ref_species: List[str], - output_file: Path, - species_id_sets: Optional[Dict[str, set]] = None, -) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - default_target = target_species[0] if target_species else "Target" - default_ref = ref_species[0] if ref_species else "Ref" - with open(output_file, "w") as out: - out.write("gene_id\tspecies\tgroup\n") - for gene_id in fasta_ids(target_fasta): - species = species_for_gene(gene_id, target_species, default_target, species_id_sets) - out.write(f"{gene_id}\t{species}\ttarget\n") - for gene_id in fasta_ids(ref_fasta): - species = species_for_gene(gene_id, ref_species, default_ref, species_id_sets) - out.write(f"{gene_id}\t{species}\tref\n") - return output_file - - -def run_reciprocal_blast( - target_protein: Path, - ref_protein: Path, - output_dir: Path, - target_species: str, - ref_species: str, - cpu: int = 8 -) -> Dict[str, Any]: - """Run pairwise reciprocal BLASTP for one target-ref species pair.""" - print(f"Reciprocal BLASTP: {target_species} vs {ref_species}") - - pair_prefix = f"{target_species}.{ref_species}" - target_db = output_dir / f"{target_species}.blastdb" - ref_db = output_dir / f"{ref_species}.blastdb" - commands = [] - - for protein_file, db_prefix in [(target_protein, target_db), (ref_protein, ref_db)]: - cmd = ["makeblastdb", "-in", str(protein_file), "-dbtype", "prot", "-out", str(db_prefix)] - commands.append(" ".join(cmd)) - result = subprocess.run(cmd, capture_output=True, text=True, check=False) - if result.returncode != 0: - return {"success": False, "stderr": result.stderr, "commands": commands} - - target_to_ref = output_dir / f"tmp.{target_species}_{ref_species}.blast" - ref_to_target = output_dir / f"tmp.{ref_species}_{target_species}.blast" - for query, db_prefix, out_file in [(target_protein, ref_db, target_to_ref), (ref_protein, target_db, ref_to_target)]: - cmd = [ - "blastp", "-query", str(query), "-db", str(db_prefix), - "-outfmt", "6", "-out", str(out_file), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ] - commands.append(" ".join(cmd)) - result = subprocess.run(cmd, capture_output=True, text=True, check=False) - if result.returncode != 0: - return {"success": False, "stderr": result.stderr, "commands": commands} - - merged_blast = output_dir / f"{pair_prefix}.blast" - with open(merged_blast, 'w') as outfile: - for blast_file in [target_to_ref, ref_to_target]: - if blast_file.exists(): - with open(blast_file, 'r') as infile: - for line in infile: - if line.strip(): - outfile.write(line) - - return {"success": merged_blast.exists() and merged_blast.stat().st_size > 0, "output_file": merged_blast, "commands": commands} - - -def run_mcscanx_interspecies(work_dir: Path, target_species: str, ref_species: str) -> Dict[str, Any]: - """Run MCScanX for one target-ref pair.""" - pair_prefix = f"{target_species}.{ref_species}" - print(f"MCScanX pair: {pair_prefix}") - - merged_gff = work_dir / f"{pair_prefix}.gff" - target_gff = work_dir / f"{target_species}.gff" - ref_gff = work_dir / f"{ref_species}.gff" - - if target_gff.exists() and ref_gff.exists(): - with open(merged_gff, 'w') as outfile: - for gff_file in [target_gff, ref_gff]: - with open(gff_file, 'r') as infile: - outfile.write(infile.read().rstrip() + "\n") - else: - return {"success": False, "error": f"Missing simplified GFF for {pair_prefix}"} - - result = subprocess.run(["MCScanX", f"./{pair_prefix}"], capture_output=True, text=True, cwd=str(work_dir), check=False) - collinearity_file = work_dir / f"{pair_prefix}.collinearity" - return { - "success": collinearity_file.exists() and collinearity_file.stat().st_size > 0, - "collinearity_file": collinearity_file if collinearity_file.exists() else None, - "stdout": result.stdout, - "stderr": result.stderr, - "returncode": result.returncode, - } - - -def plot_interspecies_tree( - tree_file: Path, - output_dir: Path, - prefix: str, - target_species: str, - ref_species: str, - target_subfamily_file: Optional[Path] = None, - ref_subfamily_file: Optional[Path] = None, - gene_species_map: Optional[Path] = None -) -> Dict[str, Any]: - """绘制物种间进化树图 - 使用专用的 interspecies_tree.R 脚本""" - print(f"绘制进化树图...") - - # 查找专用的种间进化树R脚本 - r_script = Path("scripts/interspecies_tree.R") - if not r_script.exists(): - alt_script = Path("/home/xia-lab/wutx/00test/bbav2/scripts/interspecies_tree.R") - if alt_script.exists(): - r_script = alt_script - else: - return {"success": False, "error": f"R脚本不存在: {r_script}"} - - try: - # 使用专用的 interspecies_tree.R 脚本，传递物种参数和亚家族文件 - # 使用conda环境的Rscript - conda_rscript = "/home/xia-lab/miniconda3/envs/test4py311/bin/Rscript" - cmd = [ - conda_rscript, str(r_script), - str(tree_file), - str(output_dir), - target_species, - ref_species - ] - - # 添加亚家族文件参数（如果存在） - if target_subfamily_file and target_subfamily_file.exists(): - cmd.append(str(target_subfamily_file)) - print(f" 目标物种亚家族文件: {target_subfamily_file}") - else: - cmd.append("") # 空字符串表示无文件 - - if ref_subfamily_file and ref_subfamily_file.exists(): - cmd.append(str(ref_subfamily_file)) - print(f" 参考物种亚家族文件: {ref_subfamily_file}") - else: - cmd.append("") # 空字符串表示无文件 - if gene_species_map and gene_species_map.exists(): - cmd.append(str(gene_species_map)) - print(f" Gene-to-species map: {gene_species_map}") - else: - cmd.append("") - result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=120) - - # 调试输出 - print(f" R脚本返回码: {result.returncode}") - if result.stdout: - print(f" R脚本输出: {result.stdout[:500]}") - if result.stderr: - print(f" R脚本错误: {result.stderr[:500]}") - - # 专用脚本输出固定为 interspecies_tree.pdf/png - output_pdf = output_dir / "interspecies_tree.pdf" - output_png = output_dir / "interspecies_tree.png" - - # 重命名为规范名称 - final_pdf = output_dir / f"{prefix}.interspecies_tree.pdf" - final_png = output_dir / f"{prefix}.interspecies_tree.png" - - if output_pdf.exists(): - shutil.move(output_pdf, final_pdf) - print(f" 进化树PDF: {final_pdf}") - if output_png.exists(): - shutil.move(output_png, final_png) - print(f" 进化树PNG: {final_png}") - - # PDF转PNG（如果没有生成PNG） - if final_pdf.exists() and not final_png.exists(): - png_from_pdf = output_dir / "interspecies_tree.png" - if pdf_to_png(final_pdf, png_from_pdf): - final_png = output_dir / f"{prefix}.interspecies_tree.png" - shutil.move(png_from_pdf, final_png) - print(f" PDF转PNG: {final_png}") - - return { - "success": final_pdf.exists() or final_png.exists(), - "pdf_file": final_pdf if final_pdf.exists() else None, - "png_file": final_png if final_png.exists() else None, - "stdout": result.stdout, - "stderr": result.stderr - } - except Exception as e: - return {"success": False, "error": str(e)} - - -def convert_blast_to_family_simple( - work_dir: Path, - blast_file: Path, - target_species: str, - ref_species: str, -) -> Dict[str, Any]: - """Fallback: convert pairwise BLAST hits directly to JCVI anchors/simple. - - This is not a strict MCScanX collinearity block. It is used when the ref side - only has family proteins, so MCScanX cannot satisfy whole-genome block rules. - """ - pair_prefix = f"{target_species}.{ref_species}" - target_bed = work_dir / f"{target_species}.bed" - ref_bed = work_dir / f"{ref_species}.bed" - anchors_file = work_dir / f"{pair_prefix}.anchors" - simple_file = work_dir / f"{pair_prefix}.anchors.simple" - - valid_target = {line.rstrip("\n").split("\t")[3] for line in open(target_bed) if len(line.rstrip("\n").split("\t")) >= 4} - valid_ref = {line.rstrip("\n").split("\t")[3] for line in open(ref_bed) if len(line.rstrip("\n").split("\t")) >= 4} - seen = set() - links = [] - if not blast_file.exists(): - return {"success": False, "error": "BLAST file missing for family-only fallback"} - with open(blast_file, "r") as f: - for line in f: - if not line.strip() or line.startswith("#"): - continue - parts = line.rstrip("\n").split("\t") - if len(parts) < 2: - continue - a = normalize_seq_id(parts[0]) - b = normalize_seq_id(parts[1]) - target_gene = ref_gene = None - if a in valid_target and b in valid_ref: - target_gene, ref_gene = a, b - elif b in valid_target and a in valid_ref: - target_gene, ref_gene = b, a - if not target_gene or not ref_gene: - continue - key = (target_gene, ref_gene) - if key in seen: - continue - seen.add(key) - score_text = parts[11] if len(parts) > 11 else "1000" - try: - score = str(int(float(score_text))) - except Exception: - score = "1000" - links.append((target_gene, ref_gene, score)) - if not links: - return {"success": False, "error": "No BLAST hits matched pair BED IDs for family-only fallback"} - with open(anchors_file, "w") as out: - out.write("##\n") - for target_gene, ref_gene, score in links: - out.write(f"{target_gene}\t{ref_gene}\t{score}\n") - with open(simple_file, "w") as out: - for target_gene, ref_gene, score in links: - out.write(f"{target_gene}\t{target_gene}\t{ref_gene}\t{ref_gene}\t{score}\t+\n") - return { - "success": True, - "anchors_file": anchors_file, - "simple_file": simple_file, - "anchor_count": len(links), - "fallback": "family_only_blast_links", - } - - -def convert_collinearity_to_simple( - work_dir: Path, - collinearity_file: Path, - target_species: str, - ref_species: str, -) -> Dict[str, Any]: - """Convert one MCScanX .collinearity file into JCVI anchors/simple files.""" - conda_python = "/home/xia-lab/miniconda3/envs/test4py311/bin/python" - pair_prefix = f"{target_species}.{ref_species}" - target_bed = work_dir / f"{target_species}.bed" - ref_bed = work_dir / f"{ref_species}.bed" - anchors_file = work_dir / f"{pair_prefix}.anchors" - anchors_new_file = work_dir / f"{pair_prefix}.anchors.new" - simple_file = work_dir / f"{pair_prefix}.anchors.simple" - - valid_target = {line.rstrip("\n").split("\t")[3] for line in open(target_bed) if len(line.rstrip("\n").split("\t")) >= 4} - valid_ref = {line.rstrip("\n").split("\t")[3] for line in open(ref_bed) if len(line.rstrip("\n").split("\t")) >= 4} - - current_block = [] - anchors = 0 - filtered = 0 - with open(collinearity_file, "r") as f_in, open(anchors_file, "w") as f_out: - for line in f_in: - if not line.strip(): - continue - if line.startswith("## Alignment"): - if current_block: - f_out.write("##\n") - f_out.writelines(current_block) - current_block = [] - continue - if line.startswith("#"): - continue - parts = line.strip().split("\t") - if len(parts) < 3: - continue - gene_a = normalize_seq_id(parts[1]) - gene_b = normalize_seq_id(parts[2]) - target_gene = ref_gene = None - if gene_a in valid_target and gene_b in valid_ref: - target_gene, ref_gene = gene_a, gene_b - elif gene_b in valid_target and gene_a in valid_ref: - target_gene, ref_gene = gene_b, gene_a - if target_gene and ref_gene: - current_block.append(f"{target_gene} {ref_gene} 1000\n") - anchors += 1 - else: - filtered += 1 - if current_block: - f_out.write("##\n") - f_out.writelines(current_block) - - if anchors == 0: - return {"success": False, "error": "No anchors matched pair BED IDs", "anchors_file": anchors_file, "filtered": filtered} - - cmd_screen = [ - conda_python, "-m", "jcvi.compara.synteny", "screen", - "--minspan=10", "--simple", str(anchors_file), str(anchors_new_file), - f"--qbed={target_bed.name}", f"--sbed={ref_bed.name}", - ] - result = subprocess.run(cmd_screen, capture_output=True, text=True, cwd=str(work_dir), check=False, timeout=180) - for candidate in (work_dir / f"{pair_prefix}.anchors.new.simple", work_dir / f"{pair_prefix}.new.simple"): - if candidate.exists(): - shutil.copy(candidate, simple_file) - break - if not simple_file.exists(): - shutil.copy(anchors_file, simple_file) - - return { - "success": simple_file.exists() and simple_file.stat().st_size > 0, - "anchors_file": anchors_file, - "anchors_new_file": anchors_new_file, - "simple_file": simple_file, - "anchor_count": anchors, - "filtered_count": filtered, - "screen_returncode": result.returncode, - "screen_stderr": result.stderr, - } - - -def _chr_sort_key(chrom: str): - cleaned = re.sub(r"^(chr|Chr|CHR)", "", chrom) - if cleaned.isdigit(): - return (0, int(cleaned)) - return (1, cleaned) - - -def _read_bed_chroms(bed_file: Path) -> List[str]: - skip = {"C", "M", "ChrC", "ChrM", "chrc", "chrm"} - chroms = set() - with open(bed_file, "r") as f: - for line in f: - parts = line.strip().split("\t") - if len(parts) >= 4 and parts[0] not in skip: - chroms.add(parts[0]) - return sorted(chroms, key=_chr_sort_key) - - -def _mark_simple_file(simple_file: Path, marked_file: Path, family_ids: set) -> bool: - if not simple_file.exists() or not family_ids: - return False - wrote = False - with open(simple_file, "r") as f_in, open(marked_file, "w") as f_out: - for line in f_in: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - f_out.write(line) - continue - parts = stripped.split() - normalized = {normalize_seq_id(p.lstrip("r*")) for p in parts[:4]} - if normalized & family_ids and not parts[0].startswith("r*"): - parts[0] = "r*" + parts[0] - wrote = True - f_out.write("\t".join(parts) + "\n") - else: - f_out.write(line) - return wrote - - -def plot_multi_species_synteny( - work_dir: Path, - output_dir: Path, - prefix: str, - species_order: List[str], - pair_results: List[Dict[str, Any]], - family_ids: set, -) -> Dict[str, Any]: - """Draw one multi-track JCVI karyotype figure from all successful target-ref pairs.""" - print("Plotting multi-species JCVI synteny...") - conda_python = "/home/xia-lab/miniconda3/envs/test4py311/bin/python" - usable_pairs = [p for p in pair_results if p.get("simple_file") and Path(p["simple_file"]).exists()] - if not usable_pairs: - return {"success": False, "error": "No pairwise anchors.simple files available for JCVI"} - - seqids_file = work_dir / f"{prefix}.seqids" - layout_file = work_dir / f"{prefix}.layout" - layout_mark_file = work_dir / f"{prefix}.layout_mark" - - species_with_bed = [] - with open(seqids_file, "w") as f: - for species in species_order: - bed = work_dir / f"{species}.bed" - if not bed.exists(): - continue - chroms = _read_bed_chroms(bed) - if not chroms: - continue - species_with_bed.append(species) - f.write(",".join(chroms) + "\n") - - if len(species_with_bed) < 2: - return {"success": False, "error": "Fewer than two species BED tracks are available"} - - index = {species: i for i, species in enumerate(species_with_bed)} - n = len(species_with_bed) - y_values = [round(0.85 - (0.7 * i / max(n - 1, 1)), 3) for i in range(n)] - palette = ["#d73027", "#1a9850", "#4575b4", "#984ea3", "#ff7f00", "#4daf4a", "#377eb8", "#a65628"] - - def write_layout(layout: Path, use_marked: bool) -> None: - with open(layout, "w") as f: - f.write("# y, xstart, xend, rotation, color, label, va, bed\n") - for i, species in enumerate(species_with_bed): - va = "top" if i == 0 else "bottom" - f.write(f"{y_values[i]}, .08, .92, 0, {palette[i % len(palette)]}, {species}, {va}, {species}.bed\n") - f.write("# edges\n") - for pair in usable_pairs: - target = pair["target_species"] - ref = pair["ref_species"] - if target not in index or ref not in index: - continue - edge = Path(pair.get("marked_simple_file") if use_marked and pair.get("marked_simple_file") else pair["simple_file"]).name - f.write(f"e, {index[target]}, {index[ref]}, {edge}\n") - - for pair in usable_pairs: - simple = Path(pair["simple_file"]) - marked = work_dir / f"{pair['target_species']}.{pair['ref_species']}.anchors.simple.mark" - if _mark_simple_file(simple, marked, family_ids): - pair["marked_simple_file"] = str(marked) - - write_layout(layout_file, use_marked=False) - write_layout(layout_mark_file, use_marked=True) - - def run_plot(layout: Path, final_pdf: Path, final_png: Path) -> Dict[str, Any]: - for stale in [work_dir / "karyotype.pdf", work_dir / "karyotype.png"]: - if stale.exists(): - stale.unlink() - cmd = [conda_python, "-m", "jcvi.graphics.karyotype", str(seqids_file.resolve()), str(layout.resolve()), "--notex"] - result = subprocess.run(cmd, capture_output=True, text=True, cwd=str(work_dir), check=False, timeout=180) - if (work_dir / "karyotype.pdf").exists(): - shutil.copy(work_dir / "karyotype.pdf", final_pdf.resolve()) - if (work_dir / "karyotype.png").exists(): - shutil.copy(work_dir / "karyotype.png", final_png.resolve()) - if final_pdf.exists() and not final_png.exists(): - pdf_to_png(final_pdf, final_png) - return {"returncode": result.returncode, "stdout": result.stdout, "stderr": result.stderr} - - final_pdf = output_dir / f"{prefix}.interspecies_synteny.pdf" - final_png = output_dir / f"{prefix}.interspecies_synteny.png" - final_mark_pdf = output_dir / f"{prefix}.interspecies_synteny.marked.pdf" - final_mark_png = output_dir / f"{prefix}.interspecies_synteny.marked.png" - - base_result = run_plot(layout_file, final_pdf, final_png) - mark_result = run_plot(layout_mark_file, final_mark_pdf, final_mark_png) - - return { - "success": final_pdf.exists() or final_png.exists(), - "pdf_file": final_pdf if final_pdf.exists() else None, - "png_file": final_png if final_png.exists() else None, - "marked_pdf_file": final_mark_pdf if final_mark_pdf.exists() else None, - "marked_png_file": final_mark_png if final_mark_png.exists() else None, - "seqids_file": seqids_file, - "layout_file": layout_file, - "layout_mark_file": layout_mark_file, - "base_plot": base_result, - "marked_plot": mark_result, - } - - -def write_report(workspace: Path, prefix: str, metrics: Dict, parameters: Dict, - tool_versions: Dict, start_time: datetime, end_time: datetime, - elapsed_sec: float, cmd: List[str]) -> Path: - """写入报告""" - report_path = workspace / "final" / f"{prefix}.interspecies_synteny_report.md" - - # 收集可复现性信息 - reproducibility_info = [] - reproducibility_info.append("## 可复现性信息\n\n") - reproducibility_info.append("### 执行环境\n") - reproducibility_info.append(f"- **分析时间**: {start_time.isoformat()}\n") - reproducibility_info.append(f"- **完成时间**: {end_time.isoformat()}\n") - reproducibility_info.append(f"- **运行时长**: {elapsed_sec:.2f} 秒\n") - reproducibility_info.append(f"- **工作目录**: {workspace}\n") - reproducibility_info.append(f"- **输入目录**: {parameters.get('input_dir', 'N/A')}\n\n") - - reproducibility_info.append("### 工具版本\n") - for tool, version in tool_versions.items(): - reproducibility_info.append(f"- **{tool}**: {version}\n") - reproducibility_info.append("\n") - - reproducibility_info.append("### 执行命令\n") - reproducibility_info.append(f"```bash\n{' '.join(cmd)}\n```\n\n") - - reproducibility_info.append("### 输入参数\n") - for key, value in parameters.items(): - if key != 'input_dir': - reproducibility_info.append(f"- **{key}**: {value}\n") - reproducibility_info.append("\n") - - with open(report_path, "w") as f: - f.write(f"# 物种间共线性分析 - {prefix}\n\n") - f.write(f"**时间**: {datetime.now().isoformat()}\n\n") - f.write(f"- 目标物种 (研究对象): {parameters.get('target_species')}\n") - f.write(f"- 参考物种 (对照): {parameters.get('ref_species')}\n") - f.write(f"- 目标蛋白文件: {parameters.get('target_protein_spl')}\n") - f.write(f"- 参考蛋白文件: {parameters.get('ref_protein_spl')}\n") - f.write(f"- 目标全蛋白组: {parameters.get('target_protein_all')}\n") - f.write(f"- 参考全蛋白组: {parameters.get('ref_protein_all')}\n") - f.write("\n") - f.writelines(reproducibility_info) - return report_path - - -def get_file_info(file_path: Path) -> Dict[str, Any]: - """获取文件的详细信息，用于artifact规范""" - info = { - "size": 0, - "sha256": "", - "mtime": 0 - } - if file_path.exists(): - info["size"] = file_path.stat().st_size - info["mtime"] = file_path.stat().st_mtime - info["sha256"] = compute_file_hash(file_path) - return info - - -def write_manifest(workspace: Path, job_id: str, run_id: str, prefix: str, - start_time: datetime, end_time: datetime, artifacts: Dict, - metrics: Dict, parameters: Dict, tool_versions: Dict, - workspace_root: Path, input_dir: Path, cmd: List[str], - status: str = "success") -> Path: - """写入manifest - 符合schema v1标准""" - elapsed_sec = (end_time - start_time).total_seconds() - - # 构建符合规范的artifacts结构 - formatted_artifacts = {} - for key, value in artifacts.items(): - if isinstance(value, dict): - # 已经是字典格式，确保包含必需字段 - formatted_artifacts[key] = value - elif isinstance(value, str): - # 字符串格式，转换为artifact对象 - file_path = workspace / value - file_info = get_file_info(file_path) - formatted_artifacts[key] = { - "artifact": key, - "path": value, - "full_path": str(file_path.absolute()), - "type": "file", - "size": file_info["size"], - "sha256": file_info["sha256"], - "mtime": file_info["mtime"] - } - - manifest_data = { - "manifest_version": "1.0", - "job_id": job_id, - "stage": "interspecies_synteny", - "run_id": run_id, - "prefix": prefix, - "timestamp": end_time.isoformat(), - "status": status, - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - "elapsed_sec": round(elapsed_sec, 2), - "cmd": cmd, - "workspace_root": str(workspace_root.absolute()), - "workspace": str(workspace.absolute()), - "input_dir": str(input_dir.absolute()), - "tool_versions": tool_versions, - "artifacts": formatted_artifacts, - "parameters": parameters, - "metrics": metrics - } - - manifest_path = workspace / "final" / f"{prefix}.interspecies_synteny.manifest.json" - with open(manifest_path, "w", encoding='utf-8') as f: - json.dump(manifest_data, f, indent=2, ensure_ascii=False) - - return manifest_path - - - -def _clean_bool(value: Any) -> bool: - return str(value).strip().lower() in {"true", "1", "yes", "y"} - - -def _gene_family_from_prefix(prefix: str) -> str: - return prefix.split('.', 1)[1] if '.' in prefix else prefix - - -def _resolve_path(path_text: str, input_dir: Path) -> Optional[Path]: - if not path_text: - return None - path = Path(path_text) - if path.is_absolute(): - return path if path.exists() else None - for candidate in (input_dir / path, input_dir / path.name): - if candidate.exists(): - return candidate - return None - - -def _read_config_groups(input_dir: Path, default_family: str) -> Dict[str, Any]: - config_path = input_dir / "config.xlsx" - groups = {"target": {}, "ref": {}, "gene_family": default_family} - if not config_path.exists(): - return groups - try: - import pandas as pd - df = pd.read_excel(config_path) - except Exception as exc: - print(f"[WARN] Failed to read config.xlsx: {exc}") - return groups - for _, row in df.iterrows(): - species = str(row.get("Species_Code", "")).strip() - family = str(row.get("Gene_Family", default_family)).strip() or default_family - file_type = str(row.get("File_Type", "")).strip().lower() - original_name = str(row.get("Original_File_Name", "")).strip() - if not species: - continue - group_name = "ref" if _clean_bool(row.get("Ref", False)) else "target" - item = groups[group_name].setdefault(species, {"species": species, "gene_family": family, "files": {}}) - item["gene_family"] = family - if file_type: - item["files"][file_type] = original_name - groups["gene_family"] = family or groups["gene_family"] - return groups - - -def _read_genome_prepare_artifacts(workspace: Path, input_dir: Path, prefix: str) -> Dict[str, Dict[str, Path]]: - result: Dict[str, Dict[str, Path]] = {} - for manifest_path in (workspace / "final" / f"{prefix}.genome_prepare.manifest.json", input_dir / "final" / f"{prefix}.genome_prepare.manifest.json"): - if not manifest_path.exists(): - continue - try: - data = json.loads(manifest_path.read_text(encoding="utf-8")) - except Exception as exc: - print(f"[WARN] Failed to read genome_prepare manifest {manifest_path}: {exc}") - continue - for key, artifact in (data.get("artifacts") or {}).items(): - kind = artifact.get("artifact") - species = key.rsplit("_", 1)[0] - found = _resolve_path(artifact.get("full_path") or artifact.get("path") or "", input_dir) - if found and kind: - result.setdefault(species, {})[kind] = found - return result - - -def _find_species_file(input_dir: Path, species: str, family: str, file_type: str, original_name: str = "") -> Optional[Path]: - if file_type == "all_pep": - patterns = [ - f"{species}.{family}.all_pep.fasta", f"{species}.{family}.all_pep.fa", - f"{species}.{family}.protein.fasta", f"{species}.{family}.faa", - f"{species}.{family}.pep.fasta", f"{species}.{family}.pep.fa", - f"{species}.all_pep.fasta", f"{species}.pep.fasta", f"{species}.protein.fasta", f"{species}.faa", - ] - elif file_type == "family_pep": - patterns = [ - f"{species}.{family}.family_pep.fasta", f"{species}.{family}.family_pep.fa", - f"{species}.{family}.pep.fasta", f"{species}.{family}.pep.fa", - ] - elif file_type == "pep": - patterns = [ - f"{species}.{family}.pep.fasta", f"{species}.{family}.pep.fa", - f"{species}.{family}.protein.fasta", f"{species}.{family}.faa", - f"{species}.pep.fasta", f"{species}.pep.fa", f"{species}.protein.fasta", f"{species}.faa", - ] - elif file_type == "gff": - patterns = [ - f"{species}.{family}_final.gff3", f"{species}.{family}.gff3", f"{species}.{family}.gff", - f"{species}_final.gff3", f"{species}.gff3", f"{species}.gff", - ] - else: - patterns = [] - if original_name: - patterns.append(Path(original_name).name) - for pattern in patterns: - for candidate in (input_dir / pattern, input_dir / species / pattern): - if candidate.exists(): - return candidate - return None - - -def discover_group_inputs(workspace: Path, input_dir: Path, prefix: str, args) -> Dict[str, Any]: - family = _gene_family_from_prefix(prefix) - groups = _read_config_groups(input_dir, family) - artifacts = _read_genome_prepare_artifacts(workspace, input_dir, prefix) - family = groups.get("gene_family") or family - - for group_name in ("target", "ref"): - for species, item in groups[group_name].items(): - files = item.setdefault("resolved", {}) - artifact_paths = artifacts.get(species, {}) - original_files = item.get("files", {}) - item_family = item.get("gene_family", family) - all_original = original_files.get("all_pep", "") or original_files.get("protein", "") - family_original = original_files.get("family_pep", "") - legacy_pep_original = original_files.get("pep", "") - if group_name == "ref" and legacy_pep_original and not family_original: - family_original = legacy_pep_original - if group_name == "target" and legacy_pep_original and not all_original: - all_original = legacy_pep_original - files["all_pep"] = ( - _find_species_file(input_dir, species, item_family, "all_pep", all_original) or - artifact_paths.get("all_pep") or artifact_paths.get("ref_all_pep") or artifact_paths.get("pep") - ) - files["family_pep"] = ( - _find_species_file(input_dir, species, item_family, "family_pep", family_original) or - artifact_paths.get("family_pep") or artifact_paths.get("ref_family_pep") or artifact_paths.get("ref_pep") - ) - files["pep"] = files.get("all_pep") or files.get("family_pep") - files["gff"] = ( - artifact_paths.get("gff") or artifact_paths.get("ref_gff") or - _find_species_file(input_dir, species, item_family, "gff", original_files.get("gff", "")) - ) - - target_family_protein = Path(args.target_protein) if args.target_protein else workspace / "final" / f"{prefix}.final_members.pep.fa" - ref_family_protein = Path(args.ref_protein) if args.ref_protein else None - if ref_family_protein is None: - for candidate in [input_dir / f"Ref_combined.{family}.pep.fasta", input_dir / f"Ref_combined.{family}.pep.fa", input_dir / "Ref_combined.pep.fasta"]: - if candidate.exists(): - ref_family_protein = candidate - break - if ref_family_protein is None: - ref_peps = [item.get("resolved", {}).get("family_pep") for item in groups["ref"].values() if item.get("resolved", {}).get("family_pep")] - if ref_peps: - ref_family_protein = workspace / "interspecies_synteny" / "analysis" / f"Ref_combined.{family}.pep.fasta" - merge_many_fastas(ref_peps, ref_family_protein) - - target_all_peps = {sp: item.get("resolved", {}).get("all_pep") for sp, item in groups["target"].items() if item.get("resolved", {}).get("all_pep")} - ref_all_peps = {sp: item.get("resolved", {}).get("all_pep") for sp, item in groups["ref"].items() if item.get("resolved", {}).get("all_pep")} - target_gffs = {sp: item.get("resolved", {}).get("gff") for sp, item in groups["target"].items() if item.get("resolved", {}).get("gff")} - ref_gffs = {sp: item.get("resolved", {}).get("gff") for sp, item in groups["ref"].items() if item.get("resolved", {}).get("gff")} - - return { - "gene_family": family, - "target_species": sorted(groups["target"].keys()), - "ref_species": sorted(groups["ref"].keys()), - "target_family_protein": target_family_protein, - "ref_family_protein": ref_family_protein, - "target_all_peps": target_all_peps, - "ref_all_peps": ref_all_peps, - "target_gffs": target_gffs, - "ref_gffs": ref_gffs, - "groups": groups, - } - - -def merge_many_fastas(files: List[Path], output_file: Path) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as out: - for fasta in files: - with open(fasta, "r") as f: - out.write(f.read().rstrip() + "\n") - return output_file - - -def _parse_gff_attr(attr: str) -> Dict[str, str]: - result = {} - for item in attr.strip().strip(';').split(';'): - item = item.strip() - if not item: - continue - if '=' in item: - k, v = item.split('=', 1) - elif ' ' in item: - k, v = item.split(' ', 1) - else: - continue - result[k.strip()] = v.strip().strip('"') - return result - - -def _match_gff_id(attrs: Dict[str, str], protein_ids: set) -> Optional[str]: - candidates = [attrs.get("ID"), attrs.get("transcript_id"), attrs.get("Name"), attrs.get("Parent")] - for candidate in candidates: - if not candidate: - continue - for item in str(candidate).split(','): - norm = normalize_seq_id(item) - if norm in protein_ids: - return norm - for variant in id_variants(norm): - if variant in protein_ids: - return variant - return None - - -def build_species_position_files(species: str, gff3: Path, protein_fasta: Path, work_dir: Path) -> Dict[str, Any]: - """Create MCScanX simplified GFF and JCVI BED for one species.""" - simple_gff = work_dir / f"{species}.gff" - bed_file = work_dir / f"{species}.bed" - protein_ids = fasta_id_set(protein_fasta) - seen = set() - count = 0 - features_seen = 0 - work_dir.mkdir(parents=True, exist_ok=True) - with open(gff3, "r") as f_in, open(simple_gff, "w") as gff_out, open(bed_file, "w") as bed_out: - for line in f_in: - if not line.strip() or line.startswith('#'): - continue - parts = line.rstrip('\n').split(' ') - if len(parts) < 9 or parts[2] not in {"mRNA", "transcript", "gene"}: - continue - features_seen += 1 - attrs = _parse_gff_attr(parts[8]) - gene_id = _match_gff_id(attrs, protein_ids) - if not gene_id or gene_id in seen: - continue - seen.add(gene_id) - chrom = parts[0] - start, end = parts[3], parts[4] - gff_out.write(f"{chrom} {gene_id} {start} {end}\n") - bed_out.write(f"{chrom} {start} {end} {gene_id}\n") - count += 1 - return { - "success": count > 0, - "species": species, - "gff": simple_gff, - "bed": bed_file, - "gene_count": count, - "features_seen": features_seen, - "protein_id_count": len(protein_ids), - "error": "No GFF IDs matched protein FASTA IDs" if count == 0 else "", - } - - -def collect_family_ids(target_family_fasta: Path, ref_family_fasta: Path, final_id_file: Optional[Path] = None) -> set: - ids = set() - for fasta in [target_family_fasta, ref_family_fasta]: - if fasta and Path(fasta).exists(): - ids.update(fasta_id_set(Path(fasta))) - if final_id_file and final_id_file.exists(): - with open(final_id_file, "r") as f: - for line in f: - if line.strip() and not line.startswith('#'): - ids.update(id_variants(line.strip().split()[0])) - return ids - - -def add_artifact(artifacts: Dict[str, Dict], key: str, file_path: Path, rel_path: str, artifact: str, file_type: str) -> None: - if not file_path or not file_path.exists(): - return - info = get_file_info(file_path) - artifacts[key] = { - "artifact": artifact, - "path": rel_path, - "full_path": str(file_path.absolute()), - "type": file_type, - "size": info["size"], - "sha256": info["sha256"], - "mtime": info["mtime"], - } - - -def write_failure_manifest(workspace: Path, job_id: str, run_id: str, prefix: str, start_time: datetime, - workspace_root: Path, input_dir: Path, cmd: List[str], error: str) -> Path: - end_time = datetime.now() - return write_manifest( - workspace, job_id, run_id, prefix, start_time, end_time, - artifacts={}, metrics={"error": error}, parameters={"prefix": prefix, "error": error}, - tool_versions={}, workspace_root=workspace_root, input_dir=input_dir, cmd=cmd, status="failed" - ) - - -def main(): - parser = argparse.ArgumentParser(description="????????") - parser.add_argument("--prefix", required=True, help="?????? Multi.SPL") - parser.add_argument("--workspace", required=True) - parser.add_argument("--input-dir") - parser.add_argument("--ref-species", default=None, help="???????? config.xlsx ??") - parser.add_argument("--target-species", default=None, help="???????? config.xlsx ??") - parser.add_argument("--ref-protein", help="?????????") - parser.add_argument("--target-protein", help="?????????") - parser.add_argument("--cpu", type=int, default=8) - parser.add_argument("--log-file", help="??????????????") - parser.add_argument("--workspace-root", help="??????????manifest?") - args = parser.parse_args() - - start_time = datetime.now() - run_id = f"interspecies_synteny_{start_time.strftime('%Y%m%d_%H%M%S')}" - workspace = Path(args.workspace) - job_id = workspace.name - input_dir = Path(args.input_dir) if args.input_dir else workspace.parent.parent / "input" / job_id - workspace_root = Path(args.workspace_root) if args.workspace_root else workspace.parent - cmd = sys.argv - - print("=" * 60) - print("Interspecies synteny analysis") - print("=" * 60) - - try: - paths = prepare_directories(workspace) - tool_versions = get_tool_versions() - discovery = discover_group_inputs(workspace, input_dir, args.prefix, args) - target_group = "Target_combined" - ref_group = "Ref_combined" - print(f"Target species: {', '.join(discovery['target_species']) or 'NONE'}") - print(f"Reference species: {', '.join(discovery['ref_species']) or 'NONE'}") - print(f"Target family protein: {discovery['target_family_protein']}") - print(f"Ref family protein: {discovery['ref_family_protein']}") - - if not discovery["target_family_protein"] or not Path(discovery["target_family_protein"]).exists(): - raise RuntimeError(f"Target family protein missing: {discovery['target_family_protein']}") - if not discovery["ref_family_protein"] or not Path(discovery["ref_family_protein"]).exists(): - raise RuntimeError("Reference family protein missing; expected Ref_combined..pep.fasta or Ref species pep files") - - artifacts = {} - metrics = { - "target_species": discovery["target_species"], - "ref_species": discovery["ref_species"], - "tree_success": False, - "synteny_success": False, - "synteny_skipped_reason": "", - } - - print("\nMerging family proteins for tree...") - merged_fasta = paths["analysis_dir"] / f"{args.prefix}.target_ref.family.pep.fa" - merge_result = merge_protein_sequences(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), merged_fasta) - if not merge_result.get("success"): - raise RuntimeError(f"Failed to merge family proteins: {merge_result.get('stderr')}") - metrics["target_family_count"] = merge_result.get("target_count") - metrics["ref_family_count"] = merge_result.get("ref_count") - - print("\nBuilding interspecies tree...") - tree_result = build_phylogenetic_tree(merged_fasta, paths["analysis_dir"], args.prefix, args.cpu) - if not tree_result.get("tree_file"): - raise RuntimeError(f"Tree construction failed: {tree_result.get('error', 'unknown error')}") - final_tree = paths["final_dir"] / f"{args.prefix}.interspecies_tree.nwk" - shutil.copy(Path(tree_result["tree_file"]), final_tree) - metrics["tree_success"] = True - add_artifact(artifacts, "tree_nwk", final_tree, f"final/{args.prefix}.interspecies_tree.nwk", "tree_nwk", "tree") - - gene_species_map = paths["analysis_dir"] / f"{args.prefix}.gene_to_species.tsv" - write_gene_species_map(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), discovery["target_species"], discovery["ref_species"], gene_species_map, collect_species_id_sets(discovery["groups"])) - add_artifact(artifacts, "gene_to_species", gene_species_map, f"interspecies_synteny/analysis/{args.prefix}.gene_to_species.tsv", "gene_to_species", "tsv") - - print("\nPlotting interspecies tree...") - target_subfamily_file = input_dir / f"{args.prefix}.geneID.subfamily" - if not target_subfamily_file.exists(): - target_subfamily_file = workspace / "final" / f"{args.prefix}.geneID.subfamily" - ref_subfamily_file = input_dir / f"SPL_{discovery['ref_species'][0]}.subfamily" if discovery["ref_species"] else input_dir / "SPL_ref.subfamily" - if not ref_subfamily_file.exists(): - candidates = sorted(input_dir.glob("SPL_*.subfamily")) + sorted(input_dir.glob("*.SPL.subfamily")) - ref_subfamily_file = candidates[0] if candidates else ref_subfamily_file - tree_plot_result = plot_interspecies_tree( - final_tree, - paths["final_dir"], - args.prefix, - target_group, - ref_group, - target_subfamily_file if target_subfamily_file.exists() else None, - ref_subfamily_file if ref_subfamily_file.exists() else None, - gene_species_map, - ) - if tree_plot_result.get("pdf_file"): - add_artifact(artifacts, "tree_pdf", tree_plot_result["pdf_file"], f"final/{args.prefix}.interspecies_tree.pdf", "tree_pdf", "plot") - if tree_plot_result.get("png_file"): - add_artifact(artifacts, "tree_png", tree_plot_result["png_file"], f"final/{args.prefix}.interspecies_tree.png", "tree_png", "plot") - - synteny_plot_result = {"success": False} - pair_results: List[Dict[str, Any]] = [] - species_position_results: Dict[str, Dict[str, Any]] = {} - - missing = [] - if not discovery["target_all_peps"]: - missing.append("target all protein") - if not discovery["ref_all_peps"]: - missing.append("reference all protein") - if not discovery["target_gffs"]: - missing.append("target GFF") - if not discovery["ref_gffs"]: - missing.append("reference GFF") - - if not missing: - print("\nPreparing per-species GFF/BED files for pairwise MCScanX...") - all_species = discovery["target_species"] + discovery["ref_species"] - for species in all_species: - pep = discovery["target_all_peps"].get(species) or discovery["ref_all_peps"].get(species) - gff = discovery["target_gffs"].get(species) or discovery["ref_gffs"].get(species) - if not pep or not gff: - species_position_results[species] = {"success": False, "error": "missing pep or gff"} - continue - result = build_species_position_files(species, Path(gff), Path(pep), paths["analysis_dir"]) - species_position_results[species] = result - if result.get("success"): - add_artifact(artifacts, f"{species}_mcscanx_gff", result["gff"], f"interspecies_synteny/analysis/{species}.gff", "mcscanx_gff", "gff") - add_artifact(artifacts, f"{species}_bed", result["bed"], f"interspecies_synteny/analysis/{species}.bed", "jcvi_bed", "bed") - print(f" {species}: {result.get('gene_count', 0)} matched genes") - - print("\nRunning pairwise target-ref MCScanX...") - for target_species in discovery["target_species"]: - for ref_species in discovery["ref_species"]: - pair_key = f"{target_species}.{ref_species}" - pair = {"pair": pair_key, "target_species": target_species, "ref_species": ref_species, "success": False} - target_ready = species_position_results.get(target_species, {}).get("success") - ref_ready = species_position_results.get(ref_species, {}).get("success") - target_pep = discovery["target_all_peps"].get(target_species) - ref_pep = discovery["ref_all_peps"].get(ref_species) - if not (target_ready and ref_ready and target_pep and ref_pep): - pair["error"] = "missing matched GFF/BED or protein FASTA" - pair_results.append(pair) - print(f" {pair_key}: skipped ({pair['error']})") - continue - blast_result = run_reciprocal_blast(Path(target_pep), Path(ref_pep), paths["analysis_dir"], target_species, ref_species, args.cpu) - pair["blast_success"] = bool(blast_result.get("success")) - if blast_result.get("output_file"): - pair["blast_file"] = str(blast_result["output_file"]) - add_artifact(artifacts, f"{pair_key}_blast", blast_result["output_file"], f"interspecies_synteny/analysis/{pair_key}.blast", "blast", "data") - if not blast_result.get("success"): - pair["error"] = "BLAST failed or produced no hits" - pair_results.append(pair) - print(f" {pair_key}: {pair['error']}") - continue - mcscanx_result = run_mcscanx_interspecies(paths["analysis_dir"], target_species, ref_species) - pair_gff = paths["analysis_dir"] / f"{pair_key}.gff" - if pair_gff.exists(): - pair["pair_gff"] = str(pair_gff) - add_artifact(artifacts, f"{pair_key}_gff", pair_gff, f"interspecies_synteny/analysis/{pair_key}.gff", "pair_mcscanx_gff", "gff") - pair["mcscanx_success"] = bool(mcscanx_result.get("success")) - if mcscanx_result.get("collinearity_file"): - pair["collinearity_file"] = str(mcscanx_result["collinearity_file"]) - add_artifact(artifacts, f"{pair_key}_collinearity", mcscanx_result["collinearity_file"], f"interspecies_synteny/analysis/{pair_key}.collinearity", "collinearity", "data") - if not mcscanx_result.get("success"): - pair["error"] = "MCScanX did not produce collinearity" - pair_results.append(pair) - print(f" {pair_key}: {pair['error']}") - continue - simple_result = convert_collinearity_to_simple(paths["analysis_dir"], Path(mcscanx_result["collinearity_file"]), target_species, ref_species) - if not simple_result.get("success") and blast_result.get("output_file"): - fallback_result = convert_blast_to_family_simple(paths["analysis_dir"], Path(blast_result["output_file"]), target_species, ref_species) - if fallback_result.get("success"): - fallback_result["note"] = "MCScanX produced no whole-genome blocks; using family-only BLAST links for JCVI visualization" - simple_result = fallback_result - pair.update({k: str(v) if isinstance(v, Path) else v for k, v in simple_result.items()}) - pair["success"] = bool(simple_result.get("success")) - if simple_result.get("anchors_file"): - add_artifact(artifacts, f"{pair_key}_anchors", simple_result["anchors_file"], f"interspecies_synteny/analysis/{pair_key}.anchors", "anchors", "data") - if simple_result.get("simple_file"): - add_artifact(artifacts, f"{pair_key}_anchors_simple", simple_result["simple_file"], f"interspecies_synteny/analysis/{pair_key}.anchors.simple", "anchors_simple", "data") - pair_results.append(pair) - print(f" {pair_key}: {'success' if pair['success'] else 'anchors conversion failed'}") - - family_ids = collect_family_ids(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), workspace / "final" / f"{args.prefix}.final_members.id") - synteny_plot_result = plot_multi_species_synteny( - paths["analysis_dir"], paths["final_dir"], args.prefix, - discovery["target_species"] + discovery["ref_species"], pair_results, family_ids, - ) - successful_pairs = [p for p in pair_results if p.get("success")] - metrics["synteny_success"] = bool(synteny_plot_result.get("success")) and len(successful_pairs) == len(pair_results) and bool(pair_results) - if not successful_pairs: - metrics["synteny_skipped_reason"] = synteny_plot_result.get("error") or "No target-ref pair produced JCVI anchors" - else: - metrics["synteny_skipped_reason"] = "Missing full data for MCScanX: " + ", ".join(missing) - print(metrics["synteny_skipped_reason"]) - - metrics["species_position_results"] = { - sp: {k: str(v) if isinstance(v, Path) else v for k, v in result.items()} - for sp, result in species_position_results.items() - } - metrics["pairwise_synteny"] = pair_results - metrics["pair_success_count"] = sum(1 for p in pair_results if p.get("success")) - metrics["pair_total_count"] = len(pair_results) - - if synteny_plot_result.get("pdf_file"): - add_artifact(artifacts, "synteny_pdf", synteny_plot_result["pdf_file"], f"final/{args.prefix}.interspecies_synteny.pdf", "synteny_pdf", "plot") - if synteny_plot_result.get("png_file"): - add_artifact(artifacts, "synteny_png", synteny_plot_result["png_file"], f"final/{args.prefix}.interspecies_synteny.png", "synteny_png", "plot") - if synteny_plot_result.get("marked_pdf_file"): - add_artifact(artifacts, "synteny_marked_pdf", synteny_plot_result["marked_pdf_file"], f"final/{args.prefix}.interspecies_synteny.marked.pdf", "synteny_marked_pdf", "plot") - if synteny_plot_result.get("marked_png_file"): - add_artifact(artifacts, "synteny_marked_png", synteny_plot_result["marked_png_file"], f"final/{args.prefix}.interspecies_synteny.marked.png", "synteny_marked_png", "plot") - if synteny_plot_result.get("seqids_file"): - add_artifact(artifacts, "jcvi_seqids", synteny_plot_result["seqids_file"], f"interspecies_synteny/analysis/{args.prefix}.seqids", "jcvi_seqids", "data") - if synteny_plot_result.get("layout_file"): - add_artifact(artifacts, "jcvi_layout", synteny_plot_result["layout_file"], f"interspecies_synteny/analysis/{args.prefix}.layout", "jcvi_layout", "data") - - end_time = datetime.now() - status = "success" if metrics["tree_success"] and metrics["synteny_success"] else "partial_success" - parameters = { - "prefix": args.prefix, - "gene_family": discovery["gene_family"], - "target_group": target_group, - "ref_group": ref_group, - "target_species": discovery["target_species"], - "ref_species": discovery["ref_species"], - "target_protein_spl": str(discovery["target_family_protein"]), - "ref_protein_spl": str(discovery["ref_family_protein"]), - "input_dir": str(input_dir), - "cpu": args.cpu, - } - report_path = write_report(workspace, args.prefix, metrics, parameters, tool_versions, start_time, end_time, (end_time - start_time).total_seconds(), cmd) - add_artifact(artifacts, "report", report_path, f"final/{args.prefix}.interspecies_synteny_report.md", "report", "report") - manifest_path = write_manifest(workspace, job_id, run_id, args.prefix, start_time, end_time, artifacts, metrics, parameters, tool_versions, workspace_root, input_dir, cmd, status=status) - add_artifact(artifacts, "manifest", manifest_path, f"final/{args.prefix}.interspecies_synteny.manifest.json", "manifest", "json") - write_manifest(workspace, job_id, run_id, args.prefix, start_time, end_time, artifacts, metrics, parameters, tool_versions, workspace_root, input_dir, cmd, status=status) - print(f"Interspecies synteny finished with status: {status}") - print(f"Manifest: {manifest_path}") - return 0 if metrics["tree_success"] else 1 - except Exception as exc: - print(f"Interspecies synteny failed: {exc}") - try: - write_failure_manifest(workspace, job_id, run_id, args.prefix, start_time, workspace_root, input_dir, cmd, str(exc)) - except Exception as manifest_exc: - print(f"Failed to write failure manifest: {manifest_exc}") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bioflow_kernel/cli/run_interspecies_synteny.py.bak_pairwise_20260504_165043 b/bioflow_kernel/cli/run_interspecies_synteny.py.bak_pairwise_20260504_165043 deleted file mode 100755 index c453efa..0000000 --- a/bioflow_kernel/cli/run_interspecies_synteny.py.bak_pairwise_20260504_165043 +++ /dev/null @@ -1,1265 +0,0 @@ -#!/usr/bin/env python3 -""" -Interspecies Synteny阶段：物种间共线性分析（完整版） - -输入： - - 参考物种蛋白序列 (final/{prefix}.final_members.pep.fa) - - 参考物种GFF3 - - 目标物种蛋白序列 - - 目标物种GFF3 - -输出： - - final/{prefix}.interspecies_tree.nwk (物种间进化树) - - final/{prefix}.interspecies_tree.pdf/png (进化树图) - - final/{prefix}.interspecies_synteny.pdf/png (共线性图) - - final/{prefix}.interspecies_synteny_report.md (分析报告) - - final/{prefix}.interspecies_synteny.manifest.json (机器清单) -""" - -import os -import argparse -import json -import hashlib -import subprocess -import sys -import time -import shutil -import re -from datetime import datetime -from pathlib import Path -from typing import Dict, List, Any, Optional, Tuple - - -# ??conda??bin???PATH -CONDA_BIN = "/home/xia-lab/miniconda3/envs/test4py311/bin" -if CONDA_BIN not in os.environ.get("PATH", ""): - os.environ["PATH"] = CONDA_BIN + ":" + os.environ.get("PATH", "") -def pdf_to_png(pdf_path: Path, png_path: Path, dpi: int = 300) -> bool: - """将PDF转换为PNG""" - try: - # 方法1: 使用pdftoppm (poppler-utils) - cmd = ["pdftoppm", "-png", "-r", str(dpi), "-singlefile", - str(pdf_path), str(png_path.with_suffix(''))] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - if result.returncode == 0: - return True - except Exception: - pass - - try: - # 方法2: 使用convert (ImageMagick) - cmd = ["convert", "-density", str(dpi), str(pdf_path), "-quality", "100", str(png_path)] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - if result.returncode == 0: - return True - except Exception: - pass - - return False - - -def compute_file_hash(filepath: Path) -> str: - """计算文件的SHA256哈希值""" - sha256_hash = hashlib.sha256() - try: - with open(filepath, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - except Exception: - return "" - - -def get_tool_versions() -> Dict[str, str]: - """获取所需工具版本信息""" - versions = {} - - for tool, cmd in [ - ("mafft", ["mafft", "--version"]), - ("iqtree", ["iqtree", "--version"]), - ("blastp", ["blastp", "-version"]), - ("MCScanX", ["MCScanX", "-h"]), - ]: - try: - result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=5) - versions[tool] = "available" if result.returncode in [0, 1] else "not_found" - except Exception: - versions[tool] = "not_found" - - return versions - - -def prepare_directories(workspace: Path) -> Dict[str, Path]: - """创建目录结构""" - paths = { - "analysis_dir": workspace / "interspecies_synteny" / "analysis", - "final_dir": workspace / "final", - "logs_dir": workspace / "logs", - } - for name, path in paths.items(): - path.mkdir(parents=True, exist_ok=True) - return paths - - -def merge_protein_sequences( - ref_protein: Path, - target_protein: Path, - output_fasta: Path -) -> Dict[str, Any]: - """合并两个物种的蛋白序列""" - print(f"合并蛋白序列...") - - try: - with open(output_fasta, 'w') as outfile: - with open(ref_protein, 'r') as f: - outfile.write(f.read()) - outfile.write('\n') - with open(target_protein, 'r') as f: - outfile.write(f.read()) - - ref_count = sum(1 for line in open(ref_protein) if line.startswith('>')) - target_count = sum(1 for line in open(target_protein) if line.startswith('>')) - - print(f" 成功合并 {ref_count} + {target_count} = {ref_count + target_count} 条序列") - - return { - "success": True, - "output_file": output_fasta, - "ref_count": ref_count, - "target_count": target_count - } - except Exception as e: - return {"success": False, "stderr": str(e)} - - -def build_phylogenetic_tree( - merged_fasta: Path, - output_dir: Path, - prefix: str, - cpu: int = 8 -) -> Dict[str, Any]: - """构建系统发育树 (MAFFT + IQ-TREE)""" - print(f"构建进化树...") - - # MAFFT比对 - msa_file = output_dir / f"{prefix}_interspecies.mafft.msa.fa" - mafft_cmd = ["mafft", "--auto", "--thread", str(cpu), str(merged_fasta)] - - try: - with open(msa_file, "w") as f_out: - result = subprocess.run(mafft_cmd, stdout=f_out, stderr=subprocess.PIPE, - text=True, check=False) - - if result.returncode != 0: - return {"success": False, "error": f"MAFFT失败: {result.stderr}"} - - # IQ-TREE建树 - base_name = output_dir / f"{prefix}_interspecies.iqtree" - iqtree_cmd = [ - "iqtree", "-s", str(msa_file), "-m", "MFP", "-bb", "1000", - "-nt", str(cpu), "-pre", str(base_name) - ] - - result = subprocess.run(iqtree_cmd, capture_output=True, text=True, check=False) - - # IQ-TREE 生成的 treefile 是 {prefix}.iqtree.treefile，不是 {prefix}.treefile - tree_file = Path(f"{base_name}.treefile") - - return { - "success": tree_file.exists(), - "tree_file": tree_file if tree_file.exists() else None, - "msa_file": msa_file - } - except Exception as e: - return {"success": False, "error": str(e)} - - -def filter_gff_files( - ref_gff3: Path, - target_gff3: Path, - output_dir: Path, - ref_species: str, - target_species: str -) -> Dict[str, Any]: - """过滤GFF3文件 - 提取基因ID（去掉转录本版本后缀如 .1, .2）""" - print(f"过滤GFF3文件...") - - import re - - results = {} - for gff3_file, species, prefix in [ - (ref_gff3, ref_species, "ref"), - (target_gff3, target_species, "target") - ]: - output_gff = output_dir / f"{species}.gff" - - try: - with open(gff3_file, 'r') as f: - lines = f.readlines() - - count = 0 - seen_genes = set() # 去重 - - with open(output_gff, 'w') as f: - for line in lines: - parts = line.strip().split('\t') - if len(parts) >= 9 and parts[2] == "mRNA": - attr = parts[8] - # 提取ID（如 ID=AT2G47070.1），保留版本号与BLAST结果匹配 - gene_id = attr.split(';')[0].replace('ID=', '') - - # 处理染色体名称：去掉 "Chr" 前缀（Ath用1,2,3而非Chr1,Chr2） - chr_name = parts[0] - if chr_name.startswith('Chr'): - chr_name = chr_name[3:] # Chr1 -> 1 - - # 去重：一个基因可能有多个转录本 - if gene_id not in seen_genes: - seen_genes.add(gene_id) - f.write(f"{chr_name}\t{gene_id}\t{parts[3]}\t{parts[4]}\n") - count += 1 - - print(f" {species}: {count} 个基因（去重后）") - results[prefix] = {"success": True, "output_file": output_gff, "gene_count": count} - except Exception as e: - results[prefix] = {"success": False, "stderr": str(e)} - - if results.get("ref", {}).get("success") and results.get("target", {}).get("success"): - return { - "success": True, - "ref_gff": results["ref"]["output_file"], - "target_gff": results["target"]["output_file"] - } - return {"success": False} - - -def run_reciprocal_blast( - ref_protein: Path, - target_protein: Path, - output_dir: Path, - ref_species: str, - target_species: str, - cpu: int = 8 -) -> Dict[str, Any]: - """双向BLAST比对 - 结果中基因名去掉版本后缀以匹配GFF""" - import re - print(f"双向BLAST比对...") - - # 构建数据库 - for protein_file in [ref_protein, target_protein]: - subprocess.run(["makeblastdb", "-in", str(protein_file), "-dbtype", "prot"], - capture_output=True, check=False) - - # 双向blastp - blast1_out = output_dir / f"tmp.{ref_species}_{target_species}.blast" - subprocess.run([ - "blastp", "-query", str(ref_protein), "-db", str(target_protein), - "-outfmt", "6", "-out", str(blast1_out), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ], capture_output=True, check=False) - - blast2_out = output_dir / f"tmp.{target_species}_{ref_species}.blast" - subprocess.run([ - "blastp", "-query", str(target_protein), "-db", str(ref_protein), - "-outfmt", "6", "-out", str(blast2_out), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ], capture_output=True, check=False) - - # 合并BLAST结果（保留版本号与GFF匹配） - merged_blast = output_dir / f"{ref_species}_{target_species}.blast" - with open(merged_blast, 'w') as outfile: - for blast_file in [blast1_out, blast2_out]: - if blast_file.exists(): - with open(blast_file, 'r') as infile: - for line in infile: - outfile.write(line) # 直接写入，保留版本号 - - return {"success": True, "output_file": merged_blast} - - -def run_mcscanx_interspecies( - work_dir: Path, - ref_species: str, - target_species: str -) -> Dict[str, Any]: - """MCScanX种间共线性 - 需要合并的GFF文件""" - print(f"MCScanX种间共线性...") - - prefix = f"{ref_species}_{target_species}" - - # MCScanX 需要合并的 GFF 文件 (prefix.gff) - merged_gff = work_dir / f"{prefix}.gff" - ref_gff = work_dir / f"{ref_species}.gff" - target_gff = work_dir / f"{target_species}.gff" - - if not merged_gff.exists() and ref_gff.exists() and target_gff.exists(): - print(f" 合并 GFF 文件: {ref_gff.name} + {target_gff.name} -> {merged_gff.name}") - with open(merged_gff, 'w') as outfile: - for gff_file in [target_gff, ref_gff]: # 目标物种在前，参考物种在后 - with open(gff_file, 'r') as infile: - outfile.write(infile.read()) - - cmd = ["MCScanX", f"./{prefix}"] - - result = subprocess.run(cmd, capture_output=True, text=True, - cwd=str(work_dir), check=False) - - collinearity_file = work_dir / f"{prefix}.collinearity" - - return { - "success": collinearity_file.exists(), - "collinearity_file": collinearity_file if collinearity_file.exists() else None - } - - - - -def fasta_ids(fasta_file: Path) -> List[str]: - ids = [] - with open(fasta_file, "r") as f: - for line in f: - if line.startswith(">"): - ids.append(line[1:].strip().split()[0]) - return ids - - -def display_species_code(species: str, gene_id: str = "") -> str: - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return species - - -def species_for_gene(gene_id: str, species_list: List[str], default_species: str) -> str: - for species in sorted(species_list, key=len, reverse=True): - if gene_id.lower().startswith(species.lower()): - return display_species_code(species, gene_id) - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return display_species_code(default_species, gene_id) - - -def write_gene_species_map(target_fasta: Path, ref_fasta: Path, target_species: List[str], ref_species: List[str], output_file: Path) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - default_target = target_species[0] if target_species else "Target" - default_ref = ref_species[0] if ref_species else "Ref" - with open(output_file, "w") as out: - out.write("gene_id\tspecies\tgroup\n") - for gene_id in fasta_ids(target_fasta): - out.write(f"{gene_id}\t{species_for_gene(gene_id, target_species, default_target)}\ttarget\n") - for gene_id in fasta_ids(ref_fasta): - out.write(f"{gene_id}\t{species_for_gene(gene_id, ref_species, default_ref)}\tref\n") - return output_file -def plot_interspecies_tree( - tree_file: Path, - output_dir: Path, - prefix: str, - target_species: str, - ref_species: str, - target_subfamily_file: Optional[Path] = None, - ref_subfamily_file: Optional[Path] = None, - gene_species_map: Optional[Path] = None -) -> Dict[str, Any]: - """绘制物种间进化树图 - 使用专用的 interspecies_tree.R 脚本""" - print(f"绘制进化树图...") - - # 查找专用的种间进化树R脚本 - r_script = Path("scripts/interspecies_tree.R") - if not r_script.exists(): - alt_script = Path("/home/xia-lab/wutx/00test/bbav2/scripts/interspecies_tree.R") - if alt_script.exists(): - r_script = alt_script - else: - return {"success": False, "error": f"R脚本不存在: {r_script}"} - - try: - # 使用专用的 interspecies_tree.R 脚本，传递物种参数和亚家族文件 - # 使用conda环境的Rscript - conda_rscript = "/home/xia-lab/miniconda3/envs/test4py311/bin/Rscript" - cmd = [ - conda_rscript, str(r_script), - str(tree_file), - str(output_dir), - target_species, - ref_species - ] - - # 添加亚家族文件参数（如果存在） - if target_subfamily_file and target_subfamily_file.exists(): - cmd.append(str(target_subfamily_file)) - print(f" 目标物种亚家族文件: {target_subfamily_file}") - else: - cmd.append("") # 空字符串表示无文件 - - if ref_subfamily_file and ref_subfamily_file.exists(): - cmd.append(str(ref_subfamily_file)) - print(f" 参考物种亚家族文件: {ref_subfamily_file}") - else: - cmd.append("") # 空字符串表示无文件 - if gene_species_map and gene_species_map.exists(): - cmd.append(str(gene_species_map)) - print(f" Gene-to-species map: {gene_species_map}") - else: - cmd.append("") - result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=120) - - # 调试输出 - print(f" R脚本返回码: {result.returncode}") - if result.stdout: - print(f" R脚本输出: {result.stdout[:500]}") - if result.stderr: - print(f" R脚本错误: {result.stderr[:500]}") - - # 专用脚本输出固定为 interspecies_tree.pdf/png - output_pdf = output_dir / "interspecies_tree.pdf" - output_png = output_dir / "interspecies_tree.png" - - # 重命名为规范名称 - final_pdf = output_dir / f"{prefix}.interspecies_tree.pdf" - final_png = output_dir / f"{prefix}.interspecies_tree.png" - - if output_pdf.exists(): - shutil.move(output_pdf, final_pdf) - print(f" 进化树PDF: {final_pdf}") - if output_png.exists(): - shutil.move(output_png, final_png) - print(f" 进化树PNG: {final_png}") - - # PDF转PNG（如果没有生成PNG） - if final_pdf.exists() and not final_png.exists(): - png_from_pdf = output_dir / "interspecies_tree.png" - if pdf_to_png(final_pdf, png_from_pdf): - final_png = output_dir / f"{prefix}.interspecies_tree.png" - shutil.move(png_from_pdf, final_png) - print(f" PDF转PNG: {final_png}") - - return { - "success": final_pdf.exists() or final_png.exists(), - "pdf_file": final_pdf if final_pdf.exists() else None, - "png_file": final_png if final_png.exists() else None, - "stdout": result.stdout, - "stderr": result.stderr - } - except Exception as e: - return {"success": False, "error": str(e)} - - -def plot_synteny_karyotype( - work_dir: Path, - collinearity_file: Path, - output_dir: Path, - prefix: str, - target_species: str, - ref_species: str -) -> Dict[str, Any]: - """绘制共线性染色体图 (使用 jcvi)""" - print(f"绘制共线性图...") - - # Conda 环境的 Python 路径 - CONDA_PYTHON = "/home/xia-lab/miniconda3/envs/test4py311/bin/python" - - try: - # 1. 准备 BED 文件 (从 gff 转换) - # awk '{print $1"\t"$3"\t"$4"\t"$2}' Ft.gff > Ft.bed - target_bed = work_dir / f"{target_species}.bed" - ref_bed = work_dir / f"{ref_species}.bed" - - for species in [target_species, ref_species]: - gff = work_dir / f"{species}.gff" - bed = work_dir / f"{species}.bed" - if gff.exists(): - with open(gff, 'r') as f_in, open(bed, 'w') as f_out: - for line in f_in: - parts = line.strip().split('\t') - if len(parts) >= 4: - # GFF: chr\tgene_id\tstart\tend - # BED: chr\tstart\tend\tgene_id - f_out.write(f"{parts[0]}\t{parts[2]}\t{parts[3]}\t{parts[1]}\n") - - if not target_bed.exists() or not ref_bed.exists(): - return {"success": False, "error": "BED文件未生成"} - - # 2. 修改 collinearity 文件成 jcvi anchors 格式 - # 按照代码描述：target_gene\tref_gene\tscore 格式（先目标后参考） - # 必须保留 ## block 分隔符 - anchors_file = work_dir / f"{target_species}.{ref_species}.anchors" - - # 先读取 BED 文件中的有效基因 ID（用于过滤线粒体/叶绿体基因） - valid_target_genes = set() - valid_ref_genes = set() - - with open(target_bed, 'r') as f: - for line in f: - parts = line.strip().split('\t') - if len(parts) >= 4: - valid_target_genes.add(parts[3]) - - with open(ref_bed, 'r') as f: - for line in f: - parts = line.strip().split('\t') - if len(parts) >= 4: - valid_ref_genes.add(parts[3]) - - print(f" 目标物种有效基因: {len(valid_target_genes)}, 参考物种有效基因: {len(valid_ref_genes)}") - - try: - current_block_lines = [] - total_anchors = 0 - filtered_anchors = 0 - with open(collinearity_file, 'r') as f_in, open(anchors_file, 'w') as f_out: - for line in f_in: - if not line.strip(): - continue - if line.startswith('## Alignment'): - # 如果前一个块有内容，先写入分隔符 - if current_block_lines: - f_out.write("##\n") - for l in current_block_lines: - f_out.write(l) - current_block_lines = [] - elif line.startswith('#'): - continue # 跳过其他 # 开头的行（参数、统计等） - elif (line.startswith(' ') or line[0].isdigit()) and '\t' in line: - parts = line.strip().split('\t') - if len(parts) >= 3: - ref_gene = parts[1].strip() - target_gene = parts[2].strip() - # 过滤：只保留在 BED 中的基因（排除线粒体/叶绿体基因） - if target_gene in valid_target_genes and ref_gene in valid_ref_genes: - score = "1000" - current_block_lines.append(f"{target_gene}\t{ref_gene}\t{score}\n") - total_anchors += 1 - else: - filtered_anchors += 1 - # 写入最后一个块（如果有内容） - if current_block_lines: - f_out.write("##\n") - for l in current_block_lines: - f_out.write(l) - print(f" 生成 anchors 文件: {anchors_file} ({total_anchors} anchors, 过滤 {filtered_anchors} 个无效基因)") - except Exception as e: - print(f" 警告: anchors文件生成失败: {e}") - - if not anchors_file.exists(): - return {"success": False, "error": "anchors文件未生成"} - - # 3. 生成 .simple 文件 - 使用 jcvi screen --simple（按照代码描述） - # python -m jcvi.compara.synteny screen --minspan=10 --simple Ft.Ath.anchors Ft.Ath.anchors.new --qbed=Ft.bed --sbed=Ath.bed - anchors_new_file = work_dir / f"{target_species}.{ref_species}.anchors.new" - simple_file = work_dir / f"{target_species}.{ref_species}.anchors.simple" - - cmd_screen = [ - CONDA_PYTHON, "-m", "jcvi.compara.synteny", "screen", - "--minspan=10", "--simple", - str(anchors_file), - str(anchors_new_file), - f"--qbed={target_bed.name}", - f"--sbed={ref_bed.name}" - ] - print(f" 运行: {' '.join(cmd_screen)}") - result_screen = subprocess.run(cmd_screen, capture_output=True, text=True, - cwd=str(work_dir), check=False, timeout=120) - - if result_screen.returncode != 0: - print(f" 警告: jcvi screen 失败: {result_screen.stderr}") - - # jcvi screen --simple 会生成 .simple 文件（注意：输出文件名是 {newanchorfile}.simple） - jcvi_simple_output = work_dir / f"{target_species}.{ref_species}.new.simple" - if jcvi_simple_output.exists(): - shutil.copy(jcvi_simple_output, simple_file) - print(f" 生成 simple 文件: {simple_file}") - elif simple_file.exists(): - print(f" 生成 simple 文件: {simple_file}") - else: - print(f" 警告: simple 文件未生成，尝试备用方法") - # 如果失败，尝试直接复制 anchors 作为备用 - simple_file = anchors_file - - # 4. 生成 seqids 文件 - seqids_file = work_dir / "seqids" - # 获取染色体列表（排除叶绿体和线粒体：ChrC/ChrM 或 C/M） - target_chrs = sorted(set(line.split()[0] for line in open(target_bed) - if line.split()[0] not in ['C', 'M', 'ChrC', 'ChrM'])) - ref_chrs = sorted(set(line.split()[0] for line in open(ref_bed) - if line.split()[0] not in ['C', 'M', 'ChrC', 'ChrM'])) - with open(seqids_file, 'w') as f: - f.write(','.join(target_chrs) + '\n') - f.write(','.join(ref_chrs) + '\n') - - # 5. 生成 layout 文件 - layout_file = work_dir / "layout" - with open(layout_file, 'w') as f: - f.write("# y, xstart, xend, rotation, color, label, va, bed\n") - f.write(f".6, .1, .8, 0, , {target_species}, top, {target_bed.name}\n") - f.write(f".4, .1, .8, 0, , {ref_species}, bottom, {ref_bed.name}\n") - f.write("# edges\n") - f.write(f"e, 0, 1, {simple_file.name}\n") - - # 6. 绘制共线性图（基础版） - cmd_plot = [ - CONDA_PYTHON, "-m", "jcvi.graphics.karyotype", - str(seqids_file), str(layout_file), - "--notex" - ] - print(f" 绘图: {' '.join(cmd_plot)}") - result = subprocess.run(cmd_plot, capture_output=True, text=True, - cwd=str(work_dir), check=False, timeout=120) - - if result.returncode != 0: - print(f" 绘图警告: {result.stderr}") - else: - print(f" 绘图成功") - - # 7. 重命名输出文件 - final_pdf = output_dir / f"{prefix}.interspecies_synteny.pdf" - final_png = output_dir / f"{prefix}.interspecies_synteny.png" - - jcvi_pdf = work_dir / "karyotype.pdf" - jcvi_png = work_dir / "karyotype.png" - - if jcvi_pdf.exists(): - shutil.copy(jcvi_pdf, final_pdf) - if jcvi_png.exists(): - shutil.copy(jcvi_png, final_png) - - # 8. 绘制高亮图（SPL基因高亮显示） - print("🎨 绘制SPL基因高亮图...") - - # 8.1 生成 gene2spc 文件（基因ID和物种对应表格） - # 从 geneID.subfamily 文件提取目标物种基因，如果没有则从 final_members.id 生成 - gene2spc_file = work_dir / f"{target_species}.gene2spc" - subfamily_file = output_dir.parent / "final" / f"{prefix}.geneID.subfamily" - - # 尝试多种可能的 subfamily 文件位置 - if not subfamily_file.exists(): - alt_paths = [ - output_dir.parent / "final" / f"{target_species}.SPL.geneID.subfamily", - output_dir.parent.parent / "input" / f"{target_species}.SPL.geneID.subfamily", - work_dir.parent.parent / "input" / f"{target_species}.SPL.geneID.subfamily", - ] - for alt_path in alt_paths: - if alt_path.exists(): - subfamily_file = alt_path - break - - # 优先使用 subfamily 文件，否则使用 final_members.id - id_file = output_dir.parent / "final" / f"{prefix}.final_members.id" - - if subfamily_file.exists(): - with open(subfamily_file, 'r') as f_in, open(gene2spc_file, 'w') as f_out: - for line in f_in: - gene_id = line.strip().split()[0] if line.strip() else "" - if gene_id: - f_out.write(f"{gene_id}\t{target_species}\n") - print(f" 生成 gene2spc (from subfamily): {gene2spc_file}") - elif id_file.exists(): - # 从 final_members.id 生成 - with open(id_file, 'r') as f_in, open(gene2spc_file, 'w') as f_out: - for line in f_in: - gene_id = line.strip() - if gene_id and not gene_id.startswith('#'): - f_out.write(f"{gene_id}\t{target_species}\n") - print(f" 生成 gene2spc (from final_members.id): {gene2spc_file}") - else: - print(f" ⚠️ 警告: 找不到 geneID.subfamily 或 final_members.id，无法生成 gene2spc") - - # 8.2 ?? converted Python ????????? .simple ?? - # 按照代码描述：python scripts/gene_highlight_jcvi.py Ft.gene2spc Ft.Ath.anchors.new Ft.Ath.anchors.simple > Ft.Ath.anchors.simple.mark - simple_mark_file = work_dir / f"{target_species}.{ref_species}.anchors.simple.mark" - - if gene2spc_file.exists() and simple_file.exists() and anchors_new_file.exists(): - # ?? converted Python ?? - highlight_script = Path("scripts/gene_highlight_jcvi.py").resolve() - - if highlight_script.exists(): - # ???????? Python ?? - # python scripts/gene_highlight_jcvi.py Ft.gene2spc Ft.Ath.anchors.new Ft.Ath.anchors.simple > Ft.Ath.anchors.simple.mark - cmd_highlight = [ - sys.executable, str(highlight_script), - str(gene2spc_file.name), # 使用文件名，因为工作目录是 work_dir - str(anchors_new_file.name), - str(simple_file.name) - ] - print(f" 运行高亮: {' '.join(cmd_highlight)}") - - # 使用 shell 重定向输出到 .simple.mark 文件 - with open(simple_mark_file, 'w') as f_out: - result_highlight = subprocess.run( - cmd_highlight, stdout=f_out, stderr=subprocess.PIPE, text=True, - cwd=str(work_dir), check=False, timeout=60 - ) - - # 检查是否成功生成 - if result_highlight.returncode == 0 and simple_mark_file.exists() and simple_mark_file.stat().st_size > 0: - print(f" 生成高亮标记: {simple_mark_file}") - # 显示一些高亮行 - highlighted = subprocess.run( - ["grep", "-c", "^r\\*", str(simple_mark_file)], - capture_output=True, text=True, cwd=str(work_dir) - ) - if highlighted.stdout: - print(f" 高亮基因对数: {highlighted.stdout.strip()}") - else: - print(f" 高亮脚本警告: {result_highlight.stderr}") - else: - print(f" ?? ??: ??? Python ?? gene_highlight_jcvi.py") - - # 8.3 创建高亮版本的 layout 文件（带颜色） - layout_mark_file = work_dir / "layout_mark" - with open(layout_mark_file, 'w') as f: - f.write("# y, xstart, xend, rotation, color, label, va, bed\n") - f.write(f".6, .1, .8, 0, red, {target_species}, top, {target_bed.name}\n") - f.write(f".4, .1, .8, 0, blue, {ref_species}, bottom, {ref_bed.name}\n") - f.write("# edges\n") - if simple_mark_file.exists(): - f.write(f"e, 0, 1, {simple_mark_file.name}\n") - else: - f.write(f"e, 0, 1, {simple_file.name}\n") - - # 8.4 绘制高亮共线性图 - cmd_plot_mark = [ - CONDA_PYTHON, "-m", "jcvi.graphics.karyotype", - str(seqids_file), str(layout_mark_file), - "--notex" - ] - print(f" 绘制高亮图: {' '.join(cmd_plot_mark)}") - result_mark = subprocess.run( - cmd_plot_mark, capture_output=True, text=True, - cwd=str(work_dir), check=False, timeout=120 - ) - - if result_mark.returncode != 0: - print(f" 高亮图警告: {result_mark.stderr}") - - # 8.5 重命名高亮输出文件 - final_mark_pdf = output_dir / f"{prefix}.interspecies_synteny.marked.pdf" - final_mark_png = output_dir / f"{prefix}.interspecies_synteny.marked.png" - - jcvi_mark_pdf = work_dir / "karyotype.pdf" - jcvi_mark_png = work_dir / "karyotype.png" - - if jcvi_mark_pdf.exists(): - shutil.copy(jcvi_mark_pdf, final_mark_pdf) - print(f" 高亮图PDF: {final_mark_pdf}") - if jcvi_mark_png.exists(): - shutil.copy(jcvi_mark_png, final_mark_png) - - # PDF转PNG（基础版和高亮版） - if final_pdf.exists() and not final_png.exists(): - png_path = output_dir / f"{prefix}.interspecies_synteny.png" - if pdf_to_png(final_pdf, png_path): - final_png = png_path - print(f" 基础图PDF转PNG: {final_png}") - - if final_mark_pdf.exists() and not final_mark_png.exists(): - mark_png_path = output_dir / f"{prefix}.interspecies_synteny.marked.png" - if pdf_to_png(final_mark_pdf, mark_png_path): - final_mark_png = mark_png_path - print(f" 高亮图PDF转PNG: {final_mark_png}") - - return { - "success": final_pdf.exists() and final_png.exists(), - "pdf_file": final_pdf if final_pdf.exists() else None, - "png_file": final_png if final_png.exists() else None, - "marked_pdf_file": final_mark_pdf if final_mark_pdf.exists() else None, - "marked_png_file": final_mark_png if final_mark_png.exists() else None, - "stdout": result.stdout, - "stderr": result.stderr - } - - except Exception as e: - return {"success": False, "error": str(e)} - - -def write_report(workspace: Path, prefix: str, metrics: Dict, parameters: Dict, - tool_versions: Dict, start_time: datetime, end_time: datetime, - elapsed_sec: float, cmd: List[str]) -> Path: - """写入报告""" - report_path = workspace / "final" / f"{prefix}.interspecies_synteny_report.md" - - # 收集可复现性信息 - reproducibility_info = [] - reproducibility_info.append("## 可复现性信息\n\n") - reproducibility_info.append("### 执行环境\n") - reproducibility_info.append(f"- **分析时间**: {start_time.isoformat()}\n") - reproducibility_info.append(f"- **完成时间**: {end_time.isoformat()}\n") - reproducibility_info.append(f"- **运行时长**: {elapsed_sec:.2f} 秒\n") - reproducibility_info.append(f"- **工作目录**: {workspace}\n") - reproducibility_info.append(f"- **输入目录**: {parameters.get('input_dir', 'N/A')}\n\n") - - reproducibility_info.append("### 工具版本\n") - for tool, version in tool_versions.items(): - reproducibility_info.append(f"- **{tool}**: {version}\n") - reproducibility_info.append("\n") - - reproducibility_info.append("### 执行命令\n") - reproducibility_info.append(f"```bash\n{' '.join(cmd)}\n```\n\n") - - reproducibility_info.append("### 输入参数\n") - for key, value in parameters.items(): - if key != 'input_dir': - reproducibility_info.append(f"- **{key}**: {value}\n") - reproducibility_info.append("\n") - - with open(report_path, "w") as f: - f.write(f"# 物种间共线性分析 - {prefix}\n\n") - f.write(f"**时间**: {datetime.now().isoformat()}\n\n") - f.write(f"- 目标物种 (研究对象): {parameters.get('target_species')}\n") - f.write(f"- 参考物种 (对照): {parameters.get('ref_species')}\n") - f.write(f"- 目标蛋白文件: {parameters.get('target_protein_spl')}\n") - f.write(f"- 参考蛋白文件: {parameters.get('ref_protein_spl')}\n") - f.write(f"- 目标全蛋白组: {parameters.get('target_protein_all')}\n") - f.write(f"- 参考全蛋白组: {parameters.get('ref_protein_all')}\n") - f.write("\n") - f.writelines(reproducibility_info) - return report_path - - -def get_file_info(file_path: Path) -> Dict[str, Any]: - """获取文件的详细信息，用于artifact规范""" - info = { - "size": 0, - "sha256": "", - "mtime": 0 - } - if file_path.exists(): - info["size"] = file_path.stat().st_size - info["mtime"] = file_path.stat().st_mtime - info["sha256"] = compute_file_hash(file_path) - return info - - -def write_manifest(workspace: Path, job_id: str, run_id: str, prefix: str, - start_time: datetime, end_time: datetime, artifacts: Dict, - metrics: Dict, parameters: Dict, tool_versions: Dict, - workspace_root: Path, input_dir: Path, cmd: List[str], - status: str = "success") -> Path: - """写入manifest - 符合schema v1标准""" - elapsed_sec = (end_time - start_time).total_seconds() - - # 构建符合规范的artifacts结构 - formatted_artifacts = {} - for key, value in artifacts.items(): - if isinstance(value, dict): - # 已经是字典格式，确保包含必需字段 - formatted_artifacts[key] = value - elif isinstance(value, str): - # 字符串格式，转换为artifact对象 - file_path = workspace / value - file_info = get_file_info(file_path) - formatted_artifacts[key] = { - "artifact": key, - "path": value, - "full_path": str(file_path.absolute()), - "type": "file", - "size": file_info["size"], - "sha256": file_info["sha256"], - "mtime": file_info["mtime"] - } - - manifest_data = { - "manifest_version": "1.0", - "job_id": job_id, - "stage": "interspecies_synteny", - "run_id": run_id, - "prefix": prefix, - "timestamp": end_time.isoformat(), - "status": status, - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - "elapsed_sec": round(elapsed_sec, 2), - "cmd": cmd, - "workspace_root": str(workspace_root.absolute()), - "workspace": str(workspace.absolute()), - "input_dir": str(input_dir.absolute()), - "tool_versions": tool_versions, - "artifacts": formatted_artifacts, - "parameters": parameters, - "metrics": metrics - } - - manifest_path = workspace / "final" / f"{prefix}.interspecies_synteny.manifest.json" - with open(manifest_path, "w", encoding='utf-8') as f: - json.dump(manifest_data, f, indent=2, ensure_ascii=False) - - return manifest_path - - - -def _clean_bool(value: Any) -> bool: - return str(value).strip().lower() in {"true", "1", "yes", "y"} - - -def _gene_family_from_prefix(prefix: str) -> str: - return prefix.split('.', 1)[1] if '.' in prefix else prefix - - -def _resolve_path(path_text: str, input_dir: Path) -> Optional[Path]: - if not path_text: - return None - path = Path(path_text) - if path.is_absolute(): - return path if path.exists() else None - for candidate in (input_dir / path, input_dir / path.name): - if candidate.exists(): - return candidate - return None - - -def _read_config_groups(input_dir: Path, default_family: str) -> Dict[str, Any]: - config_path = input_dir / "config.xlsx" - groups = {"target": {}, "ref": {}, "gene_family": default_family} - if not config_path.exists(): - return groups - try: - import pandas as pd - df = pd.read_excel(config_path) - except Exception as exc: - print(f"[WARN] Failed to read config.xlsx: {exc}") - return groups - for _, row in df.iterrows(): - species = str(row.get("Species_Code", "")).strip() - family = str(row.get("Gene_Family", default_family)).strip() or default_family - file_type = str(row.get("File_Type", "")).strip().lower() - original_name = str(row.get("Original_File_Name", "")).strip() - if not species: - continue - group_name = "ref" if _clean_bool(row.get("Ref", False)) else "target" - item = groups[group_name].setdefault(species, {"species": species, "gene_family": family, "files": {}}) - item["gene_family"] = family - if file_type: - item["files"][file_type] = original_name - groups["gene_family"] = family or groups["gene_family"] - return groups - - -def _read_genome_prepare_artifacts(workspace: Path, input_dir: Path, prefix: str) -> Dict[str, Dict[str, Path]]: - result: Dict[str, Dict[str, Path]] = {} - for manifest_path in (workspace / "final" / f"{prefix}.genome_prepare.manifest.json", input_dir / "final" / f"{prefix}.genome_prepare.manifest.json"): - if not manifest_path.exists(): - continue - try: - data = json.loads(manifest_path.read_text(encoding="utf-8")) - except Exception as exc: - print(f"[WARN] Failed to read genome_prepare manifest {manifest_path}: {exc}") - continue - for key, artifact in (data.get("artifacts") or {}).items(): - kind = artifact.get("artifact") - species = key.rsplit("_", 1)[0] - found = _resolve_path(artifact.get("full_path") or artifact.get("path") or "", input_dir) - if found and kind: - result.setdefault(species, {})[kind] = found - return result - - -def _find_species_file(input_dir: Path, species: str, family: str, file_type: str, original_name: str = "") -> Optional[Path]: - if file_type == "pep": - patterns = [f"{species}.{family}.pep.fasta", f"{species}.{family}.pep.fa", f"{species}.pep.fasta", f"{species}.pep.fa", f"{species}.protein.fasta", f"{species}.faa"] - elif file_type == "gff": - patterns = [f"{species}.{family}_final.gff3", f"{species}.{family}.gff3", f"{species}_final.gff3", f"{species}.gff3", f"{species}.gff"] - else: - patterns = [] - for pattern in patterns: - for candidate in (input_dir / pattern, input_dir / species / pattern): - if candidate.exists(): - return candidate - if original_name: - original = Path(original_name).name - for candidate in (input_dir / original, input_dir / species / original): - if candidate.exists(): - return candidate - return None - - -def discover_group_inputs(workspace: Path, input_dir: Path, prefix: str, args) -> Dict[str, Any]: - family = _gene_family_from_prefix(prefix) - groups = _read_config_groups(input_dir, family) - artifacts = _read_genome_prepare_artifacts(workspace, input_dir, prefix) - family = groups.get("gene_family") or family - - for group_name in ("target", "ref"): - for species, item in groups[group_name].items(): - files = item.setdefault("resolved", {}) - artifact_paths = artifacts.get(species, {}) - original_files = item.get("files", {}) - files["pep"] = artifact_paths.get("pep") or artifact_paths.get("ref_pep") or _find_species_file(input_dir, species, item.get("gene_family", family), "pep", original_files.get("pep", "")) - files["gff"] = artifact_paths.get("gff") or _find_species_file(input_dir, species, item.get("gene_family", family), "gff", original_files.get("gff", "")) - - target_family_protein = Path(args.target_protein) if args.target_protein else workspace / "final" / f"{prefix}.final_members.pep.fa" - ref_family_protein = Path(args.ref_protein) if args.ref_protein else None - if ref_family_protein is None: - for candidate in [input_dir / f"Ref_combined.{family}.pep.fasta", input_dir / f"Ref_combined.{family}.pep.fa", input_dir / "Ref_combined.pep.fasta"]: - if candidate.exists(): - ref_family_protein = candidate - break - if ref_family_protein is None: - ref_peps = [item.get("resolved", {}).get("pep") for item in groups["ref"].values() if item.get("resolved", {}).get("pep")] - if ref_peps: - ref_family_protein = workspace / "interspecies_synteny" / "analysis" / f"Ref_combined.{family}.pep.fasta" - merge_many_fastas(ref_peps, ref_family_protein) - - target_all_peps = [item.get("resolved", {}).get("pep") for item in groups["target"].values() if item.get("resolved", {}).get("pep")] - ref_all_peps = [item.get("resolved", {}).get("pep") for item in groups["ref"].values() if item.get("resolved", {}).get("pep")] - target_gffs = [(sp, item.get("resolved", {}).get("gff")) for sp, item in groups["target"].items() if item.get("resolved", {}).get("gff")] - ref_gffs = [(sp, item.get("resolved", {}).get("gff")) for sp, item in groups["ref"].items() if item.get("resolved", {}).get("gff")] - - return { - "gene_family": family, - "target_species": sorted(groups["target"].keys()), - "ref_species": sorted(groups["ref"].keys()), - "target_family_protein": target_family_protein, - "ref_family_protein": ref_family_protein, - "target_all_peps": target_all_peps, - "ref_all_peps": ref_all_peps, - "target_gffs": target_gffs, - "ref_gffs": ref_gffs, - "groups": groups, - } - - -def merge_many_fastas(files: List[Path], output_file: Path) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as out: - for fasta in files: - with open(fasta, "r") as f: - out.write(f.read().rstrip() + "\n") - return output_file - - -def _parse_gff_attr(attr: str) -> Dict[str, str]: - result = {} - for item in attr.strip().strip(';').split(';'): - item = item.strip() - if not item: - continue - if '=' in item: - k, v = item.split('=', 1) - elif ' ' in item: - k, v = item.split(' ', 1) - else: - continue - result[k.strip()] = v.strip().strip('"') - return result - - -def build_group_gff(gffs: List[Tuple[str, Path]], output_file: Path) -> Dict[str, Any]: - output_file.parent.mkdir(parents=True, exist_ok=True) - count = 0 - seen = set() - with open(output_file, "w") as out: - for species, gff3 in gffs: - with open(gff3, "r") as f: - for line in f: - if not line.strip() or line.startswith('#'): - continue - parts = line.rstrip('\n').split('\t') - if len(parts) < 9 or parts[2] not in {"mRNA", "transcript", "gene"}: - continue - attrs = _parse_gff_attr(parts[8]) - gene_id = attrs.get("ID") or attrs.get("transcript_id") or attrs.get("Name") - if not gene_id or gene_id in seen: - continue - seen.add(gene_id) - chrom = f"{species}|{parts[0]}" - out.write(f"{chrom}\t{gene_id}\t{parts[3]}\t{parts[4]}\n") - count += 1 - return {"file": output_file, "gene_count": count, "success": count > 0} - - -def add_artifact(artifacts: Dict[str, Dict], key: str, file_path: Path, rel_path: str, artifact: str, file_type: str) -> None: - if not file_path or not file_path.exists(): - return - info = get_file_info(file_path) - artifacts[key] = { - "artifact": artifact, - "path": rel_path, - "full_path": str(file_path.absolute()), - "type": file_type, - "size": info["size"], - "sha256": info["sha256"], - "mtime": info["mtime"], - } - - -def write_failure_manifest(workspace: Path, job_id: str, run_id: str, prefix: str, start_time: datetime, - workspace_root: Path, input_dir: Path, cmd: List[str], error: str) -> Path: - end_time = datetime.now() - return write_manifest( - workspace, job_id, run_id, prefix, start_time, end_time, - artifacts={}, metrics={"error": error}, parameters={"prefix": prefix, "error": error}, - tool_versions={}, workspace_root=workspace_root, input_dir=input_dir, cmd=cmd, status="failed" - ) - - -def main(): - parser = argparse.ArgumentParser(description="????????") - parser.add_argument("--prefix", required=True, help="?????? Multi.SPL") - parser.add_argument("--workspace", required=True) - parser.add_argument("--input-dir") - parser.add_argument("--ref-species", default=None, help="???????? config.xlsx ??") - parser.add_argument("--target-species", default=None, help="???????? config.xlsx ??") - parser.add_argument("--ref-protein", help="?????????") - parser.add_argument("--target-protein", help="?????????") - parser.add_argument("--cpu", type=int, default=8) - parser.add_argument("--log-file", help="??????????????") - parser.add_argument("--workspace-root", help="??????????manifest?") - args = parser.parse_args() - - start_time = datetime.now() - run_id = f"interspecies_synteny_{start_time.strftime('%Y%m%d_%H%M%S')}" - workspace = Path(args.workspace) - job_id = workspace.name - input_dir = Path(args.input_dir) if args.input_dir else workspace.parent.parent / "input" / job_id - workspace_root = Path(args.workspace_root) if args.workspace_root else workspace.parent - cmd = sys.argv - - print("=" * 60) - print("Interspecies synteny analysis") - print("=" * 60) - - try: - paths = prepare_directories(workspace) - tool_versions = get_tool_versions() - discovery = discover_group_inputs(workspace, input_dir, args.prefix, args) - target_group = "Target_combined" - ref_group = "Ref_combined" - print(f"Target species: {', '.join(discovery['target_species']) or 'NONE'}") - print(f"Reference species: {', '.join(discovery['ref_species']) or 'NONE'}") - print(f"Target family protein: {discovery['target_family_protein']}") - print(f"Ref family protein: {discovery['ref_family_protein']}") - - if not discovery["target_family_protein"] or not Path(discovery["target_family_protein"]).exists(): - raise RuntimeError(f"Target family protein missing: {discovery['target_family_protein']}") - if not discovery["ref_family_protein"] or not Path(discovery["ref_family_protein"]).exists(): - raise RuntimeError("Reference family protein missing; expected Ref_combined..pep.fasta or Ref species pep files") - - artifacts = {} - metrics = { - "target_species": discovery["target_species"], - "ref_species": discovery["ref_species"], - "tree_success": False, - "synteny_success": False, - "synteny_skipped_reason": "", - } - - print("\nMerging family proteins for tree...") - merged_fasta = paths["analysis_dir"] / f"{args.prefix}.target_ref.family.pep.fa" - merge_result = merge_protein_sequences(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), merged_fasta) - if not merge_result.get("success"): - raise RuntimeError(f"Failed to merge family proteins: {merge_result.get('stderr')}") - metrics["target_family_count"] = merge_result.get("ref_count") - metrics["ref_family_count"] = merge_result.get("target_count") - - print("\nBuilding interspecies tree...") - tree_result = build_phylogenetic_tree(merged_fasta, paths["analysis_dir"], args.prefix, args.cpu) - if not tree_result.get("tree_file"): - raise RuntimeError(f"Tree construction failed: {tree_result.get('error', 'unknown error')}") - final_tree = paths["final_dir"] / f"{args.prefix}.interspecies_tree.nwk" - shutil.copy(Path(tree_result["tree_file"]), final_tree) - metrics["tree_success"] = True - add_artifact(artifacts, "tree_nwk", final_tree, f"final/{args.prefix}.interspecies_tree.nwk", "tree_nwk", "tree") - - gene_species_map = paths["analysis_dir"] / f"{args.prefix}.gene_to_species.tsv" - write_gene_species_map(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), discovery["target_species"], discovery["ref_species"], gene_species_map) - add_artifact(artifacts, "gene_to_species", gene_species_map, f"interspecies_synteny/analysis/{args.prefix}.gene_to_species.tsv", "gene_to_species", "tsv") - - print("\nPlotting interspecies tree...") - target_subfamily_file = input_dir / f"{args.prefix}.geneID.subfamily" - if not target_subfamily_file.exists(): - target_subfamily_file = workspace / "final" / f"{args.prefix}.geneID.subfamily" - ref_subfamily_file = input_dir / f"SPL_{discovery['ref_species'][0]}.subfamily" if discovery["ref_species"] else input_dir / "SPL_ref.subfamily" - if not ref_subfamily_file.exists(): - candidates = sorted(input_dir.glob("SPL_*.subfamily")) + sorted(input_dir.glob("*.SPL.subfamily")) - ref_subfamily_file = candidates[0] if candidates else ref_subfamily_file - tree_plot_result = plot_interspecies_tree( - final_tree, - paths["final_dir"], - args.prefix, - target_group, - ref_group, - target_subfamily_file if target_subfamily_file.exists() else None, - ref_subfamily_file if ref_subfamily_file.exists() else None, - gene_species_map, - ) - if tree_plot_result.get("pdf_file"): - add_artifact(artifacts, "tree_pdf", tree_plot_result["pdf_file"], f"final/{args.prefix}.interspecies_tree.pdf", "tree_pdf", "plot") - if tree_plot_result.get("png_file"): - add_artifact(artifacts, "tree_png", tree_plot_result["png_file"], f"final/{args.prefix}.interspecies_tree.png", "tree_png", "plot") - - synteny_plot_result = {"success": False} - mcscanx_result = {"success": False} - can_run_synteny = bool(discovery["target_all_peps"] and discovery["ref_all_peps"] and discovery["target_gffs"] and discovery["ref_gffs"]) - if can_run_synteny: - print("\nPreparing combined Target/Ref files for MCScanX...") - target_all = merge_many_fastas(discovery["target_all_peps"], paths["analysis_dir"] / f"{target_group}.pep.fasta") - ref_all = merge_many_fastas(discovery["ref_all_peps"], paths["analysis_dir"] / f"{ref_group}.pep.fasta") - target_gff_result = build_group_gff(discovery["target_gffs"], paths["analysis_dir"] / f"{target_group}.gff") - ref_gff_result = build_group_gff(discovery["ref_gffs"], paths["analysis_dir"] / f"{ref_group}.gff") - if target_gff_result.get("success") and ref_gff_result.get("success"): - blast_result = run_reciprocal_blast(target_all, ref_all, paths["analysis_dir"], target_group, ref_group, args.cpu) - if blast_result.get("success"): - mcscanx_result = run_mcscanx_interspecies(paths["analysis_dir"], target_group, ref_group) - if mcscanx_result.get("collinearity_file"): - add_artifact(artifacts, "collinearity", mcscanx_result["collinearity_file"], f"interspecies_synteny/analysis/{target_group}_{ref_group}.collinearity", "collinearity", "data") - synteny_plot_result = plot_synteny_karyotype(paths["analysis_dir"], Path(mcscanx_result["collinearity_file"]), paths["final_dir"], args.prefix, target_group, ref_group) - metrics["synteny_success"] = bool(synteny_plot_result.get("success")) - else: - metrics["synteny_skipped_reason"] = "MCScanX did not produce a collinearity file" - else: - metrics["synteny_skipped_reason"] = "BLAST failed" - else: - metrics["synteny_skipped_reason"] = "Combined GFF generation failed" - else: - missing = [] - if not discovery["target_all_peps"]: - missing.append("target all protein") - if not discovery["ref_all_peps"]: - missing.append("reference all protein") - if not discovery["target_gffs"]: - missing.append("target GFF") - if not discovery["ref_gffs"]: - missing.append("reference GFF") - metrics["synteny_skipped_reason"] = "Missing full data for MCScanX: " + ", ".join(missing) - print(metrics["synteny_skipped_reason"]) - - if synteny_plot_result.get("pdf_file"): - add_artifact(artifacts, "synteny_pdf", synteny_plot_result["pdf_file"], f"final/{args.prefix}.interspecies_synteny.pdf", "synteny_pdf", "plot") - if synteny_plot_result.get("png_file"): - add_artifact(artifacts, "synteny_png", synteny_plot_result["png_file"], f"final/{args.prefix}.interspecies_synteny.png", "synteny_png", "plot") - if synteny_plot_result.get("marked_pdf_file"): - add_artifact(artifacts, "synteny_marked_pdf", synteny_plot_result["marked_pdf_file"], f"final/{args.prefix}.interspecies_synteny.marked.pdf", "synteny_marked_pdf", "plot") - if synteny_plot_result.get("marked_png_file"): - add_artifact(artifacts, "synteny_marked_png", synteny_plot_result["marked_png_file"], f"final/{args.prefix}.interspecies_synteny.marked.png", "synteny_marked_png", "plot") - - end_time = datetime.now() - status = "success" if metrics["tree_success"] and metrics["synteny_success"] else "partial_success" - parameters = { - "prefix": args.prefix, - "gene_family": discovery["gene_family"], - "target_group": target_group, - "ref_group": ref_group, - "target_species": discovery["target_species"], - "ref_species": discovery["ref_species"], - "target_protein_spl": str(discovery["target_family_protein"]), - "ref_protein_spl": str(discovery["ref_family_protein"]), - "input_dir": str(input_dir), - "cpu": args.cpu, - } - report_path = write_report(workspace, args.prefix, metrics, parameters, tool_versions, start_time, end_time, (end_time - start_time).total_seconds(), cmd) - add_artifact(artifacts, "report", report_path, f"final/{args.prefix}.interspecies_synteny_report.md", "report", "report") - manifest_path = write_manifest(workspace, job_id, run_id, args.prefix, start_time, end_time, artifacts, metrics, parameters, tool_versions, workspace_root, input_dir, cmd, status=status) - add_artifact(artifacts, "manifest", manifest_path, f"final/{args.prefix}.interspecies_synteny.manifest.json", "manifest", "json") - write_manifest(workspace, job_id, run_id, args.prefix, start_time, end_time, artifacts, metrics, parameters, tool_versions, workspace_root, input_dir, cmd, status=status) - print(f"Interspecies synteny finished with status: {status}") - print(f"Manifest: {manifest_path}") - return 0 if metrics["tree_success"] else 1 - except Exception as exc: - print(f"Interspecies synteny failed: {exc}") - try: - write_failure_manifest(workspace, job_id, run_id, args.prefix, start_time, workspace_root, input_dir, cmd, str(exc)) - except Exception as manifest_exc: - print(f"Failed to write failure manifest: {manifest_exc}") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bioflow_kernel/cli/run_interspecies_synteny.py.bak_species_select b/bioflow_kernel/cli/run_interspecies_synteny.py.bak_species_select deleted file mode 100755 index 10abf37..0000000 --- a/bioflow_kernel/cli/run_interspecies_synteny.py.bak_species_select +++ /dev/null @@ -1,1609 +0,0 @@ -#!/usr/bin/env python3 -""" -Interspecies Synteny阶段：物种间共线性分析（完整版） - -输入： - - 参考物种蛋白序列 (final/{prefix}.final_members.pep.fa) - - 参考物种GFF3 - - 目标物种蛋白序列 - - 目标物种GFF3 - -输出： - - final/{prefix}.interspecies_tree.nwk (物种间进化树) - - final/{prefix}.interspecies_tree.pdf/png (进化树图) - - final/{prefix}.interspecies_synteny.pdf/png (共线性图) - - final/{prefix}.interspecies_synteny_report.md (分析报告) - - final/{prefix}.interspecies_synteny.manifest.json (机器清单) -""" - -import os -import argparse -import json -import hashlib -import subprocess -import sys -import time -import shutil -import re -from datetime import datetime -from pathlib import Path -from typing import Dict, List, Any, Optional, Tuple - - -# ??conda??bin???PATH -CONDA_BIN = "/home/xia-lab/miniconda3/envs/test4py311/bin" -if CONDA_BIN not in os.environ.get("PATH", ""): - os.environ["PATH"] = CONDA_BIN + ":" + os.environ.get("PATH", "") -def pdf_to_png(pdf_path: Path, png_path: Path, dpi: int = 300) -> bool: - """将PDF转换为PNG""" - try: - # 方法1: 使用pdftoppm (poppler-utils) - cmd = ["pdftoppm", "-png", "-r", str(dpi), "-singlefile", - str(pdf_path), str(png_path.with_suffix(''))] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - if result.returncode == 0: - return True - except Exception: - pass - - try: - # 方法2: 使用convert (ImageMagick) - cmd = ["convert", "-density", str(dpi), str(pdf_path), "-quality", "100", str(png_path)] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) - if result.returncode == 0: - return True - except Exception: - pass - - return False - - -def compute_file_hash(filepath: Path) -> str: - """计算文件的SHA256哈希值""" - sha256_hash = hashlib.sha256() - try: - with open(filepath, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - except Exception: - return "" - - -def get_tool_versions() -> Dict[str, str]: - """获取所需工具版本信息""" - versions = {} - - for tool, cmd in [ - ("mafft", ["mafft", "--version"]), - ("iqtree", ["iqtree", "--version"]), - ("blastp", ["blastp", "-version"]), - ("MCScanX", ["MCScanX", "-h"]), - ]: - try: - result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=5) - versions[tool] = "available" if result.returncode in [0, 1] else "not_found" - except Exception: - versions[tool] = "not_found" - - return versions - - -def prepare_directories(workspace: Path) -> Dict[str, Path]: - """创建目录结构""" - paths = { - "analysis_dir": workspace / "interspecies_synteny" / "analysis", - "final_dir": workspace / "final", - "logs_dir": workspace / "logs", - } - for name, path in paths.items(): - path.mkdir(parents=True, exist_ok=True) - return paths - - -def merge_protein_sequences( - target_protein: Path, - ref_protein: Path, - output_fasta: Path -) -> Dict[str, Any]: - """Merge target and reference family proteins for the interspecies tree.""" - print("Merging family protein sequences...") - - try: - with open(output_fasta, 'w') as outfile: - with open(target_protein, 'r') as f: - outfile.write(f.read().rstrip() + "\n") - with open(ref_protein, 'r') as f: - outfile.write(f.read().rstrip() + "\n") - - target_count = sum(1 for line in open(target_protein) if line.startswith('>')) - ref_count = sum(1 for line in open(ref_protein) if line.startswith('>')) - - print(f" merged target/ref family proteins: {target_count} + {ref_count} = {target_count + ref_count}") - - return { - "success": True, - "output_file": output_fasta, - "target_count": target_count, - "ref_count": ref_count - } - except Exception as e: - return {"success": False, "stderr": str(e)} - - -def build_phylogenetic_tree( - merged_fasta: Path, - output_dir: Path, - prefix: str, - cpu: int = 8 -) -> Dict[str, Any]: - """构建系统发育树 (MAFFT + IQ-TREE)""" - print(f"构建进化树...") - - # MAFFT比对 - msa_file = output_dir / f"{prefix}_interspecies.mafft.msa.fa" - mafft_cmd = ["mafft", "--auto", "--thread", str(cpu), str(merged_fasta)] - - try: - with open(msa_file, "w") as f_out: - result = subprocess.run(mafft_cmd, stdout=f_out, stderr=subprocess.PIPE, - text=True, check=False) - - if result.returncode != 0: - return {"success": False, "error": f"MAFFT失败: {result.stderr}"} - - # IQ-TREE建树 - base_name = output_dir / f"{prefix}_interspecies.iqtree" - iqtree_cmd = [ - "iqtree", "-s", str(msa_file), "-m", "MFP", "-bb", "1000", - "-nt", str(cpu), "-pre", str(base_name) - ] - - result = subprocess.run(iqtree_cmd, capture_output=True, text=True, check=False) - - # IQ-TREE 生成的 treefile 是 {prefix}.iqtree.treefile，不是 {prefix}.treefile - tree_file = Path(f"{base_name}.treefile") - - return { - "success": tree_file.exists(), - "tree_file": tree_file if tree_file.exists() else None, - "msa_file": msa_file - } - except Exception as e: - return {"success": False, "error": str(e)} - - -def filter_gff_files( - ref_gff3: Path, - target_gff3: Path, - output_dir: Path, - ref_species: str, - target_species: str -) -> Dict[str, Any]: - """过滤GFF3文件 - 提取基因ID（去掉转录本版本后缀如 .1, .2）""" - print(f"过滤GFF3文件...") - - import re - - results = {} - for gff3_file, species, prefix in [ - (ref_gff3, ref_species, "ref"), - (target_gff3, target_species, "target") - ]: - output_gff = output_dir / f"{species}.gff" - - try: - with open(gff3_file, 'r') as f: - lines = f.readlines() - - count = 0 - seen_genes = set() # 去重 - - with open(output_gff, 'w') as f: - for line in lines: - parts = line.strip().split('\t') - if len(parts) >= 9 and parts[2] == "mRNA": - attr = parts[8] - # 提取ID（如 ID=AT2G47070.1），保留版本号与BLAST结果匹配 - gene_id = attr.split(';')[0].replace('ID=', '') - - # 处理染色体名称：去掉 "Chr" 前缀（Ath用1,2,3而非Chr1,Chr2） - chr_name = parts[0] - if chr_name.startswith('Chr'): - chr_name = chr_name[3:] # Chr1 -> 1 - - # 去重：一个基因可能有多个转录本 - if gene_id not in seen_genes: - seen_genes.add(gene_id) - f.write(f"{chr_name}\t{gene_id}\t{parts[3]}\t{parts[4]}\n") - count += 1 - - print(f" {species}: {count} 个基因（去重后）") - results[prefix] = {"success": True, "output_file": output_gff, "gene_count": count} - except Exception as e: - results[prefix] = {"success": False, "stderr": str(e)} - - if results.get("ref", {}).get("success") and results.get("target", {}).get("success"): - return { - "success": True, - "ref_gff": results["ref"]["output_file"], - "target_gff": results["target"]["output_file"] - } - return {"success": False} - - -def run_reciprocal_blast( - ref_protein: Path, - target_protein: Path, - output_dir: Path, - ref_species: str, - target_species: str, - cpu: int = 8 -) -> Dict[str, Any]: - """双向BLAST比对 - 结果中基因名去掉版本后缀以匹配GFF""" - import re - print(f"双向BLAST比对...") - - # 构建数据库 - for protein_file in [ref_protein, target_protein]: - subprocess.run(["makeblastdb", "-in", str(protein_file), "-dbtype", "prot"], - capture_output=True, check=False) - - # 双向blastp - blast1_out = output_dir / f"tmp.{ref_species}_{target_species}.blast" - subprocess.run([ - "blastp", "-query", str(ref_protein), "-db", str(target_protein), - "-outfmt", "6", "-out", str(blast1_out), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ], capture_output=True, check=False) - - blast2_out = output_dir / f"tmp.{target_species}_{ref_species}.blast" - subprocess.run([ - "blastp", "-query", str(target_protein), "-db", str(ref_protein), - "-outfmt", "6", "-out", str(blast2_out), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ], capture_output=True, check=False) - - # 合并BLAST结果（保留版本号与GFF匹配） - merged_blast = output_dir / f"{ref_species}_{target_species}.blast" - with open(merged_blast, 'w') as outfile: - for blast_file in [blast1_out, blast2_out]: - if blast_file.exists(): - with open(blast_file, 'r') as infile: - for line in infile: - outfile.write(line) # 直接写入，保留版本号 - - return {"success": True, "output_file": merged_blast} - - -def run_mcscanx_interspecies( - work_dir: Path, - ref_species: str, - target_species: str -) -> Dict[str, Any]: - """MCScanX种间共线性 - 需要合并的GFF文件""" - print(f"MCScanX种间共线性...") - - prefix = f"{ref_species}_{target_species}" - - # MCScanX 需要合并的 GFF 文件 (prefix.gff) - merged_gff = work_dir / f"{prefix}.gff" - ref_gff = work_dir / f"{ref_species}.gff" - target_gff = work_dir / f"{target_species}.gff" - - if not merged_gff.exists() and ref_gff.exists() and target_gff.exists(): - print(f" 合并 GFF 文件: {ref_gff.name} + {target_gff.name} -> {merged_gff.name}") - with open(merged_gff, 'w') as outfile: - for gff_file in [target_gff, ref_gff]: # 目标物种在前，参考物种在后 - with open(gff_file, 'r') as infile: - outfile.write(infile.read()) - - cmd = ["MCScanX", f"./{prefix}"] - - result = subprocess.run(cmd, capture_output=True, text=True, - cwd=str(work_dir), check=False) - - collinearity_file = work_dir / f"{prefix}.collinearity" - - return { - "success": collinearity_file.exists(), - "collinearity_file": collinearity_file if collinearity_file.exists() else None - } - - - - -def fasta_ids(fasta_file: Path) -> List[str]: - ids = [] - with open(fasta_file, "r") as f: - for line in f: - if line.startswith(">"): - ids.append(line[1:].strip().split()[0]) - return ids - - -def display_species_code(species: str, gene_id: str = "") -> str: - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return species - - -def species_for_gene(gene_id: str, species_list: List[str], default_species: str) -> str: - for species in sorted(species_list, key=len, reverse=True): - if gene_id.lower().startswith(species.lower()): - return display_species_code(species, gene_id) - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return display_species_code(default_species, gene_id) - - -def write_gene_species_map(target_fasta: Path, ref_fasta: Path, target_species: List[str], ref_species: List[str], output_file: Path) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - default_target = target_species[0] if target_species else "Target" - default_ref = ref_species[0] if ref_species else "Ref" - with open(output_file, "w") as out: - out.write("gene_id\tspecies\tgroup\n") - for gene_id in fasta_ids(target_fasta): - out.write(f"{gene_id}\t{species_for_gene(gene_id, target_species, default_target)}\ttarget\n") - for gene_id in fasta_ids(ref_fasta): - out.write(f"{gene_id}\t{species_for_gene(gene_id, ref_species, default_ref)}\tref\n") - return output_file -def normalize_seq_id(text: str) -> str: - """Normalize FASTA/GFF IDs while preserving biologically meaningful versions.""" - value = (text or "").strip() - if value.startswith(">"): - value = value[1:] - value = value.split()[0] if value.split() else value - if "|" in value: - value = value.split("|", 1)[0] - for prefix in ("gene:", "transcript:", "CDS:", "rna-", "gene-", "cds-"): - if value.startswith(prefix): - value = value[len(prefix):] - return value.strip().strip('"') - - -def id_variants(gene_id: str) -> set: - norm = normalize_seq_id(gene_id) - variants = {norm} - for prefix in ("gene:", "transcript:", "CDS:", "rna-", "gene-", "cds-"): - variants.add(normalize_seq_id(prefix + norm)) - if "|" in gene_id: - variants.add(normalize_seq_id(gene_id.split("|", 1)[0])) - return {v for v in variants if v} - - -def fasta_ids(fasta_file: Path) -> List[str]: - ids = [] - with open(fasta_file, "r") as f: - for line in f: - if line.startswith(">"): - seq_id = normalize_seq_id(line[1:]) - if seq_id: - ids.append(seq_id) - return ids - - -def fasta_id_set(fasta_file: Path) -> set: - ids = set() - for seq_id in fasta_ids(fasta_file): - ids.update(id_variants(seq_id)) - return ids - - -def display_species_code(species: str, gene_id: str = "") -> str: - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return species - - -def species_for_gene(gene_id: str, species_list: List[str], default_species: str, species_id_sets: Optional[Dict[str, set]] = None) -> str: - variants = id_variants(gene_id) - if species_id_sets: - for species, ids in species_id_sets.items(): - if variants & ids: - return display_species_code(species, gene_id) - for species in sorted(species_list, key=len, reverse=True): - if normalize_seq_id(gene_id).lower().startswith(species.lower()): - return display_species_code(species, gene_id) - if re.match(r"^AT[1-5CM]G", gene_id, re.IGNORECASE): - return "AT" - return display_species_code(default_species, gene_id) - - -def collect_species_id_sets(groups: Dict[str, Any]) -> Dict[str, set]: - species_id_sets: Dict[str, set] = {} - for group_name in ("target", "ref"): - for species, item in groups.get(group_name, {}).items(): - pep = item.get("resolved", {}).get("pep") - if pep and Path(pep).exists(): - species_id_sets[species] = fasta_id_set(Path(pep)) - return species_id_sets - - -def write_gene_species_map( - target_fasta: Path, - ref_fasta: Path, - target_species: List[str], - ref_species: List[str], - output_file: Path, - species_id_sets: Optional[Dict[str, set]] = None, -) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - default_target = target_species[0] if target_species else "Target" - default_ref = ref_species[0] if ref_species else "Ref" - with open(output_file, "w") as out: - out.write("gene_id\tspecies\tgroup\n") - for gene_id in fasta_ids(target_fasta): - species = species_for_gene(gene_id, target_species, default_target, species_id_sets) - out.write(f"{gene_id}\t{species}\ttarget\n") - for gene_id in fasta_ids(ref_fasta): - species = species_for_gene(gene_id, ref_species, default_ref, species_id_sets) - out.write(f"{gene_id}\t{species}\tref\n") - return output_file - - -def run_reciprocal_blast( - target_protein: Path, - ref_protein: Path, - output_dir: Path, - target_species: str, - ref_species: str, - cpu: int = 8 -) -> Dict[str, Any]: - """Run pairwise reciprocal BLASTP for one target-ref species pair.""" - print(f"Reciprocal BLASTP: {target_species} vs {ref_species}") - - pair_prefix = f"{target_species}.{ref_species}" - target_db = output_dir / f"{target_species}.blastdb" - ref_db = output_dir / f"{ref_species}.blastdb" - commands = [] - - for protein_file, db_prefix in [(target_protein, target_db), (ref_protein, ref_db)]: - cmd = ["makeblastdb", "-in", str(protein_file), "-dbtype", "prot", "-out", str(db_prefix)] - commands.append(" ".join(cmd)) - result = subprocess.run(cmd, capture_output=True, text=True, check=False) - if result.returncode != 0: - return {"success": False, "stderr": result.stderr, "commands": commands} - - target_to_ref = output_dir / f"tmp.{target_species}_{ref_species}.blast" - ref_to_target = output_dir / f"tmp.{ref_species}_{target_species}.blast" - for query, db_prefix, out_file in [(target_protein, ref_db, target_to_ref), (ref_protein, target_db, ref_to_target)]: - cmd = [ - "blastp", "-query", str(query), "-db", str(db_prefix), - "-outfmt", "6", "-out", str(out_file), "-evalue", "1e-10", - "-num_alignments", "5", "-num_threads", str(cpu) - ] - commands.append(" ".join(cmd)) - result = subprocess.run(cmd, capture_output=True, text=True, check=False) - if result.returncode != 0: - return {"success": False, "stderr": result.stderr, "commands": commands} - - merged_blast = output_dir / f"{pair_prefix}.blast" - with open(merged_blast, 'w') as outfile: - for blast_file in [target_to_ref, ref_to_target]: - if blast_file.exists(): - with open(blast_file, 'r') as infile: - for line in infile: - if line.strip(): - outfile.write(line) - - return {"success": merged_blast.exists() and merged_blast.stat().st_size > 0, "output_file": merged_blast, "commands": commands} - - -def run_mcscanx_interspecies(work_dir: Path, target_species: str, ref_species: str) -> Dict[str, Any]: - """Run MCScanX for one target-ref pair.""" - pair_prefix = f"{target_species}.{ref_species}" - print(f"MCScanX pair: {pair_prefix}") - - merged_gff = work_dir / f"{pair_prefix}.gff" - target_gff = work_dir / f"{target_species}.gff" - ref_gff = work_dir / f"{ref_species}.gff" - - if target_gff.exists() and ref_gff.exists(): - with open(merged_gff, 'w') as outfile: - for gff_file in [target_gff, ref_gff]: - with open(gff_file, 'r') as infile: - outfile.write(infile.read().rstrip() + "\n") - else: - return {"success": False, "error": f"Missing simplified GFF for {pair_prefix}"} - - result = subprocess.run(["MCScanX", f"./{pair_prefix}"], capture_output=True, text=True, cwd=str(work_dir), check=False) - collinearity_file = work_dir / f"{pair_prefix}.collinearity" - return { - "success": collinearity_file.exists() and collinearity_file.stat().st_size > 0, - "collinearity_file": collinearity_file if collinearity_file.exists() else None, - "stdout": result.stdout, - "stderr": result.stderr, - "returncode": result.returncode, - } - - -def plot_interspecies_tree( - tree_file: Path, - output_dir: Path, - prefix: str, - target_species: str, - ref_species: str, - target_subfamily_file: Optional[Path] = None, - ref_subfamily_file: Optional[Path] = None, - gene_species_map: Optional[Path] = None -) -> Dict[str, Any]: - """绘制物种间进化树图 - 使用专用的 interspecies_tree.R 脚本""" - print(f"绘制进化树图...") - - # 查找专用的种间进化树R脚本 - r_script = Path("scripts/interspecies_tree.R") - if not r_script.exists(): - alt_script = Path("/home/xia-lab/wutx/00test/bbav2/scripts/interspecies_tree.R") - if alt_script.exists(): - r_script = alt_script - else: - return {"success": False, "error": f"R脚本不存在: {r_script}"} - - try: - # 使用专用的 interspecies_tree.R 脚本，传递物种参数和亚家族文件 - # 使用conda环境的Rscript - conda_rscript = "/home/xia-lab/miniconda3/envs/test4py311/bin/Rscript" - cmd = [ - conda_rscript, str(r_script), - str(tree_file), - str(output_dir), - target_species, - ref_species - ] - - # 添加亚家族文件参数（如果存在） - if target_subfamily_file and target_subfamily_file.exists(): - cmd.append(str(target_subfamily_file)) - print(f" 目标物种亚家族文件: {target_subfamily_file}") - else: - cmd.append("") # 空字符串表示无文件 - - if ref_subfamily_file and ref_subfamily_file.exists(): - cmd.append(str(ref_subfamily_file)) - print(f" 参考物种亚家族文件: {ref_subfamily_file}") - else: - cmd.append("") # 空字符串表示无文件 - if gene_species_map and gene_species_map.exists(): - cmd.append(str(gene_species_map)) - print(f" Gene-to-species map: {gene_species_map}") - else: - cmd.append("") - result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=120) - - # 调试输出 - print(f" R脚本返回码: {result.returncode}") - if result.stdout: - print(f" R脚本输出: {result.stdout[:500]}") - if result.stderr: - print(f" R脚本错误: {result.stderr[:500]}") - - # 专用脚本输出固定为 interspecies_tree.pdf/png - output_pdf = output_dir / "interspecies_tree.pdf" - output_png = output_dir / "interspecies_tree.png" - - # 重命名为规范名称 - final_pdf = output_dir / f"{prefix}.interspecies_tree.pdf" - final_png = output_dir / f"{prefix}.interspecies_tree.png" - - if output_pdf.exists(): - shutil.move(output_pdf, final_pdf) - print(f" 进化树PDF: {final_pdf}") - if output_png.exists(): - shutil.move(output_png, final_png) - print(f" 进化树PNG: {final_png}") - - # PDF转PNG（如果没有生成PNG） - if final_pdf.exists() and not final_png.exists(): - png_from_pdf = output_dir / "interspecies_tree.png" - if pdf_to_png(final_pdf, png_from_pdf): - final_png = output_dir / f"{prefix}.interspecies_tree.png" - shutil.move(png_from_pdf, final_png) - print(f" PDF转PNG: {final_png}") - - return { - "success": final_pdf.exists() or final_png.exists(), - "pdf_file": final_pdf if final_pdf.exists() else None, - "png_file": final_png if final_png.exists() else None, - "stdout": result.stdout, - "stderr": result.stderr - } - except Exception as e: - return {"success": False, "error": str(e)} - - -def convert_blast_to_family_simple( - work_dir: Path, - blast_file: Path, - target_species: str, - ref_species: str, -) -> Dict[str, Any]: - """Fallback: convert pairwise BLAST hits directly to JCVI anchors/simple. - - This is not a strict MCScanX collinearity block. It is used when the ref side - only has family proteins, so MCScanX cannot satisfy whole-genome block rules. - """ - pair_prefix = f"{target_species}.{ref_species}" - target_bed = work_dir / f"{target_species}.bed" - ref_bed = work_dir / f"{ref_species}.bed" - anchors_file = work_dir / f"{pair_prefix}.anchors" - simple_file = work_dir / f"{pair_prefix}.anchors.simple" - - valid_target = {line.rstrip("\n").split("\t")[3] for line in open(target_bed) if len(line.rstrip("\n").split("\t")) >= 4} - valid_ref = {line.rstrip("\n").split("\t")[3] for line in open(ref_bed) if len(line.rstrip("\n").split("\t")) >= 4} - seen = set() - links = [] - if not blast_file.exists(): - return {"success": False, "error": "BLAST file missing for family-only fallback"} - with open(blast_file, "r") as f: - for line in f: - if not line.strip() or line.startswith("#"): - continue - parts = line.rstrip("\n").split("\t") - if len(parts) < 2: - continue - a = normalize_seq_id(parts[0]) - b = normalize_seq_id(parts[1]) - target_gene = ref_gene = None - if a in valid_target and b in valid_ref: - target_gene, ref_gene = a, b - elif b in valid_target and a in valid_ref: - target_gene, ref_gene = b, a - if not target_gene or not ref_gene: - continue - key = (target_gene, ref_gene) - if key in seen: - continue - seen.add(key) - score_text = parts[11] if len(parts) > 11 else "1000" - try: - score = str(int(float(score_text))) - except Exception: - score = "1000" - links.append((target_gene, ref_gene, score)) - if not links: - return {"success": False, "error": "No BLAST hits matched pair BED IDs for family-only fallback"} - with open(anchors_file, "w") as out: - out.write("##\n") - for target_gene, ref_gene, score in links: - out.write(f"{target_gene}\t{ref_gene}\t{score}\n") - with open(simple_file, "w") as out: - for target_gene, ref_gene, score in links: - out.write(f"{target_gene}\t{target_gene}\t{ref_gene}\t{ref_gene}\t{score}\t+\n") - return { - "success": True, - "anchors_file": anchors_file, - "simple_file": simple_file, - "anchor_count": len(links), - "fallback": "family_only_blast_links", - } - - -def convert_collinearity_to_simple( - work_dir: Path, - collinearity_file: Path, - target_species: str, - ref_species: str, -) -> Dict[str, Any]: - """Convert one MCScanX .collinearity file into JCVI anchors/simple files.""" - conda_python = "/home/xia-lab/miniconda3/envs/test4py311/bin/python" - pair_prefix = f"{target_species}.{ref_species}" - target_bed = work_dir / f"{target_species}.bed" - ref_bed = work_dir / f"{ref_species}.bed" - anchors_file = work_dir / f"{pair_prefix}.anchors" - anchors_new_file = work_dir / f"{pair_prefix}.anchors.new" - simple_file = work_dir / f"{pair_prefix}.anchors.simple" - - valid_target = {line.rstrip("\n").split("\t")[3] for line in open(target_bed) if len(line.rstrip("\n").split("\t")) >= 4} - valid_ref = {line.rstrip("\n").split("\t")[3] for line in open(ref_bed) if len(line.rstrip("\n").split("\t")) >= 4} - - current_block = [] - anchors = 0 - filtered = 0 - with open(collinearity_file, "r") as f_in, open(anchors_file, "w") as f_out: - for line in f_in: - if not line.strip(): - continue - if line.startswith("## Alignment"): - if current_block: - f_out.write("##\n") - f_out.writelines(current_block) - current_block = [] - continue - if line.startswith("#"): - continue - parts = line.strip().split("\t") - if len(parts) < 3: - continue - gene_a = normalize_seq_id(parts[1]) - gene_b = normalize_seq_id(parts[2]) - target_gene = ref_gene = None - if gene_a in valid_target and gene_b in valid_ref: - target_gene, ref_gene = gene_a, gene_b - elif gene_b in valid_target and gene_a in valid_ref: - target_gene, ref_gene = gene_b, gene_a - if target_gene and ref_gene: - current_block.append(f"{target_gene} {ref_gene} 1000\n") - anchors += 1 - else: - filtered += 1 - if current_block: - f_out.write("##\n") - f_out.writelines(current_block) - - if anchors == 0: - return {"success": False, "error": "No anchors matched pair BED IDs", "anchors_file": anchors_file, "filtered": filtered} - - cmd_screen = [ - conda_python, "-m", "jcvi.compara.synteny", "screen", - "--minspan=10", "--simple", str(anchors_file), str(anchors_new_file), - f"--qbed={target_bed.name}", f"--sbed={ref_bed.name}", - ] - result = subprocess.run(cmd_screen, capture_output=True, text=True, cwd=str(work_dir), check=False, timeout=180) - for candidate in (work_dir / f"{pair_prefix}.anchors.new.simple", work_dir / f"{pair_prefix}.new.simple"): - if candidate.exists(): - shutil.copy(candidate, simple_file) - break - if not simple_file.exists(): - shutil.copy(anchors_file, simple_file) - - return { - "success": simple_file.exists() and simple_file.stat().st_size > 0, - "anchors_file": anchors_file, - "anchors_new_file": anchors_new_file, - "simple_file": simple_file, - "anchor_count": anchors, - "filtered_count": filtered, - "screen_returncode": result.returncode, - "screen_stderr": result.stderr, - } - - -def _chr_sort_key(chrom: str): - cleaned = re.sub(r"^(chr|Chr|CHR)", "", chrom) - if cleaned.isdigit(): - return (0, int(cleaned)) - return (1, cleaned) - - -def _read_bed_chroms(bed_file: Path) -> List[str]: - skip = {"C", "M", "ChrC", "ChrM", "chrc", "chrm"} - chroms = set() - with open(bed_file, "r") as f: - for line in f: - parts = line.strip().split("\t") - if len(parts) >= 4 and parts[0] not in skip: - chroms.add(parts[0]) - return sorted(chroms, key=_chr_sort_key) - - -def _bed_gene_ids(bed_file: Path) -> set: - ids = set() - if not bed_file.exists(): - return ids - with open(bed_file, "r") as f: - for line in f: - parts = line.rstrip("\n").split("\t") - if len(parts) >= 4: - ids.add(normalize_seq_id(parts[3])) - return ids - - -def _is_family_gene(gene_id: str, family_ids: set) -> bool: - return bool(set(id_variants(gene_id)) & family_ids) - - -def _family_links_from_blast( - blast_file: Path, - family_ids: set, - target_bed_ids: set, - ref_bed_ids: set, - max_links: int = 300, -) -> List[Tuple[str, str, int]]: - if not blast_file or not blast_file.exists() or not family_ids: - return [] - best: Dict[Tuple[str, str], float] = {} - with open(blast_file, "r") as f: - for line in f: - parts = line.rstrip("\n").split("\t") - if len(parts) < 12: - continue - q, s = normalize_seq_id(parts[0]), normalize_seq_id(parts[1]) - if q not in target_bed_ids or s not in ref_bed_ids: - continue - if not (_is_family_gene(q, family_ids) and _is_family_gene(s, family_ids)): - continue - try: - score = float(parts[11]) - except Exception: - score = 1.0 - key = (q, s) - if score > best.get(key, -1): - best[key] = score - ranked = sorted(best.items(), key=lambda item: item[1], reverse=True)[:max_links] - return [(q, s, max(1, int(round(score)))) for (q, s), score in ranked] - - -def _write_marked_simple_file( - simple_file: Path, - marked_file: Path, - family_ids: set, - family_links: List[Tuple[str, str, int]], -) -> int: - if not simple_file.exists() or not family_ids: - return 0 - marked_count = 0 - seen_rows = set() - with open(simple_file, "r") as f_in, open(marked_file, "w") as f_out: - for line in f_in: - stripped = line.strip() - if not stripped or stripped.startswith("#"): - f_out.write(line) - continue - parts = stripped.split() - genes = [normalize_seq_id(p.split("*", 1)[-1]) for p in parts[:4]] - row_key = tuple(parts[:6]) if len(parts) >= 6 else tuple(parts) - seen_rows.add(row_key) - if any(_is_family_gene(g, family_ids) for g in genes) and "*" not in parts[0]: - f_out.write("#e31a1c*" + "\t".join(parts) + "\n") - marked_count += 1 - else: - f_out.write(line) - for q, s, score in family_links: - row = (q, q, s, s, str(score), "+") - if row in seen_rows: - continue - f_out.write("#e31a1c*" + "\t".join(row) + "\n") - seen_rows.add(row) - marked_count += 1 - return marked_count - - -def plot_multi_species_synteny( - work_dir: Path, - output_dir: Path, - prefix: str, - species_order: List[str], - pair_results: List[Dict[str, Any]], - family_ids: set, -) -> Dict[str, Any]: - """Draw one multi-track JCVI karyotype figure from all successful target-ref pairs.""" - print("Plotting multi-species JCVI synteny...") - conda_python = "/home/xia-lab/miniconda3/envs/test4py311/bin/python" - usable_pairs = [p for p in pair_results if p.get("simple_file") and Path(p["simple_file"]).exists()] - if not usable_pairs: - return {"success": False, "error": "No pairwise anchors.simple files available for JCVI"} - - seqids_file = work_dir / f"{prefix}.seqids" - layout_file = work_dir / f"{prefix}.layout" - layout_mark_file = work_dir / f"{prefix}.layout_mark" - - species_with_bed = [] - with open(seqids_file, "w") as f: - for species in species_order: - bed = work_dir / f"{species}.bed" - if not bed.exists(): - continue - chroms = _read_bed_chroms(bed) - if not chroms: - continue - species_with_bed.append(species) - f.write(",".join(chroms) + "\n") - - if len(species_with_bed) < 2: - return {"success": False, "error": "Fewer than two species BED tracks are available"} - - index = {species: i for i, species in enumerate(species_with_bed)} - n = len(species_with_bed) - y_values = [round(0.85 - (0.7 * i / max(n - 1, 1)), 3) for i in range(n)] - palette = ["#d73027", "#1a9850", "#4575b4", "#984ea3", "#ff7f00", "#4daf4a", "#377eb8", "#a65628"] - - def write_layout(layout: Path, use_marked: bool) -> None: - with open(layout, "w") as f: - f.write("# y, xstart, xend, rotation, color, label, va, bed\n") - for i, species in enumerate(species_with_bed): - va = "top" if i == 0 else "bottom" - f.write(f"{y_values[i]}, .08, .92, 0, {palette[i % len(palette)]}, {species}, {va}, {species}.bed\n") - f.write("# edges\n") - for pair in usable_pairs: - target = pair["target_species"] - ref = pair["ref_species"] - if target not in index or ref not in index: - continue - edge = Path(pair.get("marked_simple_file") if use_marked and pair.get("marked_simple_file") else pair["simple_file"]).name - f.write(f"e, {index[target]}, {index[ref]}, {edge}\n") - - for pair in usable_pairs: - simple = Path(pair["simple_file"]) - marked = work_dir / f"{pair['target_species']}.{pair['ref_species']}.anchors.simple.mark" - target_bed_ids = _bed_gene_ids(work_dir / f"{pair['target_species']}.bed") - ref_bed_ids = _bed_gene_ids(work_dir / f"{pair['ref_species']}.bed") - family_links = _family_links_from_blast(Path(pair.get("blast_file", "")), family_ids, target_bed_ids, ref_bed_ids) - marked_count = _write_marked_simple_file(simple, marked, family_ids, family_links) - pair["family_highlight_links"] = len(family_links) - pair["marked_block_count"] = marked_count - if marked_count > 0: - pair["marked_simple_file"] = str(marked) - - write_layout(layout_file, use_marked=False) - write_layout(layout_mark_file, use_marked=True) - - def run_plot(layout: Path, final_pdf: Path, final_png: Path) -> Dict[str, Any]: - for stale in [work_dir / "karyotype.pdf", work_dir / "karyotype.png", final_pdf, final_png]: - if stale.exists(): - stale.unlink() - cmd = [conda_python, "-m", "jcvi.graphics.karyotype", str(seqids_file.resolve()), str(layout.resolve()), "--notex"] - result = subprocess.run(cmd, capture_output=True, text=True, cwd=str(work_dir), check=False, timeout=180) - if (work_dir / "karyotype.pdf").exists(): - shutil.copy(work_dir / "karyotype.pdf", final_pdf.resolve()) - if (work_dir / "karyotype.png").exists(): - shutil.copy(work_dir / "karyotype.png", final_png.resolve()) - if final_pdf.exists() and (not final_png.exists() or final_png.stat().st_mtime < final_pdf.stat().st_mtime): - pdf_to_png(final_pdf, final_png) - return {"returncode": result.returncode, "stdout": result.stdout, "stderr": result.stderr} - - final_pdf = output_dir / f"{prefix}.interspecies_synteny.pdf" - final_png = output_dir / f"{prefix}.interspecies_synteny.png" - final_mark_pdf = output_dir / f"{prefix}.interspecies_synteny.marked.pdf" - final_mark_png = output_dir / f"{prefix}.interspecies_synteny.marked.png" - - base_result = run_plot(layout_file, final_pdf, final_png) - mark_result = run_plot(layout_mark_file, final_mark_pdf, final_mark_png) - - return { - "success": final_pdf.exists() or final_png.exists(), - "pdf_file": final_pdf if final_pdf.exists() else None, - "png_file": final_png if final_png.exists() else None, - "marked_pdf_file": final_mark_pdf if final_mark_pdf.exists() else None, - "marked_png_file": final_mark_png if final_mark_png.exists() else None, - "seqids_file": seqids_file, - "layout_file": layout_file, - "layout_mark_file": layout_mark_file, - "base_plot": base_result, - "marked_plot": mark_result, - } - - -def write_report(workspace: Path, prefix: str, metrics: Dict, parameters: Dict, - tool_versions: Dict, start_time: datetime, end_time: datetime, - elapsed_sec: float, cmd: List[str]) -> Path: - """写入报告""" - report_path = workspace / "final" / f"{prefix}.interspecies_synteny_report.md" - - # 收集可复现性信息 - reproducibility_info = [] - reproducibility_info.append("## 可复现性信息\n\n") - reproducibility_info.append("### 执行环境\n") - reproducibility_info.append(f"- **分析时间**: {start_time.isoformat()}\n") - reproducibility_info.append(f"- **完成时间**: {end_time.isoformat()}\n") - reproducibility_info.append(f"- **运行时长**: {elapsed_sec:.2f} 秒\n") - reproducibility_info.append(f"- **工作目录**: {workspace}\n") - reproducibility_info.append(f"- **输入目录**: {parameters.get('input_dir', 'N/A')}\n\n") - - reproducibility_info.append("### 工具版本\n") - for tool, version in tool_versions.items(): - reproducibility_info.append(f"- **{tool}**: {version}\n") - reproducibility_info.append("\n") - - reproducibility_info.append("### 执行命令\n") - reproducibility_info.append(f"```bash\n{' '.join(cmd)}\n```\n\n") - - reproducibility_info.append("### 输入参数\n") - for key, value in parameters.items(): - if key != 'input_dir': - reproducibility_info.append(f"- **{key}**: {value}\n") - reproducibility_info.append("\n") - - with open(report_path, "w") as f: - f.write(f"# 物种间共线性分析 - {prefix}\n\n") - f.write(f"**时间**: {datetime.now().isoformat()}\n\n") - f.write(f"- 目标物种 (研究对象): {parameters.get('target_species')}\n") - f.write(f"- 参考物种 (对照): {parameters.get('ref_species')}\n") - f.write(f"- 目标蛋白文件: {parameters.get('target_protein_spl')}\n") - f.write(f"- 参考蛋白文件: {parameters.get('ref_protein_spl')}\n") - f.write(f"- 目标全蛋白组: {parameters.get('target_protein_all')}\n") - f.write(f"- 参考全蛋白组: {parameters.get('ref_protein_all')}\n") - f.write("\n") - f.writelines(reproducibility_info) - return report_path - - -def get_file_info(file_path: Path) -> Dict[str, Any]: - """获取文件的详细信息，用于artifact规范""" - info = { - "size": 0, - "sha256": "", - "mtime": 0 - } - if file_path.exists(): - info["size"] = file_path.stat().st_size - info["mtime"] = file_path.stat().st_mtime - info["sha256"] = compute_file_hash(file_path) - return info - - -def write_manifest(workspace: Path, job_id: str, run_id: str, prefix: str, - start_time: datetime, end_time: datetime, artifacts: Dict, - metrics: Dict, parameters: Dict, tool_versions: Dict, - workspace_root: Path, input_dir: Path, cmd: List[str], - status: str = "success") -> Path: - """写入manifest - 符合schema v1标准""" - elapsed_sec = (end_time - start_time).total_seconds() - - # 构建符合规范的artifacts结构 - formatted_artifacts = {} - for key, value in artifacts.items(): - if isinstance(value, dict): - # 已经是字典格式，确保包含必需字段 - formatted_artifacts[key] = value - elif isinstance(value, str): - # 字符串格式，转换为artifact对象 - file_path = workspace / value - file_info = get_file_info(file_path) - formatted_artifacts[key] = { - "artifact": key, - "path": value, - "full_path": str(file_path.absolute()), - "type": "file", - "size": file_info["size"], - "sha256": file_info["sha256"], - "mtime": file_info["mtime"] - } - - manifest_data = { - "manifest_version": "1.0", - "job_id": job_id, - "stage": "interspecies_synteny", - "run_id": run_id, - "prefix": prefix, - "timestamp": end_time.isoformat(), - "status": status, - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - "elapsed_sec": round(elapsed_sec, 2), - "cmd": cmd, - "workspace_root": str(workspace_root.absolute()), - "workspace": str(workspace.absolute()), - "input_dir": str(input_dir.absolute()), - "tool_versions": tool_versions, - "artifacts": formatted_artifacts, - "parameters": parameters, - "metrics": metrics - } - - manifest_path = workspace / "final" / f"{prefix}.interspecies_synteny.manifest.json" - with open(manifest_path, "w", encoding='utf-8') as f: - json.dump(manifest_data, f, indent=2, ensure_ascii=False) - - return manifest_path - - - -def _clean_bool(value: Any) -> bool: - return str(value).strip().lower() in {"true", "1", "yes", "y"} - - -def _gene_family_from_prefix(prefix: str) -> str: - return prefix.split('.', 1)[1] if '.' in prefix else prefix - - -def _resolve_path(path_text: str, input_dir: Path) -> Optional[Path]: - if not path_text: - return None - path = Path(path_text) - if path.is_absolute(): - return path if path.exists() else None - for candidate in (input_dir / path, input_dir / path.name): - if candidate.exists(): - return candidate - return None - - -def _read_config_groups(input_dir: Path, default_family: str) -> Dict[str, Any]: - config_path = input_dir / "config.xlsx" - groups = {"target": {}, "ref": {}, "gene_family": default_family} - if not config_path.exists(): - return groups - try: - import pandas as pd - df = pd.read_excel(config_path) - except Exception as exc: - print(f"[WARN] Failed to read config.xlsx: {exc}") - return groups - for _, row in df.iterrows(): - species = str(row.get("Species_Code", "")).strip() - family = str(row.get("Gene_Family", default_family)).strip() or default_family - file_type = str(row.get("File_Type", "")).strip().lower() - original_name = str(row.get("Original_File_Name", "")).strip() - if not species: - continue - group_name = "ref" if _clean_bool(row.get("Ref", False)) else "target" - item = groups[group_name].setdefault(species, {"species": species, "gene_family": family, "files": {}}) - item["gene_family"] = family - if file_type: - item["files"][file_type] = original_name - groups["gene_family"] = family or groups["gene_family"] - return groups - - -def _read_genome_prepare_artifacts(workspace: Path, input_dir: Path, prefix: str) -> Dict[str, Dict[str, Path]]: - result: Dict[str, Dict[str, Path]] = {} - for manifest_path in (workspace / "final" / f"{prefix}.genome_prepare.manifest.json", input_dir / "final" / f"{prefix}.genome_prepare.manifest.json"): - if not manifest_path.exists(): - continue - try: - data = json.loads(manifest_path.read_text(encoding="utf-8")) - except Exception as exc: - print(f"[WARN] Failed to read genome_prepare manifest {manifest_path}: {exc}") - continue - for key, artifact in (data.get("artifacts") or {}).items(): - kind = artifact.get("artifact") - species = key.rsplit("_", 1)[0] - found = _resolve_path(artifact.get("full_path") or artifact.get("path") or "", input_dir) - if found and kind: - result.setdefault(species, {})[kind] = found - return result - - -def _find_species_file(input_dir: Path, species: str, family: str, file_type: str, original_name: str = "") -> Optional[Path]: - if file_type == "all_pep": - patterns = [ - f"{species}.{family}.all_pep.fasta", f"{species}.{family}.all_pep.fa", - f"{species}.{family}.protein.fasta", f"{species}.{family}.faa", - f"{species}.{family}.pep.fasta", f"{species}.{family}.pep.fa", - f"{species}.all_pep.fasta", f"{species}.pep.fasta", f"{species}.protein.fasta", f"{species}.faa", - ] - elif file_type == "family_pep": - patterns = [ - f"{species}.{family}.family_pep.fasta", f"{species}.{family}.family_pep.fa", - f"{species}.{family}.pep.fasta", f"{species}.{family}.pep.fa", - ] - elif file_type == "pep": - patterns = [ - f"{species}.{family}.pep.fasta", f"{species}.{family}.pep.fa", - f"{species}.{family}.protein.fasta", f"{species}.{family}.faa", - f"{species}.pep.fasta", f"{species}.pep.fa", f"{species}.protein.fasta", f"{species}.faa", - ] - elif file_type == "gff": - patterns = [ - f"{species}.{family}_final.gff3", f"{species}.{family}.gff3", f"{species}.{family}.gff", - f"{species}_final.gff3", f"{species}.gff3", f"{species}.gff", - ] - else: - patterns = [] - if original_name: - patterns.append(Path(original_name).name) - for pattern in patterns: - for candidate in (input_dir / pattern, input_dir / species / pattern): - if candidate.exists(): - return candidate - return None - - -def discover_group_inputs(workspace: Path, input_dir: Path, prefix: str, args) -> Dict[str, Any]: - family = _gene_family_from_prefix(prefix) - groups = _read_config_groups(input_dir, family) - artifacts = _read_genome_prepare_artifacts(workspace, input_dir, prefix) - family = groups.get("gene_family") or family - - for group_name in ("target", "ref"): - for species, item in groups[group_name].items(): - files = item.setdefault("resolved", {}) - artifact_paths = artifacts.get(species, {}) - original_files = item.get("files", {}) - item_family = item.get("gene_family", family) - all_original = original_files.get("all_pep", "") or original_files.get("protein", "") - family_original = original_files.get("family_pep", "") - legacy_pep_original = original_files.get("pep", "") - if group_name == "ref" and legacy_pep_original and not family_original: - family_original = legacy_pep_original - if group_name == "target" and legacy_pep_original and not all_original: - all_original = legacy_pep_original - files["all_pep"] = ( - _find_species_file(input_dir, species, item_family, "all_pep", all_original) or - artifact_paths.get("all_pep") or artifact_paths.get("ref_all_pep") or artifact_paths.get("pep") - ) - files["family_pep"] = ( - _find_species_file(input_dir, species, item_family, "family_pep", family_original) or - artifact_paths.get("family_pep") or artifact_paths.get("ref_family_pep") or artifact_paths.get("ref_pep") - ) - files["pep"] = files.get("all_pep") or files.get("family_pep") - files["gff"] = ( - artifact_paths.get("gff") or artifact_paths.get("ref_gff") or - _find_species_file(input_dir, species, item_family, "gff", original_files.get("gff", "")) - ) - - target_family_protein = Path(args.target_protein) if args.target_protein else workspace / "final" / f"{prefix}.final_members.pep.fa" - ref_family_protein = Path(args.ref_protein) if args.ref_protein else None - if ref_family_protein is None: - for candidate in [input_dir / f"Ref_combined.{family}.pep.fasta", input_dir / f"Ref_combined.{family}.pep.fa", input_dir / "Ref_combined.pep.fasta"]: - if candidate.exists(): - ref_family_protein = candidate - break - if ref_family_protein is None: - ref_peps = [item.get("resolved", {}).get("family_pep") for item in groups["ref"].values() if item.get("resolved", {}).get("family_pep")] - if ref_peps: - ref_family_protein = workspace / "interspecies_synteny" / "analysis" / f"Ref_combined.{family}.pep.fasta" - merge_many_fastas(ref_peps, ref_family_protein) - - target_all_peps = {sp: item.get("resolved", {}).get("all_pep") for sp, item in groups["target"].items() if item.get("resolved", {}).get("all_pep")} - ref_all_peps = {sp: item.get("resolved", {}).get("all_pep") for sp, item in groups["ref"].items() if item.get("resolved", {}).get("all_pep")} - target_gffs = {sp: item.get("resolved", {}).get("gff") for sp, item in groups["target"].items() if item.get("resolved", {}).get("gff")} - ref_gffs = {sp: item.get("resolved", {}).get("gff") for sp, item in groups["ref"].items() if item.get("resolved", {}).get("gff")} - - return { - "gene_family": family, - "target_species": sorted(groups["target"].keys()), - "ref_species": sorted(groups["ref"].keys()), - "target_family_protein": target_family_protein, - "ref_family_protein": ref_family_protein, - "target_all_peps": target_all_peps, - "ref_all_peps": ref_all_peps, - "target_gffs": target_gffs, - "ref_gffs": ref_gffs, - "groups": groups, - } - - -def merge_many_fastas(files: List[Path], output_file: Path) -> Path: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w") as out: - for fasta in files: - with open(fasta, "r") as f: - out.write(f.read().rstrip() + "\n") - return output_file - - -def _parse_gff_attr(attr: str) -> Dict[str, str]: - result = {} - for item in attr.strip().strip(';').split(';'): - item = item.strip() - if not item: - continue - if '=' in item: - k, v = item.split('=', 1) - elif ' ' in item: - k, v = item.split(' ', 1) - else: - continue - result[k.strip()] = v.strip().strip('"') - return result - - -def _match_gff_id(attrs: Dict[str, str], protein_ids: set) -> Optional[str]: - candidates = [attrs.get("ID"), attrs.get("transcript_id"), attrs.get("Name"), attrs.get("Parent")] - for candidate in candidates: - if not candidate: - continue - for item in str(candidate).split(','): - norm = normalize_seq_id(item) - if norm in protein_ids: - return norm - for variant in id_variants(norm): - if variant in protein_ids: - return variant - return None - - -def build_species_position_files(species: str, gff3: Path, protein_fasta: Path, work_dir: Path) -> Dict[str, Any]: - """Create MCScanX simplified GFF and JCVI BED for one species.""" - simple_gff = work_dir / f"{species}.gff" - bed_file = work_dir / f"{species}.bed" - protein_ids = fasta_id_set(protein_fasta) - seen = set() - count = 0 - features_seen = 0 - work_dir.mkdir(parents=True, exist_ok=True) - with open(gff3, "r") as f_in, open(simple_gff, "w") as gff_out, open(bed_file, "w") as bed_out: - for line in f_in: - if not line.strip() or line.startswith('#'): - continue - parts = line.rstrip('\n').split(' ') - if len(parts) < 9 or parts[2] not in {"mRNA", "transcript", "gene"}: - continue - features_seen += 1 - attrs = _parse_gff_attr(parts[8]) - gene_id = _match_gff_id(attrs, protein_ids) - if not gene_id or gene_id in seen: - continue - seen.add(gene_id) - chrom = parts[0] - start, end = parts[3], parts[4] - gff_out.write(f"{chrom} {gene_id} {start} {end}\n") - bed_out.write(f"{chrom} {start} {end} {gene_id}\n") - count += 1 - return { - "success": count > 0, - "species": species, - "gff": simple_gff, - "bed": bed_file, - "gene_count": count, - "features_seen": features_seen, - "protein_id_count": len(protein_ids), - "error": "No GFF IDs matched protein FASTA IDs" if count == 0 else "", - } - - -def collect_family_ids(target_family_fasta: Path, ref_family_fasta: Path, final_id_file: Optional[Path] = None) -> set: - ids = set() - for fasta in [target_family_fasta, ref_family_fasta]: - if fasta and Path(fasta).exists(): - ids.update(fasta_id_set(Path(fasta))) - if final_id_file and final_id_file.exists(): - with open(final_id_file, "r") as f: - for line in f: - if line.strip() and not line.startswith('#'): - ids.update(id_variants(line.strip().split()[0])) - return ids - - -def add_artifact(artifacts: Dict[str, Dict], key: str, file_path: Path, rel_path: str, artifact: str, file_type: str) -> None: - if not file_path or not file_path.exists(): - return - info = get_file_info(file_path) - artifacts[key] = { - "artifact": artifact, - "path": rel_path, - "full_path": str(file_path.absolute()), - "type": file_type, - "size": info["size"], - "sha256": info["sha256"], - "mtime": info["mtime"], - } - - -def write_failure_manifest(workspace: Path, job_id: str, run_id: str, prefix: str, start_time: datetime, - workspace_root: Path, input_dir: Path, cmd: List[str], error: str) -> Path: - end_time = datetime.now() - return write_manifest( - workspace, job_id, run_id, prefix, start_time, end_time, - artifacts={}, metrics={"error": error}, parameters={"prefix": prefix, "error": error}, - tool_versions={}, workspace_root=workspace_root, input_dir=input_dir, cmd=cmd, status="failed" - ) - - -def main(): - parser = argparse.ArgumentParser(description="????????") - parser.add_argument("--prefix", required=True, help="?????? Multi.SPL") - parser.add_argument("--workspace", required=True) - parser.add_argument("--input-dir") - parser.add_argument("--ref-species", default=None, help="???????? config.xlsx ??") - parser.add_argument("--target-species", default=None, help="???????? config.xlsx ??") - parser.add_argument("--ref-protein", help="?????????") - parser.add_argument("--target-protein", help="?????????") - parser.add_argument("--cpu", type=int, default=8) - parser.add_argument("--log-file", help="??????????????") - parser.add_argument("--workspace-root", help="??????????manifest?") - args = parser.parse_args() - - start_time = datetime.now() - run_id = f"interspecies_synteny_{start_time.strftime('%Y%m%d_%H%M%S')}" - workspace = Path(args.workspace) - job_id = workspace.name - input_dir = Path(args.input_dir) if args.input_dir else workspace.parent.parent / "input" / job_id - workspace_root = Path(args.workspace_root) if args.workspace_root else workspace.parent - cmd = sys.argv - - print("=" * 60) - print("Interspecies synteny analysis") - print("=" * 60) - - try: - paths = prepare_directories(workspace) - tool_versions = get_tool_versions() - discovery = discover_group_inputs(workspace, input_dir, args.prefix, args) - target_group = "Target_combined" - ref_group = "Ref_combined" - print(f"Target species: {', '.join(discovery['target_species']) or 'NONE'}") - print(f"Reference species: {', '.join(discovery['ref_species']) or 'NONE'}") - print(f"Target family protein: {discovery['target_family_protein']}") - print(f"Ref family protein: {discovery['ref_family_protein']}") - - if not discovery["target_family_protein"] or not Path(discovery["target_family_protein"]).exists(): - raise RuntimeError(f"Target family protein missing: {discovery['target_family_protein']}") - if not discovery["ref_family_protein"] or not Path(discovery["ref_family_protein"]).exists(): - raise RuntimeError("Reference family protein missing; expected Ref_combined..pep.fasta or Ref species pep files") - - artifacts = {} - metrics = { - "target_species": discovery["target_species"], - "ref_species": discovery["ref_species"], - "tree_success": False, - "synteny_success": False, - "synteny_skipped_reason": "", - } - - print("\nMerging family proteins for tree...") - merged_fasta = paths["analysis_dir"] / f"{args.prefix}.target_ref.family.pep.fa" - merge_result = merge_protein_sequences(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), merged_fasta) - if not merge_result.get("success"): - raise RuntimeError(f"Failed to merge family proteins: {merge_result.get('stderr')}") - metrics["target_family_count"] = merge_result.get("target_count") - metrics["ref_family_count"] = merge_result.get("ref_count") - - print("\nBuilding interspecies tree...") - tree_result = build_phylogenetic_tree(merged_fasta, paths["analysis_dir"], args.prefix, args.cpu) - if not tree_result.get("tree_file"): - raise RuntimeError(f"Tree construction failed: {tree_result.get('error', 'unknown error')}") - final_tree = paths["final_dir"] / f"{args.prefix}.interspecies_tree.nwk" - shutil.copy(Path(tree_result["tree_file"]), final_tree) - metrics["tree_success"] = True - add_artifact(artifacts, "tree_nwk", final_tree, f"final/{args.prefix}.interspecies_tree.nwk", "tree_nwk", "tree") - - gene_species_map = paths["analysis_dir"] / f"{args.prefix}.gene_to_species.tsv" - write_gene_species_map(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), discovery["target_species"], discovery["ref_species"], gene_species_map, collect_species_id_sets(discovery["groups"])) - add_artifact(artifacts, "gene_to_species", gene_species_map, f"interspecies_synteny/analysis/{args.prefix}.gene_to_species.tsv", "gene_to_species", "tsv") - - print("\nPlotting interspecies tree...") - target_subfamily_file = input_dir / f"{args.prefix}.geneID.subfamily" - if not target_subfamily_file.exists(): - target_subfamily_file = workspace / "final" / f"{args.prefix}.geneID.subfamily" - ref_subfamily_file = input_dir / f"SPL_{discovery['ref_species'][0]}.subfamily" if discovery["ref_species"] else input_dir / "SPL_ref.subfamily" - if not ref_subfamily_file.exists(): - candidates = sorted(input_dir.glob("SPL_*.subfamily")) + sorted(input_dir.glob("*.SPL.subfamily")) - ref_subfamily_file = candidates[0] if candidates else ref_subfamily_file - tree_plot_result = plot_interspecies_tree( - final_tree, - paths["final_dir"], - args.prefix, - target_group, - ref_group, - target_subfamily_file if target_subfamily_file.exists() else None, - ref_subfamily_file if ref_subfamily_file.exists() else None, - gene_species_map, - ) - if tree_plot_result.get("pdf_file"): - add_artifact(artifacts, "tree_pdf", tree_plot_result["pdf_file"], f"final/{args.prefix}.interspecies_tree.pdf", "tree_pdf", "plot") - if tree_plot_result.get("png_file"): - add_artifact(artifacts, "tree_png", tree_plot_result["png_file"], f"final/{args.prefix}.interspecies_tree.png", "tree_png", "plot") - - synteny_plot_result = {"success": False} - pair_results: List[Dict[str, Any]] = [] - species_position_results: Dict[str, Dict[str, Any]] = {} - - missing = [] - if not discovery["target_all_peps"]: - missing.append("target all protein") - if not discovery["ref_all_peps"]: - missing.append("reference all protein") - if not discovery["target_gffs"]: - missing.append("target GFF") - if not discovery["ref_gffs"]: - missing.append("reference GFF") - - if not missing: - print("\nPreparing per-species GFF/BED files for pairwise MCScanX...") - all_species = discovery["target_species"] + discovery["ref_species"] - for species in all_species: - pep = discovery["target_all_peps"].get(species) or discovery["ref_all_peps"].get(species) - gff = discovery["target_gffs"].get(species) or discovery["ref_gffs"].get(species) - if not pep or not gff: - species_position_results[species] = {"success": False, "error": "missing pep or gff"} - continue - result = build_species_position_files(species, Path(gff), Path(pep), paths["analysis_dir"]) - species_position_results[species] = result - if result.get("success"): - add_artifact(artifacts, f"{species}_mcscanx_gff", result["gff"], f"interspecies_synteny/analysis/{species}.gff", "mcscanx_gff", "gff") - add_artifact(artifacts, f"{species}_bed", result["bed"], f"interspecies_synteny/analysis/{species}.bed", "jcvi_bed", "bed") - print(f" {species}: {result.get('gene_count', 0)} matched genes") - - print("\nRunning pairwise target-ref MCScanX...") - for target_species in discovery["target_species"]: - for ref_species in discovery["ref_species"]: - pair_key = f"{target_species}.{ref_species}" - pair = {"pair": pair_key, "target_species": target_species, "ref_species": ref_species, "success": False} - target_ready = species_position_results.get(target_species, {}).get("success") - ref_ready = species_position_results.get(ref_species, {}).get("success") - target_pep = discovery["target_all_peps"].get(target_species) - ref_pep = discovery["ref_all_peps"].get(ref_species) - if not (target_ready and ref_ready and target_pep and ref_pep): - pair["error"] = "missing matched GFF/BED or protein FASTA" - pair_results.append(pair) - print(f" {pair_key}: skipped ({pair['error']})") - continue - blast_result = run_reciprocal_blast(Path(target_pep), Path(ref_pep), paths["analysis_dir"], target_species, ref_species, args.cpu) - pair["blast_success"] = bool(blast_result.get("success")) - if blast_result.get("output_file"): - pair["blast_file"] = str(blast_result["output_file"]) - add_artifact(artifacts, f"{pair_key}_blast", blast_result["output_file"], f"interspecies_synteny/analysis/{pair_key}.blast", "blast", "data") - if not blast_result.get("success"): - pair["error"] = "BLAST failed or produced no hits" - pair_results.append(pair) - print(f" {pair_key}: {pair['error']}") - continue - mcscanx_result = run_mcscanx_interspecies(paths["analysis_dir"], target_species, ref_species) - pair_gff = paths["analysis_dir"] / f"{pair_key}.gff" - if pair_gff.exists(): - pair["pair_gff"] = str(pair_gff) - add_artifact(artifacts, f"{pair_key}_gff", pair_gff, f"interspecies_synteny/analysis/{pair_key}.gff", "pair_mcscanx_gff", "gff") - pair["mcscanx_success"] = bool(mcscanx_result.get("success")) - if mcscanx_result.get("collinearity_file"): - pair["collinearity_file"] = str(mcscanx_result["collinearity_file"]) - add_artifact(artifacts, f"{pair_key}_collinearity", mcscanx_result["collinearity_file"], f"interspecies_synteny/analysis/{pair_key}.collinearity", "collinearity", "data") - if not mcscanx_result.get("success"): - pair["error"] = "MCScanX did not produce collinearity" - pair_results.append(pair) - print(f" {pair_key}: {pair['error']}") - continue - simple_result = convert_collinearity_to_simple(paths["analysis_dir"], Path(mcscanx_result["collinearity_file"]), target_species, ref_species) - if not simple_result.get("success") and blast_result.get("output_file"): - fallback_result = convert_blast_to_family_simple(paths["analysis_dir"], Path(blast_result["output_file"]), target_species, ref_species) - if fallback_result.get("success"): - fallback_result["note"] = "MCScanX produced no whole-genome blocks; using family-only BLAST links for JCVI visualization" - simple_result = fallback_result - pair.update({k: str(v) if isinstance(v, Path) else v for k, v in simple_result.items()}) - pair["success"] = bool(simple_result.get("success")) - if simple_result.get("anchors_file"): - add_artifact(artifacts, f"{pair_key}_anchors", simple_result["anchors_file"], f"interspecies_synteny/analysis/{pair_key}.anchors", "anchors", "data") - if simple_result.get("simple_file"): - add_artifact(artifacts, f"{pair_key}_anchors_simple", simple_result["simple_file"], f"interspecies_synteny/analysis/{pair_key}.anchors.simple", "anchors_simple", "data") - pair_results.append(pair) - print(f" {pair_key}: {'success' if pair['success'] else 'anchors conversion failed'}") - - family_ids = collect_family_ids(Path(discovery["target_family_protein"]), Path(discovery["ref_family_protein"]), workspace / "final" / f"{args.prefix}.final_members.id") - synteny_plot_result = plot_multi_species_synteny( - paths["analysis_dir"], paths["final_dir"], args.prefix, - discovery["target_species"] + discovery["ref_species"], pair_results, family_ids, - ) - successful_pairs = [p for p in pair_results if p.get("success")] - metrics["synteny_success"] = bool(synteny_plot_result.get("success")) and len(successful_pairs) == len(pair_results) and bool(pair_results) - if not successful_pairs: - metrics["synteny_skipped_reason"] = synteny_plot_result.get("error") or "No target-ref pair produced JCVI anchors" - else: - metrics["synteny_skipped_reason"] = "Missing full data for MCScanX: " + ", ".join(missing) - print(metrics["synteny_skipped_reason"]) - - metrics["species_position_results"] = { - sp: {k: str(v) if isinstance(v, Path) else v for k, v in result.items()} - for sp, result in species_position_results.items() - } - metrics["pairwise_synteny"] = pair_results - metrics["pair_success_count"] = sum(1 for p in pair_results if p.get("success")) - metrics["pair_total_count"] = len(pair_results) - - if synteny_plot_result.get("pdf_file"): - add_artifact(artifacts, "synteny_pdf", synteny_plot_result["pdf_file"], f"final/{args.prefix}.interspecies_synteny.pdf", "synteny_pdf", "plot") - if synteny_plot_result.get("png_file"): - add_artifact(artifacts, "synteny_png", synteny_plot_result["png_file"], f"final/{args.prefix}.interspecies_synteny.png", "synteny_png", "plot") - if synteny_plot_result.get("marked_pdf_file"): - add_artifact(artifacts, "synteny_marked_pdf", synteny_plot_result["marked_pdf_file"], f"final/{args.prefix}.interspecies_synteny.marked.pdf", "synteny_marked_pdf", "plot") - if synteny_plot_result.get("marked_png_file"): - add_artifact(artifacts, "synteny_marked_png", synteny_plot_result["marked_png_file"], f"final/{args.prefix}.interspecies_synteny.marked.png", "synteny_marked_png", "plot") - if synteny_plot_result.get("seqids_file"): - add_artifact(artifacts, "jcvi_seqids", synteny_plot_result["seqids_file"], f"interspecies_synteny/analysis/{args.prefix}.seqids", "jcvi_seqids", "data") - if synteny_plot_result.get("layout_file"): - add_artifact(artifacts, "jcvi_layout", synteny_plot_result["layout_file"], f"interspecies_synteny/analysis/{args.prefix}.layout", "jcvi_layout", "data") - - end_time = datetime.now() - status = "success" if metrics["tree_success"] and metrics["synteny_success"] else "partial_success" - parameters = { - "prefix": args.prefix, - "gene_family": discovery["gene_family"], - "target_group": target_group, - "ref_group": ref_group, - "target_species": discovery["target_species"], - "ref_species": discovery["ref_species"], - "target_protein_spl": str(discovery["target_family_protein"]), - "ref_protein_spl": str(discovery["ref_family_protein"]), - "input_dir": str(input_dir), - "cpu": args.cpu, - } - report_path = write_report(workspace, args.prefix, metrics, parameters, tool_versions, start_time, end_time, (end_time - start_time).total_seconds(), cmd) - add_artifact(artifacts, "report", report_path, f"final/{args.prefix}.interspecies_synteny_report.md", "report", "report") - manifest_path = write_manifest(workspace, job_id, run_id, args.prefix, start_time, end_time, artifacts, metrics, parameters, tool_versions, workspace_root, input_dir, cmd, status=status) - add_artifact(artifacts, "manifest", manifest_path, f"final/{args.prefix}.interspecies_synteny.manifest.json", "manifest", "json") - write_manifest(workspace, job_id, run_id, args.prefix, start_time, end_time, artifacts, metrics, parameters, tool_versions, workspace_root, input_dir, cmd, status=status) - print(f"Interspecies synteny finished with status: {status}") - print(f"Manifest: {manifest_path}") - return 0 if metrics["tree_success"] else 1 - except Exception as exc: - print(f"Interspecies synteny failed: {exc}") - try: - write_failure_manifest(workspace, job_id, run_id, args.prefix, start_time, workspace_root, input_dir, cmd, str(exc)) - except Exception as manifest_exc: - print(f"Failed to write failure manifest: {manifest_exc}") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bioflow_kernel/cli/run_rnaseq_qc.py.bak.20260430_050459 b/bioflow_kernel/cli/run_rnaseq_qc.py.bak.20260430_050459 deleted file mode 100755 index 831cf4d..0000000 --- a/bioflow_kernel/cli/run_rnaseq_qc.py.bak.20260430_050459 +++ /dev/null @@ -1,617 +0,0 @@ -#!/home/xia-lab/miniconda3/envs/test4py311/bin/python -""" -RNA-Seq QC阶段：对原始测序数据进行质量控制 - -输入： - - 原始FASTQ文件（双端） - - 样本配置文件（group_sample.yaml） - -输出： - - FastQC报告 - - Cutadapt处理后的clean reads - - 质控报告和manifest -""" - -import argparse -import json -import subprocess -import sys -import time -import yaml -from datetime import datetime -from pathlib import Path -from typing import Any, Dict, List, Optional -import hashlib -import os - - -# 添加conda环境bin目录到PATHCONDA_BIN = "/home/xia-lab/miniconda3/envs/test4py311/bin"if CONDA_BIN not in os.environ.get("PATH", ""): os.environ["PATH"] = CONDA_BIN + ":" + os.environ.get("PATH", "") -def validate_input_files(input_dir: Path, config: Dict[str, Any]) -> Dict[str, Dict[str, Path]]: - """ - 严格验证输入文件：检查R1+R2配对、文件存在、文件大小>0 - - Returns: - 验证通过的样本字典 {sample_name: {"r1": Path, "r2": Path}} - - Raises: - SystemExit: 验证失败时退出并返回错误信息 - """ - samples = config.get("samples", {}) - if not samples: - print("❌ ERROR: No samples defined in configuration file", file=sys.stderr) - sys.exit(1) - - validated_samples = {} - errors = [] - - for sample_name, paths in samples.items(): - # 解析路径 - if isinstance(paths, list): - if len(paths) < 2: - errors.append(f"Sample '{sample_name}': insufficient paths (need 2, got {len(paths)})") - continue - r1_path, r2_path = paths[0], paths[1] - elif isinstance(paths, dict): - r1_path = paths.get("r1") or paths.get("read1") - r2_path = paths.get("r2") or paths.get("read2") - if not r1_path or not r2_path: - errors.append(f"Sample '{sample_name}': missing r1/r2 paths") - continue - else: - errors.append(f"Sample '{sample_name}': unknown format {type(paths)}") - continue - - # 解析完整路径 - r1 = Path(r1_path) if Path(r1_path).is_absolute() else input_dir / r1_path - r2 = Path(r2_path) if Path(r2_path).is_absolute() else input_dir / r2_path - - # 验证文件存在 - if not r1.exists(): - errors.append(f"Sample '{sample_name}': R1 file not found: {r1}") - continue - if not r2.exists(): - errors.append(f"Sample '{sample_name}': R2 file not found: {r2}") - continue - - # 验证文件大小>0 - r1_size = r1.stat().st_size - r2_size = r2.stat().st_size - if r1_size == 0: - errors.append(f"Sample '{sample_name}': R1 file is empty: {r1}") - continue - if r2_size == 0: - errors.append(f"Sample '{sample_name}': R2 file is empty: {r2}") - continue - - # 验证R1和R2大小合理（差距不超过10倍） - if r1_size > 0 and r2_size > 0: - ratio = max(r1_size, r2_size) / min(r1_size, r2_size) - if ratio > 10: - errors.append(f"Sample '{sample_name}': R1/R2 file size mismatch (ratio: {ratio:.1f})") - continue - - validated_samples[sample_name] = {"r1": r1, "r2": r2, "r1_size": r1_size, "r2_size": r2_size} - - # 报告错误并退出 - if errors: - print(f"\n❌ ERROR: Input validation failed for {len(errors)} sample(s):\n", file=sys.stderr) - for err in errors: - print(f" - {err}", file=sys.stderr) - print(f"\nTotal samples in config: {len(samples)}", file=sys.stderr) - print(f"Valid samples: {len(validated_samples)}", file=sys.stderr) - sys.exit(1) - - print(f"✅ Input validation passed: {len(validated_samples)} sample(s) ready") - for name, info in validated_samples.items(): - print(f" - {name}: R1={info['r1_size']/1024/1024:.1f}MB, R2={info['r2_size']/1024/1024:.1f}MB") - - return validated_samples - - -def compute_file_hash(filepath: Path) -> str: - """计算文件的SHA256哈希值""" - sha256_hash = hashlib.sha256() - try: - with open(filepath, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - except Exception: - return "" - - -def get_tool_versions() -> Dict[str, str]: - """获取所需工具版本信息""" - versions = {} - - # fastqc - try: - result = subprocess.run( - [f"{RNASEQ_BIN}/fastqc", "--version"], - capture_output=True, - text=True, - check=False, - timeout=5 - ) - output = result.stdout.strip() or result.stderr.strip() - versions["fastqc"] = output.split('\n')[0] if output else "unknown" - except Exception: - versions["fastqc"] = "not_found" - - # cutadapt - try: - result = subprocess.run( - [f"{RNASEQ_BIN}/cutadapt", "--version"], - capture_output=True, - text=True, - check=False, - timeout=5 - ) - output = result.stdout.strip() or result.stderr.strip() - versions["cutadapt"] = output.split('\n')[0] if output else "unknown" - except Exception: - versions["cutadapt"] = "not_found" - - return versions - - -def load_samples_config(config_file: Path) -> Dict[str, Any]: - """加载样本配置文件（支持合并格式：samples + groups + compare）""" - with open(config_file, 'r') as f: - config = yaml.safe_load(f) - - # 确保有 samples 字段 - if "samples" not in config: - raise ValueError(f"Config file {config_file} must contain 'samples' field") - - return config - - -RNASEQ_BIN = "/home/xia-lab/miniconda3/envs/rnaseq/bin" - -def run_fastqc(fastq_files: List[Path], output_dir: Path, cpu: int = 4) -> Dict[str, Any]: - """ - 运行FastQC质控 - - Returns: - Dict with success status and timing info - """ - results = {"success": True, "files": {}, "total_time_sec": 0} - fastqc_path = f"{RNASEQ_BIN}/fastqc" - - for fq in fastq_files: - cmd = [fastqc_path, "-o", str(output_dir), "-t", str(cpu), str(fq)] - file_size_mb = fq.stat().st_size / 1024 / 1024 - - print(f" [FastQC] Starting: {fq.name} ({file_size_mb:.1f} MB)...", flush=True) - start_ts = time.time() - - try: - result = subprocess.run(cmd, check=True, capture_output=True, text=True) - elapsed = time.time() - start_ts - results["files"][fq.name] = {"success": True, "time_sec": elapsed} - results["total_time_sec"] += elapsed - print(f" [FastQC] Completed: {fq.name} in {elapsed:.1f}s", flush=True) - except subprocess.CalledProcessError as e: - elapsed = time.time() - start_ts - results["files"][fq.name] = {"success": False, "time_sec": elapsed, "error": str(e)} - results["success"] = False - print(f" [FastQC] FAILED: {fq.name} after {elapsed:.1f}s: {e.stderr}", file=sys.stderr) - raise RuntimeError(f"FastQC failed for {fq.name}: {e.stderr}") - - return results - - -def run_cutadapt( - r1: Path, - r2: Path, - out_r1: Path, - out_r2: Path, - log_file: Path, - adapter_r1: str = "AGATCGGAAGAGC", - adapter_r2: str = "AGATCGGAAGAGC", - quality_cutoff: int = 20, - min_length: int = 50 -) -> Dict[str, Any]: - """ - 运行Cutadapt去除接头 - - Returns: - Dict with success status, timing info, and stats - """ - cmd = [ - f"{RNASEQ_BIN}/cutadapt", - "-a", adapter_r1, - "-A", adapter_r2, - "-o", str(out_r1), - "-p", str(out_r2), - "-q", str(quality_cutoff), - "-m", str(min_length), - str(r1), - str(r2) - ] - - r1_size_mb = r1.stat().st_size / 1024 / 1024 - r2_size_mb = r2.stat().st_size / 1024 / 1024 - - print(f" [Cutadapt] Starting: {r1.name} + {r2.name} ({r1_size_mb+r2_size_mb:.1f} MB total)...", flush=True) - start_ts = time.time() - - try: - result = subprocess.run(cmd, check=True, capture_output=True, text=True) - elapsed = time.time() - start_ts - - with open(log_file, 'w') as f: - f.write(result.stdout) - - # 验证输出文件 - if not out_r1.exists() or out_r1.stat().st_size == 0: - raise RuntimeError(f"Cutadapt output R1 is empty or missing: {out_r1}") - if not out_r2.exists() or out_r2.stat().st_size == 0: - raise RuntimeError(f"Cutadapt output R2 is empty or missing: {out_r2}") - - out_size_mb = (out_r1.stat().st_size + out_r2.stat().st_size) / 1024 / 1024 - print(f" [Cutadapt] Completed in {elapsed:.1f}s (output: {out_size_mb:.1f} MB)", flush=True) - - return {"success": True, "time_sec": elapsed, "log": result.stdout} - - except subprocess.CalledProcessError as e: - elapsed = time.time() - start_ts - with open(log_file, 'w') as f: - f.write(e.stdout + "\n" + e.stderr) - print(f" [Cutadapt] FAILED after {elapsed:.1f}s: {e.stderr}", file=sys.stderr) - raise RuntimeError(f"Cutadapt failed: {e.stderr}") - - except Exception as e: - elapsed = time.time() - start_ts - print(f" [Cutadapt] FAILED after {elapsed:.1f}s: {str(e)}", file=sys.stderr) - raise - - -def get_sample_stats(sample_dir: Path, sample_name: str) -> Dict[str, Any]: - """获取样本统计信息""" - import re - stats = { - "raw_reads": 0, - "clean_reads": 0, - "adapter_removed": 0, - "quality_filtered": 0 - } - - # 读取FastQC数据（从zip文件中提取） - fastqc_zip = sample_dir / f"{sample_name}_1_fastqc.zip" - if fastqc_zip.exists(): - try: - import zipfile - with zipfile.ZipFile(fastqc_zip, 'r') as zf: - # 找到 fastqc_data.txt 文件 - data_file = None - for name in zf.namelist(): - if name.endswith('fastqc_data.txt'): - data_file = name - break - if data_file: - with zf.open(data_file) as f: - for line in f: - line = line.decode('utf-8') - if line.startswith("Total Sequences"): - stats["raw_reads"] = int(line.split("\t")[1]) - elif line.startswith("Sequence length"): - stats["read_length"] = line.split("\t")[1].strip() - except Exception as e: - print(f"Warning: Could not read FastQC data for {sample_name}: {e}") - - # 读取Cutadapt日志 - log_file = sample_dir / f"{sample_name}_cutadapt.log" - if log_file.exists(): - with open(log_file, 'r') as f: - for line in f: - if "Total reads processed:" in line: - match = re.search(r'([\d,]+)', line) - if match: - stats["raw_reads"] = int(match.group(1).replace(',', '')) - elif "Reads with adapters:" in line: - match = re.search(r'([\d,]+)', line) - if match: - stats["adapter_removed"] = int(match.group(1).replace(',', '')) - elif "Reads written (passing filters):" in line: - match = re.search(r'([\d,]+)', line) - if match: - stats["clean_reads"] = int(match.group(1).replace(',', '')) - - return stats - - -def generate_report( - output_dir: Path, - sample_stats: Dict[str, Dict[str, Any]], - versions: Dict[str, str], - start_time: datetime, - end_time: datetime, - prefix: str = "" -) -> Path: - """生成质控报告""" - report_file = output_dir / f"{prefix}.rnaseq_qc_report.md" if prefix else output_dir / "rnaseq_qc_report.md" - - elapsed = (end_time - start_time).total_seconds() - - content = f"""# RNA-Seq Quality Control Report - -## Summary - -- **Analysis Date**: {end_time.strftime("%Y-%m-%d %H:%M:%S")} -- **Total Samples**: {len(sample_stats)} -- **Total Runtime**: {elapsed:.1f} seconds - -## Tool Versions - -""" - for tool, version in versions.items(): - content += f"- **{tool}**: {version}\n" - - content += "\n## Sample Statistics\n\n" - content += "| Sample | Raw Reads | Clean Reads | Adapter Removed |\n" - content += "|--------|-----------|-------------|-----------------|\n" - - for sample, stats in sample_stats.items(): - content += f"| {sample} | {stats.get('raw_reads', 'N/A')} | {stats.get('clean_reads', 'N/A')} | {stats.get('adapter_removed', 'N/A')} |\n" - - with open(report_file, 'w') as f: - f.write(content) - - return report_file - - -def generate_manifest( - output_dir: Path, - sample_stats: Dict[str, Dict[str, Any]], - versions: Dict[str, str], - start_time: datetime, - end_time: datetime, - cmd: List[str], - input_dir: Path, - workspace_dir: Path, - job_id: str, - run_id: str, - prefix: str = "", - analysis_dir: Path = None # analysis 目录路径 -) -> Path: - """生成manifest文件""" - manifest_file = output_dir / f"{prefix}.rnaseq_qc.manifest.json" if prefix else output_dir / "rnaseq_qc.manifest.json" - - total_raw = sum(s.get('raw_reads', 0) for s in sample_stats.values()) - total_clean = sum(s.get('clean_reads', 0) for s in sample_stats.values()) - - manifest = { - "manifest_version": "1.0", - "stage": "rnaseq_qc", - "status": "success", - "job_id": job_id, - "run_id": run_id, - "timestamp": end_time.isoformat(), - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - "elapsed_sec": (end_time - start_time).total_seconds(), - "cmd": cmd, - "workspace_root": str(output_dir.parent.parent), - "workspace": str(workspace_dir), - "analysis_dir": str(analysis_dir) if analysis_dir else str(workspace_dir / "analysis"), - "input_dir": str(input_dir), - "metrics": { - "total_samples": len(sample_stats), - "total_raw_reads": total_raw, - "total_clean_reads": total_clean, - "samples": sample_stats - }, - "parameters": { - "cpu": 4, - "fastqc_params": "-t 4", - "cutadapt_params": "-q 20 -m 50" - }, - "artifacts": {}, - "tools": versions - } - - # 添加artifacts（文件生成在 final/ 目录，使用 prefix 命名） - # 参考 synteny 契约，包含 full_path、size、sha256 等字段 - report_name = f"{prefix}.rnaseq_qc_report.md" if prefix else "rnaseq_qc_report.md" - manifest_name = f"{prefix}.rnaseq_qc.manifest.json" if prefix else "rnaseq_qc.manifest.json" - - def add_artifact(name: str, filepath: Path, artifact_type: str, **extra): - """添加 artifact 到 manifest""" - if filepath.exists(): - manifest["artifacts"][name] = { - "artifact": name, - "path": f"final/{filepath.name}", - "full_path": str(filepath.absolute()), - "type": artifact_type, - "size": filepath.stat().st_size, - "sha256": compute_file_hash(filepath), - **extra - } - - # analysis_dir 用于引用质控报告文件（保留在 analysis 目录） - analysis_path = analysis_dir if analysis_dir else workspace_dir / "analysis" - - # final_dir 是 output_dir 的父目录下的 final 目录 - final_dir = output_dir.parent / "final" - - for sample in sample_stats.keys(): - # FastQC HTML 报告（复制到 final，前端展示） - html_r1 = final_dir / f"{prefix}.rnaseq_qc.{sample}_1_fastqc.html" - html_r2 = final_dir / f"{prefix}.rnaseq_qc.{sample}_2_fastqc.html" - add_artifact(f"qc_report_{sample}_r1", html_r1, "html", format="html") - add_artifact(f"qc_report_{sample}_r2", html_r2, "html", format="html") - - # FastQC 数据（zip，保留在 analysis 目录） - zip_r1 = analysis_path / f"{sample}_1_fastqc.zip" - zip_r2 = analysis_path / f"{sample}_2_fastqc.zip" - if zip_r1.exists(): - manifest["artifacts"][f"qc_data_{sample}_r1"] = { - "artifact": f"qc_data_{sample}_r1", - "path": f"rnaseq_qc/analysis/{zip_r1.name}", - "full_path": str(zip_r1.absolute()), - "type": "zip", - "size": zip_r1.stat().st_size, - "sha256": compute_file_hash(zip_r1), - "format": "zip" - } - if zip_r2.exists(): - manifest["artifacts"][f"qc_data_{sample}_r2"] = { - "artifact": f"qc_data_{sample}_r2", - "path": f"rnaseq_qc/analysis/{zip_r2.name}", - "full_path": str(zip_r2.absolute()), - "type": "zip", - "size": zip_r2.stat().st_size, - "sha256": compute_file_hash(zip_r2), - "format": "zip" - } - - # Clean reads（FASTQ）- 大文件 - clean_r1 = output_dir / f"{sample}_1.clean.fastq.gz" - clean_r2 = output_dir / f"{sample}_2.clean.fastq.gz" - add_artifact(f"clean_reads_r1_{sample}", clean_r1, "fastq", format="fastq.gz") - add_artifact(f"clean_reads_r2_{sample}", clean_r2, "fastq", format="fastq.gz") - - # Cutadapt 日志 - log_file = output_dir / f"{sample}_cutadapt.log" - add_artifact(f"cutadapt_log_{sample}", log_file, "log", format="txt") - - # 报告和 manifest - report_file = output_dir / report_name - add_artifact("report", report_file, "markdown", format="md") - - # manifest 自身（在生成后添加） - manifest["artifacts"]["manifest"] = { - "artifact": "manifest", - "path": f"final/{manifest_name}", - "type": "json", - "format": "json" - } - - with open(manifest_file, 'w') as f: - json.dump(manifest, f, indent=2) - - return manifest_file - - -def main(): - parser = argparse.ArgumentParser(description="RNA-Seq Quality Control") - parser.add_argument("-i", "--input", required=True, help="Input directory with FASTQ files") - parser.add_argument("-c", "--config", required=True, help="Sample configuration file (YAML)") - parser.add_argument("-o", "--output", required=True, help="Output directory") - parser.add_argument("-j", "--job-id", default="unknown", help="Job ID") - parser.add_argument("-r", "--run-id", default="unknown", help="Run ID") - parser.add_argument("-t", "--cpu", type=int, default=4, help="Number of CPU threads") - parser.add_argument("-p", "--prefix", default="", help="Output file prefix") - parser.add_argument("--adapter-r1", default="AGATCGGAAGAGC", help="Adapter sequence for read1") - parser.add_argument("--adapter-r2", default="AGATCGGAAGAGC", help="Adapter sequence for read2") - - args = parser.parse_args() - - start_time = datetime.now() - input_dir = Path(args.input) - output_dir = Path(args.output) # rnaseq_qc 工作目录 - config_file = Path(args.config) - - # 创建工作目录 - output_dir.mkdir(parents=True, exist_ok=True) - - # 最终结果目录: workspace//final/ - # 根据契约，最终产物必须位于 workspace//final/ - workspace_dir = output_dir.parent # job_id 目录 - final_dir = workspace_dir / "final" - final_dir.mkdir(exist_ok=True) - - # 中间文件目录: rnaseq_qc/analysis/ - analysis_dir = output_dir / "analysis" - analysis_dir.mkdir(exist_ok=True) - - # 加载样本配置并严格验证输入文件 - config = load_samples_config(config_file) - - # 严格验证输入文件（失败时直接退出） - validated_samples = validate_input_files(input_dir, config) - - print(f"\nProcessing {len(validated_samples)} samples...") - - # 获取工具版本 - versions = get_tool_versions() - print(f"Tool versions: {versions}") - - sample_stats = {} - total_samples = len(validated_samples) - current_sample = 0 - - # 处理每个样本 - for sample_name, file_info in validated_samples.items(): - current_sample += 1 - r1 = file_info["r1"] - r2 = file_info["r2"] - - print(f"\n[PROGRESS {current_sample}/{total_samples}] Processing: {sample_name}", flush=True) - print(f" Input: {r1.name} + {r2.name}", flush=True) - - # 运行FastQC - 输出到 analysis 目录 - try: - fastqc_result = run_fastqc([r1, r2], analysis_dir, args.cpu) - except RuntimeError as e: - print(f"\n❌ ERROR: FastQC failed for {sample_name}: {e}", file=sys.stderr) - sys.exit(1) - - # 运行Cutadapt - 输出到 analysis 目录 - out_r1 = analysis_dir / f"{sample_name}_1.clean.fastq.gz" - out_r2 = analysis_dir / f"{sample_name}_2.clean.fastq.gz" - log_file = analysis_dir / f"{sample_name}_cutadapt.log" - - try: - cutadapt_result = run_cutadapt(r1, r2, out_r1, out_r2, log_file, - args.adapter_r1, args.adapter_r2) - except RuntimeError as e: - print(f"\n❌ ERROR: Cutadapt failed for {sample_name}: {e}", file=sys.stderr) - sys.exit(1) - - # 获取统计信息（从 analysis 目录读取） - stats = get_sample_stats(analysis_dir, sample_name) - stats["fastqc_time_sec"] = fastqc_result.get("total_time_sec", 0) - stats["cutadapt_time_sec"] = cutadapt_result.get("time_sec", 0) - sample_stats[sample_name] = stats - - print(f" ✓ Sample {sample_name} completed", flush=True) - - # 复制结果到 final 目录，使用标准命名格式 - import shutil - safe_prefix = args.prefix if args.prefix else "rnaseq" - - print(f"\n[PROGRESS] Copying results to final directory...") - copied_files = 0 - expected_files_per_sample = 5 # 2 clean FASTQ + 2 FastQC HTML + 1 log - - for sample_name, file_info in validated_samples.items(): - sample_copied = 0 - r1_name = file_info["r1"].stem.replace('.fastq', '').replace('.fq', '') # 原始R1文件名（不含扩展名） - r2_name = file_info["r2"].stem.replace('.fastq', '').replace('.fq', '') # 原始R2文件名（不含扩展名） - - # 复制 clean reads 到 final（下游分析需要） - for suffix in ["_1.clean.fastq.gz", "_2.clean.fastq.gz"]: - src = analysis_dir / f"{sample_name}{suffix}" - dst = final_dir / f"{safe_prefix}.rnaseq_qc.{sample_name}{suffix}" - if src.exists() and src.stat().st_size > 0: - shutil.copy2(src, dst) - copied_files += 1 - sample_copied += 1 - else: - print(f" ⚠ Missing/empty: {src.name}", file=sys.stderr) - - # 复制 FastQC HTML 报告到 final（前端展示需要） - # FastQC使用原始FASTQ文件名生成报告 - fastqc_patterns = [ - (f"{r1_name}_fastqc.html", f"{sample_name}_1_fastqc.html"), - (f"{r2_name}_fastqc.html", f"{sample_name}_2_fastqc.html") - ] - for src_pattern, dst_suffix in fastqc_patterns: - src = analysis_dir / src - - -if __name__ == "__main__": - main() diff --git a/bioflow_kernel/cli/run_rnaseq_qc.py.bak_20260813_225803 b/bioflow_kernel/cli/run_rnaseq_qc.py.bak_20260813_225803 deleted file mode 100755 index 823508f..0000000 --- a/bioflow_kernel/cli/run_rnaseq_qc.py.bak_20260813_225803 +++ /dev/null @@ -1,1072 +0,0 @@ -#!/usr/bin/env python3 -""" -RNA-Seq QC阶段：对原始测序数据进行质量控制 - -输入： - - 原始FASTQ文件（双端） - - 样本配置文件（group_sample.yaml） - -输出： - - FastQC报告 - - Cutadapt处理后的clean reads - - 质控报告和manifest -""" - -import argparse -import json -import subprocess -import sys -import time -import yaml -import shutil -from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime -from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence -import hashlib -import os - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from gfa_core.rnaseq_context import apply_rnaseq_context, build_rnaseq_context, resolve_rnaseq_project_context -from apps.cli.rnaseq_env import prepend_path, rnaseq_bin - - -RNASEQ_BIN = str(rnaseq_bin()) -prepend_path(RNASEQ_BIN) - -def validate_input_files(input_dir: Path, config: Dict[str, Any]) -> Dict[str, Dict[str, Path]]: - """ - 严格验证输入文件：检查R1+R2配对、文件存在、文件大小>0 - - Returns: - 验证通过的样本字典 {sample_name: {"r1": Path, "r2": Path}} - - Raises: - SystemExit: 验证失败时退出并返回错误信息 - """ - samples = config.get("samples", {}) - if not samples: - print("❌ ERROR: No samples defined in configuration file", file=sys.stderr) - sys.exit(1) - - validated_samples = {} - errors = [] - - for sample_name, paths in samples.items(): - # 解析路径 - if isinstance(paths, list): - if len(paths) < 2: - errors.append(f"Sample '{sample_name}': insufficient paths (need 2, got {len(paths)})") - continue - r1_path, r2_path = paths[0], paths[1] - elif isinstance(paths, dict): - r1_path = paths.get("r1") or paths.get("read1") - r2_path = paths.get("r2") or paths.get("read2") - if not r1_path or not r2_path: - errors.append(f"Sample '{sample_name}': missing r1/r2 paths") - continue - else: - errors.append(f"Sample '{sample_name}': unknown format {type(paths)}") - continue - - # 解析完整路径 - r1 = Path(r1_path) if Path(r1_path).is_absolute() else input_dir / r1_path - r2 = Path(r2_path) if Path(r2_path).is_absolute() else input_dir / r2_path - - # 验证文件存在 - if not r1.exists(): - errors.append(f"Sample '{sample_name}': R1 file not found: {r1}") - continue - if not r2.exists(): - errors.append(f"Sample '{sample_name}': R2 file not found: {r2}") - continue - - # 验证文件大小>0 - r1_size = r1.stat().st_size - r2_size = r2.stat().st_size - if r1_size == 0: - errors.append(f"Sample '{sample_name}': R1 file is empty: {r1}") - continue - if r2_size == 0: - errors.append(f"Sample '{sample_name}': R2 file is empty: {r2}") - continue - - # 验证R1和R2大小合理（差距不超过10倍） - if r1_size > 0 and r2_size > 0: - ratio = max(r1_size, r2_size) / min(r1_size, r2_size) - if ratio > 10: - errors.append(f"Sample '{sample_name}': R1/R2 file size mismatch (ratio: {ratio:.1f})") - continue - - validated_samples[sample_name] = {"r1": r1, "r2": r2, "r1_size": r1_size, "r2_size": r2_size} - - # 报告错误并退出 - if errors: - print(f"\n❌ ERROR: Input validation failed for {len(errors)} sample(s):\n", file=sys.stderr) - for err in errors: - print(f" - {err}", file=sys.stderr) - print(f"\nTotal samples in config: {len(samples)}", file=sys.stderr) - print(f"Valid samples: {len(validated_samples)}", file=sys.stderr) - sys.exit(1) - - print(f"✅ Input validation passed: {len(validated_samples)} sample(s) ready") - for name, info in validated_samples.items(): - print(f" - {name}: R1={info['r1_size']/1024/1024:.1f}MB, R2={info['r2_size']/1024/1024:.1f}MB") - - return validated_samples - - -def compute_file_hash(filepath: Path) -> str: - """计算文件的SHA256哈希值""" - sha256_hash = hashlib.sha256() - try: - with open(filepath, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - except Exception: - return "" - - -def fastp_version(fastp_bin: Optional[Path] = None) -> str: - if not fastp_bin: - return "not_configured" - try: - result = subprocess.run( - [str(fastp_bin), "--version"], - capture_output=True, - text=True, - check=False, - timeout=5, - ) - output = result.stdout.strip() or result.stderr.strip() - return output.split("\n")[0] if output else "unknown" - except Exception: - return "not_found" - - -def resolve_fastp_bin(fastp_bin_arg: str = "") -> Optional[Path]: - candidates = [] - if fastp_bin_arg: - candidates.append(Path(fastp_bin_arg)) - env_fastp = os.environ.get("FASTP_BIN", "").strip() - if env_fastp: - candidates.append(Path(env_fastp)) - path_fastp = shutil.which("fastp") - if path_fastp: - candidates.append(Path(path_fastp)) - for candidate in candidates: - if candidate.exists() and candidate.is_file() and os.access(candidate, os.X_OK): - return candidate - return None - - -def get_tool_versions(fastp_bin: Optional[Path] = None) -> Dict[str, str]: - """获取所需工具版本信息""" - versions = {} - - # fastqc - try: - result = subprocess.run( - [f"{RNASEQ_BIN}/fastqc", "--version"], - capture_output=True, - text=True, - check=False, - timeout=5 - ) - output = result.stdout.strip() or result.stderr.strip() - versions["fastqc"] = output.split('\n')[0] if output else "unknown" - except Exception: - versions["fastqc"] = "not_found" - - # cutadapt - try: - result = subprocess.run( - [f"{RNASEQ_BIN}/cutadapt", "--version"], - capture_output=True, - text=True, - check=False, - timeout=5 - ) - output = result.stdout.strip() or result.stderr.strip() - versions["cutadapt"] = output.split('\n')[0] if output else "unknown" - except Exception: - versions["cutadapt"] = "not_found" - - # MultiQC - try: - result = subprocess.run( - [f"{RNASEQ_BIN}/multiqc", "--version"], - capture_output=True, - text=True, - check=False, - timeout=10 - ) - output = result.stdout.strip() or result.stderr.strip() - versions["multiqc"] = output.split('\n')[0] if output else "unknown" - except Exception: - versions["multiqc"] = "not_found" - - versions["fastp"] = fastp_version(fastp_bin) - - return versions - - -def load_samples_config(config_file: Path) -> Dict[str, Any]: - """加载样本配置文件（支持合并格式：samples + groups + compare）""" - with open(config_file, 'r') as f: - config = yaml.safe_load(f) - - # 确保有 samples 字段 - if "samples" not in config: - raise ValueError(f"Config file {config_file} must contain 'samples' field") - - return config - - -RNASEQ_BIN = str(rnaseq_bin()) -prepend_path(RNASEQ_BIN) - - -def resolve_thread_plan(cpu: int, sample_parallel_jobs: int = 1, threads_per_sample: Optional[int] = None) -> Dict[str, int]: - """Resolve conservative RNA-seq sample/thread settings from CLI args.""" - total_cpu = max(1, int(cpu or 1)) - jobs = max(1, int(sample_parallel_jobs or 1)) - if threads_per_sample is None or int(threads_per_sample) < 1: - threads = max(1, total_cpu // jobs) - else: - threads = max(1, int(threads_per_sample)) - if jobs * threads > total_cpu: - threads = max(1, total_cpu // jobs) - return {"cpu": total_cpu, "sample_parallel_jobs": jobs, "threads_per_sample": threads} - - - -def fastqc_output_base(fastq_path: Path) -> str: - """Return FastQC output basename without _fastqc suffix for .fastq/.fq(.gz).""" - name = fastq_path.name - for suffix in (".fastq.gz", ".fq.gz", ".fastq", ".fq"): - if name.lower().endswith(suffix): - return name[:-len(suffix)] - return fastq_path.stem - - -def sample_qc_outputs_complete(sample_name: str, file_info: Dict[str, Path], analysis_dir: Path, qc_tool: str = "fastp") -> bool: - r1 = file_info["r1"] - r2 = file_info["r2"] - if qc_tool == "fastp": - required = [ - analysis_dir / f"{sample_name}_1.clean.fastq.gz", - analysis_dir / f"{sample_name}_2.clean.fastq.gz", - analysis_dir / f"{sample_name}_fastp.html", - analysis_dir / f"{sample_name}_fastp.json", - analysis_dir / f"{sample_name}_fastp.log", - ] - else: - required = [ - analysis_dir / f"{fastqc_output_base(r1)}_fastqc.html", - analysis_dir / f"{fastqc_output_base(r1)}_fastqc.zip", - analysis_dir / f"{fastqc_output_base(r2)}_fastqc.html", - analysis_dir / f"{fastqc_output_base(r2)}_fastqc.zip", - analysis_dir / f"{sample_name}_1.clean.fastq.gz", - analysis_dir / f"{sample_name}_2.clean.fastq.gz", - analysis_dir / f"{sample_name}_cutadapt.log", - ] - return all(path.exists() and path.stat().st_size > 0 for path in required) - -def run_fastqc(fastq_files: List[Path], output_dir: Path, cpu: int = 4) -> Dict[str, Any]: - """ - 运行FastQC质控 - - Returns: - Dict with success status and timing info - """ - results = {"success": True, "files": {}, "total_time_sec": 0} - fastqc_path = f"{RNASEQ_BIN}/fastqc" - - for fq in fastq_files: - cmd = [fastqc_path, "-o", str(output_dir), "-t", str(cpu), str(fq)] - file_size_mb = fq.stat().st_size / 1024 / 1024 - - print(f" [FastQC] Starting: {fq.name} ({file_size_mb:.1f} MB)...", flush=True) - start_ts = time.time() - - try: - result = subprocess.run(cmd, check=True, capture_output=True, text=True) - elapsed = time.time() - start_ts - results["files"][fq.name] = {"success": True, "time_sec": elapsed} - results["total_time_sec"] += elapsed - print(f" [FastQC] Completed: {fq.name} in {elapsed:.1f}s", flush=True) - except subprocess.CalledProcessError as e: - elapsed = time.time() - start_ts - results["files"][fq.name] = {"success": False, "time_sec": elapsed, "error": str(e)} - results["success"] = False - print(f" [FastQC] FAILED: {fq.name} after {elapsed:.1f}s: {e.stderr}", file=sys.stderr) - raise RuntimeError(f"FastQC failed for {fq.name}: {e.stderr}") - - return results - - -def run_cutadapt( - r1: Path, - r2: Path, - out_r1: Path, - out_r2: Path, - log_file: Path, - adapter_r1: str = "AGATCGGAAGAGC", - adapter_r2: str = "AGATCGGAAGAGC", - quality_cutoff: int = 20, - min_length: int = 50, - cpu: int = 1 -) -> Dict[str, Any]: - """ - 运行Cutadapt去除接头 - - Returns: - Dict with success status, timing info, and stats - """ - cmd = [ - f"{RNASEQ_BIN}/cutadapt", - "-a", adapter_r1, - "-A", adapter_r2, - "-o", str(out_r1), - "-p", str(out_r2), - "-q", str(quality_cutoff), - "-m", str(min_length), - "-j", str(max(1, int(cpu or 1))), - str(r1), - str(r2) - ] - - r1_size_mb = r1.stat().st_size / 1024 / 1024 - r2_size_mb = r2.stat().st_size / 1024 / 1024 - - print(f" [Cutadapt] Starting: {r1.name} + {r2.name} ({r1_size_mb+r2_size_mb:.1f} MB total)...", flush=True) - start_ts = time.time() - - try: - result = subprocess.run(cmd, check=True, capture_output=True, text=True) - elapsed = time.time() - start_ts - - with open(log_file, 'w') as f: - f.write(result.stdout) - - # 验证输出文件 - if not out_r1.exists() or out_r1.stat().st_size == 0: - raise RuntimeError(f"Cutadapt output R1 is empty or missing: {out_r1}") - if not out_r2.exists() or out_r2.stat().st_size == 0: - raise RuntimeError(f"Cutadapt output R2 is empty or missing: {out_r2}") - - out_size_mb = (out_r1.stat().st_size + out_r2.stat().st_size) / 1024 / 1024 - print(f" [Cutadapt] Completed in {elapsed:.1f}s (output: {out_size_mb:.1f} MB)", flush=True) - - return {"success": True, "time_sec": elapsed, "log": result.stdout} - - except subprocess.CalledProcessError as e: - elapsed = time.time() - start_ts - with open(log_file, 'w') as f: - f.write(e.stdout + "\n" + e.stderr) - print(f" [Cutadapt] FAILED after {elapsed:.1f}s: {e.stderr}", file=sys.stderr) - raise RuntimeError(f"Cutadapt failed: {e.stderr}") - - except Exception as e: - elapsed = time.time() - start_ts - print(f" [Cutadapt] FAILED after {elapsed:.1f}s: {str(e)}", file=sys.stderr) - raise - - -def run_fastp( - fastp_bin: Path, - r1: Path, - r2: Path, - out_r1: Path, - out_r2: Path, - html_report: Path, - json_report: Path, - log_file: Path, - cpu: int = 1, - detect_adapter_for_pe: bool = True, - cut_front: bool = True, - cut_tail: bool = True, - cut_window_size: int = 4, - cut_mean_quality: int = 20, - qualified_quality_phred: int = 20, - unqualified_percent_limit: int = 30, - n_base_limit: int = 5, - length_required: int = 30, -) -> Dict[str, Any]: - cmd = [ - str(fastp_bin), - "-i", str(r1), - "-I", str(r2), - "-o", str(out_r1), - "-O", str(out_r2), - "--html", str(html_report), - "--json", str(json_report), - "--thread", str(max(1, int(cpu or 1))), - "--cut_window_size", str(cut_window_size), - "--cut_mean_quality", str(cut_mean_quality), - "--qualified_quality_phred", str(qualified_quality_phred), - "--unqualified_percent_limit", str(unqualified_percent_limit), - "--n_base_limit", str(n_base_limit), - "--length_required", str(length_required), - ] - if detect_adapter_for_pe: - cmd.append("--detect_adapter_for_pe") - if cut_front: - cmd.append("--cut_front") - if cut_tail: - cmd.append("--cut_tail") - - print(f" [fastp] Starting: {r1.name} + {r2.name}...", flush=True) - start_ts = time.time() - result = subprocess.run(cmd, capture_output=True, text=True, check=False) - elapsed = time.time() - start_ts - log_file.write_text( - "Command: " + " ".join(cmd) + "\n\n" - + "--- STDOUT ---\n" - + result.stdout - + "\n--- STDERR ---\n" - + result.stderr, - encoding="utf-8", - ) - if result.returncode != 0: - print(f" [fastp] FAILED after {elapsed:.1f}s: {result.stderr}", file=sys.stderr) - raise RuntimeError(f"fastp failed with exit code {result.returncode}: {result.stderr}") - for path in (out_r1, out_r2, html_report, json_report): - if not path.exists() or path.stat().st_size == 0: - raise RuntimeError(f"fastp output is missing or empty: {path}") - print(f" [fastp] Completed in {elapsed:.1f}s", flush=True) - return {"success": True, "time_sec": elapsed, "cmd": cmd, "log": str(log_file)} - - -def run_multiqc(search_paths: Sequence[Path], output_dir: Path, report_name: str) -> Optional[Path]: - """Run MultiQC over selected QC outputs and return the report path.""" - multiqc = Path(f"{RNASEQ_BIN}/multiqc") - existing = [str(path) for path in search_paths if path.exists()] - if not multiqc.exists() or not existing: - return None - cmd = [str(multiqc), *existing, "-o", str(output_dir), "-n", report_name, "-f"] - try: - subprocess.run(cmd, check=True, capture_output=True, text=True) - report = output_dir / report_name - return report if report.exists() else None - except subprocess.CalledProcessError as e: - print(f"MultiQC failed: {e.stderr}", file=sys.stderr) - return None - - -def get_sample_stats(sample_dir: Path, sample_name: str) -> Dict[str, Any]: - """获取样本统计信息""" - import re - - def first_count_after_colon(text: str) -> Optional[int]: - value = text.split(":", 1)[1] if ":" in text else text - match = re.search(r'([\d,]+)', value) - if match: - return int(match.group(1).replace(',', '')) - return None - - stats = { - "raw_reads": 0, - "clean_reads": 0, - "adapter_removed": 0, - "quality_filtered": 0 - } - fastp_json = sample_dir / f"{sample_name}_fastp.json" - if fastp_json.exists(): - try: - payload = json.loads(fastp_json.read_text(encoding="utf-8")) - before = payload.get("summary", {}).get("before_filtering", {}) - after = payload.get("summary", {}).get("after_filtering", {}) - stats["raw_reads"] = int(before.get("total_reads", 0)) - stats["clean_reads"] = int(after.get("total_reads", 0)) - stats["fastp_q20_rate_before"] = before.get("q20_rate") - stats["fastp_q30_rate_before"] = before.get("q30_rate") - stats["fastp_q20_rate_after"] = after.get("q20_rate") - stats["fastp_q30_rate_after"] = after.get("q30_rate") - stats["read1_mean_length"] = after.get("read1_mean_length") - stats["read2_mean_length"] = after.get("read2_mean_length") - stats["gc_content_after"] = after.get("gc_content") - except Exception as e: - print(f"Warning: Could not read fastp JSON for {sample_name}: {e}") - - # 读取FastQC数据（从zip文件中提取） - fastqc_zip = sample_dir / f"{sample_name}_1_fastqc.zip" - if fastqc_zip.exists(): - try: - import zipfile - with zipfile.ZipFile(fastqc_zip, 'r') as zf: - # 找到 fastqc_data.txt 文件 - data_file = None - for name in zf.namelist(): - if name.endswith('fastqc_data.txt'): - data_file = name - break - if data_file: - with zf.open(data_file) as f: - for line in f: - line = line.decode('utf-8') - if line.startswith("Total Sequences"): - stats["raw_reads"] = int(line.split("\t")[1]) - elif line.startswith("Sequence length"): - stats["read_length"] = line.split("\t")[1].strip() - except Exception as e: - print(f"Warning: Could not read FastQC data for {sample_name}: {e}") - - # 读取Cutadapt日志 - log_file = sample_dir / f"{sample_name}_cutadapt.log" - if log_file.exists(): - with open(log_file, 'r') as f: - for line in f: - if "Total reads processed:" in line or "Total read pairs processed:" in line: - count = first_count_after_colon(line) - if count is not None: - stats["raw_reads"] = count - elif "Reads with adapters:" in line or "Read 1 with adapter:" in line or "Read 2 with adapter:" in line: - count = first_count_after_colon(line) - if count is not None: - stats["adapter_removed"] += count - elif "Pairs that were too short:" in line or "Reads that were too short:" in line: - count = first_count_after_colon(line) - if count is not None: - stats["quality_filtered"] += count - elif "Reads written (passing filters):" in line or "Pairs written (passing filters):" in line: - count = first_count_after_colon(line) - if count is not None: - stats["clean_reads"] = count - - return stats - - -def generate_report( - output_dir: Path, - sample_stats: Dict[str, Dict[str, Any]], - versions: Dict[str, str], - start_time: datetime, - end_time: datetime, - prefix: str = "", - qc_tool: str = "fastp", -) -> Path: - """生成质控报告""" - report_file = output_dir / f"{prefix}.rnaseq_qc_report.md" if prefix else output_dir / "rnaseq_qc_report.md" - - elapsed = (end_time - start_time).total_seconds() - - content = f"""# RNA-Seq Quality Control Report - -## Summary - -- **Analysis Date**: {end_time.strftime("%Y-%m-%d %H:%M:%S")} -- **Total Samples**: {len(sample_stats)} -- **Total Runtime**: {elapsed:.1f} seconds -- **QC Tool**: {qc_tool} - -## Tool Versions - -""" - for tool, version in versions.items(): - content += f"- **{tool}**: {version}\n" - - content += "\n## Sample Statistics\n\n" - content += "| Sample | Raw Reads | Clean Reads | Adapter Removed |\n" - content += "|--------|-----------|-------------|-----------------|\n" - - for sample, stats in sample_stats.items(): - content += f"| {sample} | {stats.get('raw_reads', 'N/A')} | {stats.get('clean_reads', 'N/A')} | {stats.get('adapter_removed', 'N/A')} |\n" - - with open(report_file, 'w') as f: - f.write(content) - - return report_file - - -def generate_manifest( - output_dir: Path, - sample_stats: Dict[str, Dict[str, Any]], - versions: Dict[str, str], - start_time: datetime, - end_time: datetime, - cmd: List[str], - input_dir: Path, - workspace_dir: Path, - job_id: str, - run_id: str, - prefix: str = "", - analysis_dir: Path = None, # analysis 目录路径 - cpu: int = 4, - sample_parallel_jobs: int = 1, - threads_per_sample: int = 4, - sample_config: Optional[Path] = None, - qc_tool: str = "fastp", - fastp_bin: str = "", - fastp_params: Optional[Dict[str, Any]] = None, -) -> Path: - """生成manifest文件""" - manifest_file = output_dir / f"{prefix}.rnaseq_qc.manifest.json" if prefix else output_dir / "rnaseq_qc.manifest.json" - - total_raw = sum(s.get('raw_reads', 0) for s in sample_stats.values()) - total_clean = sum(s.get('clean_reads', 0) for s in sample_stats.values()) - - manifest = { - "manifest_version": "1.0", - "stage": "rnaseq_qc", - "status": "success", - "prefix": prefix, - "job_id": job_id, - "run_id": run_id, - "timestamp": end_time.isoformat(), - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - "elapsed_sec": (end_time - start_time).total_seconds(), - "cmd": cmd, - "workspace_root": str(output_dir.parent.parent), - "workspace": str(workspace_dir), - "analysis_dir": str(analysis_dir) if analysis_dir else str(workspace_dir / "analysis"), - "input_dir": str(input_dir), - "metrics": { - "total_samples": len(sample_stats), - "total_raw_reads": total_raw, - "total_clean_reads": total_clean, - "samples": sample_stats - }, - "parameters": { - "cpu": cpu, - "sample_parallel_jobs": sample_parallel_jobs, - "threads_per_sample": threads_per_sample, - "qc_tool": qc_tool, - "fastqc_params": f"-t {threads_per_sample}", - "cutadapt_params": f"-q 20 -m 50 -j {threads_per_sample}" if qc_tool == "cutadapt" else "", - "fastp_bin": fastp_bin, - "fastp_params": fastp_params or {}, - }, - "artifacts": {}, - "tools": versions - } - context = build_rnaseq_context( - prefix=prefix, - sample_config=sample_config, - foreground_mode="global", - stage="rnaseq_qc", - ) - apply_rnaseq_context(manifest, context) - - # 添加artifacts（文件生成在 final/ 目录，使用 prefix 命名） - # 参考 synteny 契约，包含 full_path、size、sha256 等字段 - report_name = f"{prefix}.rnaseq_qc_report.md" if prefix else "rnaseq_qc_report.md" - manifest_name = f"{prefix}.rnaseq_qc.manifest.json" if prefix else "rnaseq_qc.manifest.json" - - def add_artifact(name: str, filepath: Path, artifact_type: str, **extra): - """添加 artifact 到 manifest""" - if filepath.exists(): - manifest["artifacts"][name] = { - "artifact": name, - "path": f"final/{filepath.name}", - "full_path": str(filepath.absolute()), - "type": artifact_type, - "size": filepath.stat().st_size, - "sha256": compute_file_hash(filepath), - **extra - } - - # analysis_dir 用于引用质控报告文件（保留在 analysis 目录） - analysis_path = analysis_dir if analysis_dir else workspace_dir / "analysis" - - # final_dir 是 output_dir 的父目录下的 final 目录 - final_dir = output_dir.parent / "final" - - artifact_prefix = prefix if prefix else "rnaseq" - for sample in sample_stats.keys(): - # FastQC HTML 报告（复制到 final，前端展示） - html_r1 = final_dir / f"{artifact_prefix}.rnaseq_qc.{sample}_1_fastqc.html" - html_r2 = final_dir / f"{artifact_prefix}.rnaseq_qc.{sample}_2_fastqc.html" - add_artifact(f"qc_report_{sample}_r1", html_r1, "html", format="html") - add_artifact(f"qc_report_{sample}_r2", html_r2, "html", format="html") - - # FastQC 数据（zip，保留在 analysis 目录） - zip_r1 = analysis_path / f"{sample}_1_fastqc.zip" - zip_r2 = analysis_path / f"{sample}_2_fastqc.zip" - if zip_r1.exists(): - manifest["artifacts"][f"qc_data_{sample}_r1"] = { - "artifact": f"qc_data_{sample}_r1", - "path": f"rnaseq_qc/analysis/{zip_r1.name}", - "full_path": str(zip_r1.absolute()), - "type": "zip", - "size": zip_r1.stat().st_size, - "sha256": compute_file_hash(zip_r1), - "format": "zip" - } - if zip_r2.exists(): - manifest["artifacts"][f"qc_data_{sample}_r2"] = { - "artifact": f"qc_data_{sample}_r2", - "path": f"rnaseq_qc/analysis/{zip_r2.name}", - "full_path": str(zip_r2.absolute()), - "type": "zip", - "size": zip_r2.stat().st_size, - "sha256": compute_file_hash(zip_r2), - "format": "zip" - } - - # Clean reads（FASTQ）- 大文件，复制到 final 后使用标准前缀命名 - clean_r1 = output_dir / f"{artifact_prefix}.rnaseq_qc.{sample}_1.clean.fastq.gz" - clean_r2 = output_dir / f"{artifact_prefix}.rnaseq_qc.{sample}_2.clean.fastq.gz" - add_artifact(f"clean_reads_r1_{sample}", clean_r1, "fastq", format="fastq.gz") - add_artifact(f"clean_reads_r2_{sample}", clean_r2, "fastq", format="fastq.gz") - - # Cutadapt 日志 - log_file = output_dir / f"{artifact_prefix}.rnaseq_qc.{sample}_cutadapt.log" - add_artifact(f"cutadapt_log_{sample}", log_file, "log", format="txt") - - fastp_html = output_dir / f"{artifact_prefix}.rnaseq_qc.{sample}_fastp.html" - fastp_json = output_dir / f"{artifact_prefix}.rnaseq_qc.{sample}_fastp.json" - fastp_log = output_dir / f"{artifact_prefix}.rnaseq_qc.{sample}_fastp.log" - add_artifact(f"fastp_report_{sample}", fastp_html, "html", format="html") - add_artifact(f"fastp_json_{sample}", fastp_json, "json", format="json") - add_artifact(f"fastp_log_{sample}", fastp_log, "log", format="txt") - - # 报告、MultiQC 和 manifest - report_file = output_dir / report_name - add_artifact("report", report_file, "markdown", format="md") - multiqc_name = f"{prefix}.rnaseq_qc_multiqc_report.html" if prefix else "rnaseq_qc_multiqc_report.html" - multiqc_file = output_dir / multiqc_name - add_artifact("multiqc_report", multiqc_file, "html", format="html") - multiqc_data = output_dir / multiqc_name.replace(".html", "_data") - if multiqc_data.exists(): - manifest["artifacts"]["multiqc_data"] = { - "artifact": "multiqc_data", - "path": f"final/{multiqc_data.name}", - "full_path": str(multiqc_data.absolute()), - "type": "directory", - "format": "multiqc_data" - } - - # manifest 自身（在生成后添加） - manifest["artifacts"]["manifest"] = { - "artifact": "manifest", - "path": f"final/{manifest_name}", - "type": "json", - "format": "json" - } - - with open(manifest_file, 'w') as f: - json.dump(manifest, f, indent=2) - - return manifest_file - - -def main(): - parser = argparse.ArgumentParser(description="RNA-Seq Quality Control") - parser.add_argument("-i", "--input", required=True, help="Input directory with FASTQ files") - parser.add_argument("-c", "--config", required=True, help="Sample configuration file (YAML)") - parser.add_argument("-o", "--output", required=True, help="Output directory") - parser.add_argument("-j", "--job-id", default="unknown", help="Job ID") - parser.add_argument("-r", "--run-id", default="unknown", help="Run ID") - parser.add_argument("-t", "--cpu", type=int, default=4, help="Total CPU threads for this stage") - parser.add_argument("--sample-parallel-jobs", type=int, default=1, help="Number of samples to process concurrently") - parser.add_argument("--threads-per-sample", type=int, help="Threads used by each per-sample tool invocation") - parser.add_argument("--resume-existing", action="store_true", help="Skip samples whose FastQC, clean reads, and cutadapt outputs already exist in the output directory") - parser.add_argument("-p", "--prefix", default="", help="Output file prefix") - parser.add_argument("--adapter-r1", default="AGATCGGAAGAGC", help="Adapter sequence for read1") - parser.add_argument("--adapter-r2", default="AGATCGGAAGAGC", help="Adapter sequence for read2") - parser.add_argument("--qc-tool", choices=["cutadapt", "fastp"], default="fastp", help="QC/cleaning backend. Default uses fastp; cutadapt remains available by explicit selection.") - parser.add_argument("--fastp-bin", default="tools/fastp", help="Path to fastp executable. If omitted, FASTP_BIN or PATH is used.") - parser.add_argument("--detect-adapter-for-pe", action=argparse.BooleanOptionalAction, default=True, help="Enable fastp paired-end adapter auto-detection.") - parser.add_argument("--cut-front", action=argparse.BooleanOptionalAction, default=True, help="Enable fastp front quality cutting.") - parser.add_argument("--cut-tail", action=argparse.BooleanOptionalAction, default=True, help="Enable fastp tail quality cutting.") - parser.add_argument("--cut-window-size", type=int, default=4, help="fastp quality cutting window size.") - parser.add_argument("--cut-mean-quality", type=int, default=20, help="fastp mean quality threshold for cutting.") - parser.add_argument("--qualified-quality-phred", type=int, default=20, help="fastp qualified quality phred threshold.") - parser.add_argument("--unqualified-percent-limit", type=int, default=30, help="fastp maximum unqualified base percent.") - parser.add_argument("--n-base-limit", type=int, default=5, help="fastp maximum N bases per read.") - parser.add_argument("--length-required", type=int, default=30, help="fastp minimum output read length.") - - args = parser.parse_args() - thread_plan = resolve_thread_plan(args.cpu, args.sample_parallel_jobs, args.threads_per_sample) - threads_per_sample = thread_plan["threads_per_sample"] - - start_time = datetime.now() - input_dir = Path(args.input) - output_dir = Path(args.output) # rnaseq_qc 工作目录 - config_file = Path(args.config) - context_for_prefix = resolve_rnaseq_project_context( - input_dir=input_dir, - requested_prefix=args.prefix, - sample_config=config_file, - ) - resolved_prefix = context_for_prefix.get("output_prefix") - if resolved_prefix and resolved_prefix != args.prefix: - print( - f"[INFO] Using RNA-seq output prefix {resolved_prefix} " - f"instead of requested {args.prefix or ''} " - f"(source={context_for_prefix.get('source', '')})" - ) - args.prefix = resolved_prefix - - # 创建工作目录 - output_dir.mkdir(parents=True, exist_ok=True) - - # 最终结果目录: workspace//final/ - # 根据契约，最终产物必须位于 workspace//final/ - workspace_dir = output_dir.parent # job_id 目录 - final_dir = workspace_dir / "final" - final_dir.mkdir(exist_ok=True) - - # 中间文件目录: rnaseq_qc/analysis/ - analysis_dir = output_dir / "analysis" - analysis_dir.mkdir(exist_ok=True) - - # 加载样本配置并严格验证输入文件 - config = load_samples_config(config_file) - - # 严格验证输入文件（失败时直接退出） - validated_samples = validate_input_files(input_dir, config) - - print(f"\nProcessing {len(validated_samples)} samples...") - - fastp_bin = resolve_fastp_bin(args.fastp_bin) if args.qc_tool == "fastp" else None - if args.qc_tool == "fastp" and not fastp_bin: - print("❌ ERROR: --qc-tool fastp requires an executable --fastp-bin, FASTP_BIN, or fastp on PATH", file=sys.stderr) - sys.exit(1) - - fastp_params = { - "detect_adapter_for_pe": bool(args.detect_adapter_for_pe), - "cut_front": bool(args.cut_front), - "cut_tail": bool(args.cut_tail), - "cut_window_size": args.cut_window_size, - "cut_mean_quality": args.cut_mean_quality, - "qualified_quality_phred": args.qualified_quality_phred, - "unqualified_percent_limit": args.unqualified_percent_limit, - "n_base_limit": args.n_base_limit, - "length_required": args.length_required, - } - - # 获取工具版本 - versions = get_tool_versions(fastp_bin) - print(f"Tool versions: {versions}") - - sample_stats = {} - total_samples = len(validated_samples) - - def qc_one_sample(sample_name: str, file_info: Dict[str, Path]) -> tuple[str, Optional[Dict[str, Any]], Optional[str]]: - r1 = file_info["r1"] - r2 = file_info["r2"] - print(f"\n[PROGRESS] Processing: {sample_name}", flush=True) - print(f" Input: {r1.name} + {r2.name}", flush=True) - if args.resume_existing and sample_qc_outputs_complete(sample_name, file_info, analysis_dir, args.qc_tool): - stats = get_sample_stats(analysis_dir, sample_name) - stats["resumed_from_existing"] = True - stats["fastqc_time_sec"] = 0 - stats["cutadapt_time_sec"] = 0 - stats["fastp_time_sec"] = 0 - print(f" [RESUME] Skipping completed sample {sample_name}; existing QC outputs are complete", flush=True) - return sample_name, stats, None - - out_r1 = analysis_dir / f"{sample_name}_1.clean.fastq.gz" - out_r2 = analysis_dir / f"{sample_name}_2.clean.fastq.gz" - if args.qc_tool == "fastp": - try: - fastp_result = run_fastp( - fastp_bin=fastp_bin, - r1=r1, - r2=r2, - out_r1=out_r1, - out_r2=out_r2, - html_report=analysis_dir / f"{sample_name}_fastp.html", - json_report=analysis_dir / f"{sample_name}_fastp.json", - log_file=analysis_dir / f"{sample_name}_fastp.log", - cpu=threads_per_sample, - **fastp_params, - ) - except RuntimeError as e: - return sample_name, None, f"fastp failed for {sample_name}: {e}" - stats = get_sample_stats(analysis_dir, sample_name) - stats["fastqc_time_sec"] = 0 - stats["cutadapt_time_sec"] = 0 - stats["fastp_time_sec"] = fastp_result.get("time_sec", 0) - else: - try: - fastqc_result = run_fastqc([r1, r2], analysis_dir, threads_per_sample) - except RuntimeError as e: - return sample_name, None, f"FastQC failed for {sample_name}: {e}" - log_file = analysis_dir / f"{sample_name}_cutadapt.log" - try: - cutadapt_result = run_cutadapt( - r1, r2, out_r1, out_r2, log_file, - args.adapter_r1, args.adapter_r2, - cpu=threads_per_sample, - ) - except RuntimeError as e: - return sample_name, None, f"Cutadapt failed for {sample_name}: {e}" - stats = get_sample_stats(analysis_dir, sample_name) - stats["fastqc_time_sec"] = fastqc_result.get("total_time_sec", 0) - stats["cutadapt_time_sec"] = cutadapt_result.get("time_sec", 0) - stats["fastp_time_sec"] = 0 - - print(f" ✓ Sample {sample_name} completed", flush=True) - return sample_name, stats, None - - print( - f"Thread plan: sample_parallel_jobs={thread_plan['sample_parallel_jobs']}, " - f"threads_per_sample={threads_per_sample}, total_cpu={thread_plan['cpu']}" - ) - with ThreadPoolExecutor(max_workers=max(1, int(thread_plan["sample_parallel_jobs"]))) as executor: - future_map = { - executor.submit(qc_one_sample, sample_name, file_info): sample_name - for sample_name, file_info in validated_samples.items() - } - for future in as_completed(future_map): - sample_name = future_map[future] - try: - completed_name, stats, error = future.result() - except Exception as exc: - print(f"\n❌ ERROR: QC failed for {sample_name}: {exc}", file=sys.stderr) - sys.exit(1) - if error: - print(f"\n❌ ERROR: {error}", file=sys.stderr) - sys.exit(1) - if stats is not None: - sample_stats[completed_name] = stats - - # 复制结果到 final 目录，使用标准命名格式 - safe_prefix = args.prefix if args.prefix else "rnaseq" - - print(f"\n[PROGRESS] Copying results to final directory...") - copied_files = 0 - expected_files_per_sample = 5 # cutadapt: 2 clean + 2 FastQC HTML + 1 log; fastp: 2 clean + HTML + JSON + log - - for sample_name, file_info in validated_samples.items(): - sample_copied = 0 - r1_name = file_info["r1"].stem.replace('.fastq', '').replace('.fq', '') # 原始R1文件名（不含扩展名） - r2_name = file_info["r2"].stem.replace('.fastq', '').replace('.fq', '') # 原始R2文件名（不含扩展名） - - # 复制 clean reads 到 final（下游分析需要） - for suffix in ["_1.clean.fastq.gz", "_2.clean.fastq.gz"]: - src = analysis_dir / f"{sample_name}{suffix}" - dst = final_dir / f"{safe_prefix}.rnaseq_qc.{sample_name}{suffix}" - if src.exists() and src.stat().st_size > 0: - shutil.copy2(src, dst) - copied_files += 1 - sample_copied += 1 - else: - print(f" ⚠ Missing/empty: {src.name}", file=sys.stderr) - - if args.qc_tool == "fastp": - for src_name, dst_suffix in [ - (f"{sample_name}_fastp.html", f"{sample_name}_fastp.html"), - (f"{sample_name}_fastp.json", f"{sample_name}_fastp.json"), - (f"{sample_name}_fastp.log", f"{sample_name}_fastp.log"), - ]: - src = analysis_dir / src_name - dst = final_dir / f"{safe_prefix}.rnaseq_qc.{dst_suffix}" - if src.exists() and src.stat().st_size > 0: - shutil.copy2(src, dst) - copied_files += 1 - sample_copied += 1 - else: - print(f" ⚠ Missing/empty fastp output: {src_name}", file=sys.stderr) - else: - # 复制 FastQC HTML 报告到 final（前端展示需要） - # FastQC使用原始FASTQ文件名生成报告 - fastqc_patterns = [ - (f"{r1_name}_fastqc.html", f"{sample_name}_1_fastqc.html"), - (f"{r2_name}_fastqc.html", f"{sample_name}_2_fastqc.html") - ] - for src_pattern, dst_suffix in fastqc_patterns: - src = analysis_dir / src_pattern - dst = final_dir / f"{safe_prefix}.rnaseq_qc.{dst_suffix}" - if src.exists() and src.stat().st_size > 0: - shutil.copy2(src, dst) - copied_files += 1 - sample_copied += 1 - else: - print(f" ⚠ Missing/empty FastQC HTML: {src_pattern}", file=sys.stderr) - - # 复制 cutadapt 日志到 final（小文件，包含统计信息） - src = analysis_dir / f"{sample_name}_cutadapt.log" - dst = final_dir / f"{safe_prefix}.rnaseq_qc.{sample_name}_cutadapt.log" - if src.exists() and src.stat().st_size > 0: - shutil.copy2(src, dst) - copied_files += 1 - sample_copied += 1 - else: - print(f" ⚠ Missing/empty: {src.name}", file=sys.stderr) - - print(f" {sample_name}: {sample_copied}/{expected_files_per_sample} files copied") - - # 验证输出完整性 - total_expected = len(validated_samples) * expected_files_per_sample - if copied_files < total_expected: - print(f"\n❌ ERROR: Output validation failed", file=sys.stderr) - print(f" Expected: {total_expected} files", file=sys.stderr) - print(f" Copied: {copied_files} files", file=sys.stderr) - sys.exit(1) - - print(f"\n✅ All {copied_files} output files validated and copied") - - # 生成 MultiQC 报告 - multiqc_name = f"{args.prefix}.rnaseq_qc_multiqc_report.html" if args.prefix else "rnaseq_qc_multiqc_report.html" - multiqc_report = run_multiqc([analysis_dir, final_dir], final_dir, multiqc_name) - if multiqc_report: - print(f"MultiQC report: {multiqc_report}") - - # 生成报告 - end_time = datetime.now() - generate_report(final_dir, sample_stats, versions, start_time, end_time, args.prefix, qc_tool=args.qc_tool) - - # 生成manifest - cmd = ["python", __file__, "-i", str(input_dir), "-c", str(config_file), - "-o", str(output_dir), "-j", args.job_id, "-r", args.run_id, "-p", args.prefix, - "-t", str(thread_plan["cpu"]), - "--sample-parallel-jobs", str(thread_plan["sample_parallel_jobs"]), - "--threads-per-sample", str(thread_plan["threads_per_sample"]), - "--qc-tool", args.qc_tool] - if args.qc_tool == "fastp": - cmd.extend(["--fastp-bin", str(fastp_bin)]) - for flag, value in { - "--cut-window-size": args.cut_window_size, - "--cut-mean-quality": args.cut_mean_quality, - "--qualified-quality-phred": args.qualified_quality_phred, - "--unqualified-percent-limit": args.unqualified_percent_limit, - "--n-base-limit": args.n_base_limit, - "--length-required": args.length_required, - }.items(): - cmd.extend([flag, str(value)]) - if args.detect_adapter_for_pe: - cmd.append("--detect-adapter-for-pe") - else: - cmd.append("--no-detect-adapter-for-pe") - if args.cut_front: - cmd.append("--cut-front") - else: - cmd.append("--no-cut-front") - if args.cut_tail: - cmd.append("--cut-tail") - else: - cmd.append("--no-cut-tail") - generate_manifest( - final_dir, sample_stats, versions, start_time, end_time, - cmd, input_dir, output_dir, args.job_id, args.run_id, args.prefix, - analysis_dir=analysis_dir, # 传入 analysis 目录 - cpu=args.cpu, - sample_parallel_jobs=thread_plan["sample_parallel_jobs"], - threads_per_sample=threads_per_sample, - sample_config=config_file, - qc_tool=args.qc_tool, - fastp_bin=str(fastp_bin) if fastp_bin else "", - fastp_params=fastp_params if args.qc_tool == "fastp" else {}, - ) - - total_elapsed = (end_time - start_time).total_seconds() - print(f"\n{'='*60}") - print(f"✅ RNA-seq QC completed successfully") - print(f" Samples processed: {len(validated_samples)}") - print(f" Total runtime: {total_elapsed:.1f}s ({total_elapsed/60:.1f} min)") - print(f" Results: {output_dir}") - print(f"{'='*60}") - - -if __name__ == "__main__": - main() diff --git a/bioflow_kernel/cli/run_structure.py.bak_20260719_target_only b/bioflow_kernel/cli/run_structure.py.bak_20260719_target_only deleted file mode 100755 index aa5209d..0000000 --- a/bioflow_kernel/cli/run_structure.py.bak_20260719_target_only +++ /dev/null @@ -1,1329 +0,0 @@ -#!/usr/bin/env python3 -""" -Structure stage: 基因结构可视化 -符合 workspace_contract_v1.md 和 manifest_v1.schema.json 契约 -""" - -import os -import argparse -import json -import subprocess -import sys -import re -import uuid -import gzip -from datetime import datetime -from pathlib import Path -from typing import Dict, List, Optional, Tuple -import hashlib - -PROJECT_ROOT = Path(__file__).resolve().parents[2] -if str(PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(PROJECT_ROOT)) - -from gfa_core.gene_family_scope import read_scope, sanitize_token -from gfa_core.gene_family_stage_context import apply_stage_context, build_stage_context, user_output_prefix_for, write_legacy_manifest_copy -from gfa_core.mas.effective_members import MemberResolutionError, resolve_effective_members - - -CONDA_BIN = "/home/xzq/miniconda3/envs/test4py311/bin" -if CONDA_BIN not in os.environ.get("PATH", ""): - os.environ["PATH"] = CONDA_BIN + ":" + os.environ.get("PATH", "") - - -def display_path(path: Path | str) -> str: - """Return the user-facing symlink path instead of resolving data-root links.""" - return str(Path(path).absolute()) - - -def current_input_dir(workspace: Path) -> Path: - return workspace.parent.parent / "input" / workspace.name - - -def inside_current_job(workspace: Path, path: Path) -> bool: - input_dir = current_input_dir(workspace) - try: - resolved = path.resolve() - return resolved.is_relative_to(workspace.resolve()) or resolved.is_relative_to(input_dir.resolve()) - except OSError: - return False - - -def rebase_current_job_path(workspace: Path, path: Path) -> Optional[Path]: - """Map copied legacy workspace/input paths back onto the current job roots.""" - parts = path.parts - for root_name, current_root in (("workspace", workspace), ("input", current_input_dir(workspace))): - for idx, part in enumerate(parts): - if part != root_name: - continue - if idx + 1 >= len(parts) or parts[idx + 1] != workspace.name: - continue - suffix = Path(*parts[idx + 2:]) if idx + 2 < len(parts) else Path() - candidate = current_root / suffix - if candidate.exists(): - return candidate - return None - - -def resolve_job_path(workspace: Path, raw_path: str | Path, manifest_path: Optional[Path] = None) -> Optional[Path]: - """Resolve a copied job path without trusting stale absolute paths.""" - raw_text = str(raw_path or "").strip() - if not raw_text: - return None - path = Path(raw_text) - if path.is_absolute(): - if inside_current_job(workspace, path): - return path - rebased = rebase_current_job_path(workspace, path) - if rebased: - return rebased - return None - - candidates: List[Path] = [workspace / path, current_input_dir(workspace) / path] - if manifest_path: - candidates.extend([manifest_path.parent / path, manifest_path.parent / path.name]) - if str(path).startswith("final/") and manifest_path: - candidates.append(manifest_path.parent / path.name) - for candidate in candidates: - if candidate.exists(): - return candidate - return None - - -def manifest_artifact_path(workspace: Path, manifest_path: Path, artifact: Dict) -> Optional[Path]: - if not isinstance(artifact, dict): - return None - for key in ("path", "relpath_or_hint", "file", "full_path"): - resolved = resolve_job_path(workspace, artifact.get(key) or "", manifest_path) - if resolved: - return resolved - return None - - -def get_tool_versions() -> Dict[str, str]: - """获取工具版本信息""" - versions = {} - try: - r = subprocess.run(["R", "--version"], capture_output=True, text=True, check=False) - if r.returncode == 0: - versions["R"] = r.stdout.split("\n")[0] - except: - versions["R"] = "not_found" - - try: - import Bio - versions["biopython"] = Bio.__version__ - except: - versions["biopython"] = "not_found" - - return versions - - -def compute_file_hash(filepath: Path) -> str: - """计算文件 SHA256 哈希""" - sha256_hash = hashlib.sha256() - try: - with open(filepath, "rb") as f: - for byte_block in iter(lambda: f.read(4096), b""): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - except: - return "" - - -def read_gene_ids(id_file: Path) -> List[str]: - """读取基因 ID 列表""" - ids = [] - with open(id_file, 'r') as f: - for line in f: - line = line.strip() - if line and not line.startswith('#'): - ids.append(line) - return ids - - -def open_text_auto(path: Path): - if path.suffix == ".gz": - return gzip.open(path, "rt", encoding="utf-8", errors="ignore") - return open(path, "r", encoding="utf-8", errors="ignore") - - -def read_tree_order(tree_file: Path) -> List[str]: - """从树文件读取叶节点顺序""" - try: - from Bio import Phylo - tree = Phylo.read(tree_file, 'newick') - tips = [clade.name for clade in tree.get_terminals()] - return tips if tips else [] - except: - return [] - - - -def should_strip_version(raw_id: str) -> bool: - lowered = (raw_id or "").lower() - return not (lowered.startswith("evm.model.") or lowered.startswith("evm.tu.")) - - -def strip_gff_id(raw_id: Optional[str]) -> str: - if not raw_id: - return "" - value = str(raw_id).strip() - # Tree/Newick labels may replace ':' with '_' (mRNA:HanXR -> mRNA_HanXR). - # Strip repeated typed wrappers so GFF attributes, FASTA IDs and tree-safe - # labels all resolve to the same member accession. - while True: - updated = re.sub(r"^(gene|transcript|CDS|mRNA)[:_]", "", value, flags=re.IGNORECASE) - if updated == value: - break - value = updated - value = re.sub(r'_[PT]\d+$', '', value) - if should_strip_version(value): - value = re.sub(r'\.\d+$', '', value) - return value - - -def id_variants(raw_id: Optional[str]) -> set[str]: - value = strip_gff_id(raw_id) - if not value: - return set() - variants = {value} - if should_strip_version(value) and re.search(r'\.\d+$', value): - variants.add(re.sub(r'\.\d+$', '', value)) - variants.add(re.sub(r'_[PT]\d+$', '', value)) - expanded = set() - for item in variants: - if item: - expanded.update({item, item.upper(), item.lower()}) - expanded.update({f"gene:{item}", f"transcript:{item}", f"CDS:{item}", f"mRNA:{item}"}) - return expanded - - -def build_target_id_lookup(gene_ids: List[str]) -> Dict[str, str]: - lookup: Dict[str, str] = {} - for gid in gene_ids: - canonical = str(gid).strip() - if not canonical: - continue - for variant in id_variants(canonical): - lookup.setdefault(variant, canonical) - return lookup - - -def canonical_target_id(raw_id: Optional[str], target_lookup: Dict[str, str]) -> Optional[str]: - for variant in id_variants(raw_id): - if variant in target_lookup: - return target_lookup[variant] - return None - - -def parse_gff3_attributes(attr: str) -> Dict[str, str]: - attr_map: Dict[str, str] = {} - for item in str(attr or "").split(";"): - if "=" not in item: - continue - key, value = item.split("=", 1) - attr_map[key] = value - return attr_map - - -def canonical_from_attr_values(attr_map: Dict[str, str], keys: List[str], target_lookup: Dict[str, str]) -> Optional[str]: - for key in keys: - value = attr_map.get(key) - if not value: - continue - for token in str(value).split(","): - canonical = canonical_target_id(token, target_lookup) - if canonical: - return canonical - return None - -def resolve_gff3_path(ws: Path, prefix: str, cli_gff3: Optional[str] = None) -> Path: - """ - 解析 GFF3 文件路径 - - 命名规则: {species_short}_final.gff3 (如 Ft_final.gff3) - 优先级: final/ > input/ - - Args: - ws: workspace 目录 - prefix: 分析前缀 (如 Ft.SPL)，从中提取物种简写 Ft - cli_gff3: 命令行指定的 GFF3 路径（可选） - - Returns: - GFF3 文件路径 - - Raises: - FileNotFoundError: 找不到符合命名规范的 GFF3 文件 - """ - # 方式一：命令行指定（如果提供） - if cli_gff3: - gff3_path = resolve_job_path(ws, cli_gff3) or Path(cli_gff3) - if gff3_path.exists(): - print(f"[INFO] Using specified GFF3: {gff3_path}") - return gff3_path - else: - raise FileNotFoundError(f"Specified GFF3 not found: {cli_gff3}") - - # 从 prefix 提取物种简写 (Ft.SPL -> Ft) - species_short = prefix.split('.')[0] if '.' in prefix else prefix - - # 标准命名: Ft_final.gff3 - gff3_name = f"{species_short}_final.gff3" - - final_dir = ws / "final" - input_dir = ws / "input" - - # 也检查 bbav2/input/{job_id}/ 路径 - job_id = ws.name - alt_input_dir = ws.parent.parent / "input" / job_id - - # 优先级: final/ > workspace/input/ > bbav2/input/{job_id}/ - search_paths = [] - if final_dir.exists(): - search_paths.append(final_dir / gff3_name) - if input_dir.exists(): - search_paths.append(input_dir / gff3_name) - if alt_input_dir.exists(): - search_paths.append(alt_input_dir / gff3_name) - - # 查找 - for candidate in search_paths: - if candidate.exists(): - print(f"[INFO] Found GFF3: {candidate}") - return candidate - - # 都找不到，报错 - searched = ", ".join([str(p) for p in search_paths]) if search_paths else "no directories available" - raise FileNotFoundError( - f"Cannot find GFF3 file '{gff3_name}' for prefix '{prefix}'. " - f"Searched in: {searched}. " - f"Please ensure the file follows naming convention: {{species_short}}_final.gff3 " - f"or specify --gff3 explicitly." - ) - - -def find_prepared_gff3(ws: Path, species: str) -> Optional[Path]: - job_id = ws.name - candidates = [ - ws / "final" / f"{species}_final.gff3", - ws / "input" / f"{species}_final.gff3", - ws.parent.parent / "input" / job_id / f"{species}_final.gff3", - ws.parent.parent / "input" / job_id / species / f"{species}_final.gff3", - ] - for candidate in candidates: - if candidate.exists() and candidate.stat().st_size > 0: - return candidate.absolute() - return None - - -def resolve_scope_gff3_paths(scope_data: Dict, ws: Optional[Path] = None) -> List[Tuple[str, Path]]: - target_species = [str(item).strip() for item in scope_data.get("target_species", []) if str(item).strip()] - selected_species = [str(item).strip() for item in scope_data.get("selected_species", []) if str(item).strip()] - scope_mode = str(scope_data.get("scope_mode") or "") - species_for_structure = selected_species if scope_mode == "target_plus_reference" and selected_species else target_species - if not species_for_structure: - raise FileNotFoundError("Scope file does not define species for structure stage") - - target_set = set(target_species) - resolved: List[Tuple[str, Path]] = [] - missing: List[str] = [] - files = scope_data.get("files", {}) - for species in species_for_structure: - selected: Optional[Path] = None - if ws is not None and species in target_set: - selected = find_prepared_gff3(ws, species) - species_files = files.get(species, {}) - if selected is None: - for record in species_files.get("gff", []): - path_text = str(record.get("path") or "") - path = resolve_job_path(ws, path_text) if (ws is not None and path_text) else (Path(path_text) if path_text else None) - if path and path.exists() and path.stat().st_size > 0: - selected = path - break - if selected: - resolved.append((species, selected.absolute())) - else: - missing.append(species) - - if missing: - raise FileNotFoundError( - "Missing target GFF3 files in scope for structure stage: " + ", ".join(missing) - ) - return resolved - - -def resolve_scope_tree_file(ws: Path, prefix: str, scope_data: Optional[Dict], cli_tree: Optional[str] = None) -> Path: - final_dir = ws / "final" - if cli_tree: - tree_path = resolve_job_path(ws, cli_tree) or Path(cli_tree) - if tree_path.exists() and tree_path.stat().st_size > 0: - return tree_path.absolute() - raise FileNotFoundError(f"Specified tree file not found: {cli_tree}") - - manifest_candidates: List[Path] = [] - scope_id = str((scope_data or {}).get("scope_id") or "").strip() - if scope_id: - manifest_candidates.append(final_dir / f"{prefix}.{sanitize_token(scope_id)}.phylogeny.manifest.json") - else: - manifest_candidates.append(final_dir / f"{prefix}.phylogeny.manifest.json") - - for manifest_file in manifest_candidates: - if not manifest_file.exists() or manifest_file.stat().st_size == 0: - continue - try: - manifest = json.loads(manifest_file.read_text(encoding="utf-8")) - except Exception: - continue - tree_path = None - artifact = (manifest.get("artifacts", {}) or {}).get("tree") - if isinstance(artifact, dict): - tree_path = manifest_artifact_path(ws, manifest_file, artifact) - elif isinstance(artifact, str): - tree_path = resolve_job_path(ws, artifact, manifest_file) - if tree_path and tree_path.exists() and tree_path.stat().st_size > 0: - return tree_path.absolute() - - fallback = final_dir / f"{prefix}.tree.nwk" - if not scope_id and fallback.exists() and fallback.stat().st_size > 0: - return fallback.absolute() - raise FileNotFoundError(f"Tree file not found for prefix {prefix}. Please run phylogeny first or specify --tree.") - - -def filter_scope_gff3( - gff3_sources: List[Tuple[str, Path]], - gene_ids: List[str], - output_gff3: Path, - structure_dir: Path, - prefix: str, -) -> Tuple[bool, Dict]: - if len(gff3_sources) == 1: - return filter_gff3(gff3_sources[0][1], gene_ids, output_gff3) - - aggregate = { - "total_lines": 0, - "filtered_lines": 0, - "genes_found": 0, - "mrna_count": 0, - "cds_count": 0, - "utr_count": 0, - "per_species": {}, - } - temp_paths: List[Path] = [] - for species, gff3_file in gff3_sources: - temp_path = structure_dir / f"{prefix}.{species}.structure.filtered.gff3" - success, stats = filter_gff3(gff3_file, gene_ids, temp_path) - if not success: - return False, aggregate - temp_paths.append(temp_path) - aggregate["per_species"][species] = stats - for key in ("total_lines", "filtered_lines", "genes_found", "mrna_count", "cds_count", "utr_count"): - aggregate[key] += int(stats.get(key, 0)) - - with output_gff3.open("w", encoding="utf-8") as out: - for temp_path in temp_paths: - with temp_path.open(encoding="utf-8", errors="ignore") as handle: - for raw in handle: - out.write(raw) - return True, aggregate - - -def resolve_subfamily_file(ws: Path, prefix: str, cli_subfamily: Optional[str] = None, output_prefix: Optional[str] = None, scope_data: Optional[Dict] = None) -> Optional[Path]: - """ - 解析亚家族文件路径 - - 命名规则: {prefix}.confirmed_subfamily.tsv (如 Ft.SPL.confirmed_subfamily.tsv) - 优先级: final/ > input/ - - Args: - ws: workspace 目录 - prefix: 分析前缀 (如 Ft.SPL) - cli_subfamily: 命令行指定的亚家族文件路径（可选） - - Returns: - 亚家族文件路径，或 None（可选） - """ - # 方式一：命令行指定 - if cli_subfamily: - if str(cli_subfamily).strip().lower() in {"none", "skip", "disabled"}: - print("[INFO] Subfamily coloring explicitly disabled") - return None - subfamily_path = resolve_job_path(ws, cli_subfamily) or Path(cli_subfamily) - if subfamily_path.exists(): - print(f"[INFO] Using specified subfamily file: {subfamily_path}") - return subfamily_path - else: - print(f"[WARNING] Specified subfamily file not found: {cli_subfamily}") - return None - - # 自动查找只接受 confirmed_subfamily；prediction/geneID 必须由 --subfamily 显式传入。 - names = [] - bases = [] - for base in [output_prefix, prefix]: - if base and base not in bases: - bases.append(base) - if output_prefix and ".subfamily_evidence" in output_prefix: - bases.append(output_prefix.replace(".subfamily_evidence", ".subfamily_prediction")) - scope_mode = str((scope_data or {}).get("scope_mode") or "").strip() - family = str((scope_data or {}).get("family") or (prefix.split(".")[-1] if "." in prefix else "")).strip() - targets = [str(x) for x in (scope_data or {}).get("target_species", []) if str(x).strip()] - refs = [str(x) for x in (scope_data or {}).get("reference_species", []) if str(x).strip()] - if scope_mode == "target_plus_reference" and len(targets) == 1 and family: - bases.append(f"{targets[0]}.{family}.target_plus_reference") - ref_part = "-".join(refs) if refs else "" - if ref_part: - bases.append(f"Target.{family}.target-{targets[0]}.ref-{ref_part}.subfamily_prediction") - seen_bases = set() - for base in bases: - if not base or base in seen_bases: - continue - seen_bases.add(base) - names.extend([ - f"{base}.confirmed_subfamily.tsv", - ]) - subfamily_name = f"{prefix}.confirmed_subfamily.tsv" - - final_dir = ws / "final" - input_dir = ws / "input" - - # 也检查 bbav2/input/{job_id}/ 路径（用户上传文件的实际位置） - job_id = ws.name - alt_input_dir = ws.parent.parent / "input" / job_id - - # 优先级: final/ > workspace/input/ > bbav2/input/{job_id}/ - search_paths = [] - if final_dir.exists(): - search_paths.extend(final_dir / name for name in names) - if input_dir.exists(): - search_paths.extend(input_dir / name for name in names) - if alt_input_dir.exists(): - search_paths.extend(alt_input_dir / name for name in names) - - for candidate in search_paths: - if candidate.exists(): - print(f"[INFO] Found subfamily file: {candidate}") - return candidate - - print(f"[INFO] No subfamily file found ({subfamily_name}), will use default coloring") - return None - - -def filter_gff3(gff3_file: Path, gene_ids: List[str], output_gff3: Path) -> Tuple[bool, Dict]: - """ - 过滤 GFF3 文件，只保留目标基因。 - - ID matching is scope-aware: GFF3 gene/transcript/CDS prefixes, case, version - suffixes and _P/_T suffixes are normalized back to the requested member IDs. - NCBI/Gnomon records often use protein IDs as mRNA/CDS IDs while storing the - transcript accession in Name/transcript_id; those transcript aliases are - captured so CDS rows are assigned back to the requested XM_* member IDs. - """ - target_lookup = build_target_id_lookup(gene_ids) - transcript_to_gene: Dict[str, str] = {} - stats = { - "total_lines": 0, - "filtered_lines": 0, - "genes_found": set(), - "mrna_count": 0, - "cds_count": 0, - "utr_count": 0 - } - - with open_text_auto(gff3_file) as f_in, open(output_gff3, 'w') as f_out: - for line in f_in: - stats["total_lines"] += 1 - - if line.startswith('#'): - continue - - line = re.sub(r'\t(\S+UTR)\t', '\tUTR\t', line, flags=re.IGNORECASE) - - parts = line.strip().split('\t') - if len(parts) != 9: - continue - - feat_type = parts[2] - - if feat_type == "gene": - continue - - attr_map = parse_gff3_attributes(parts[8]) - feat_type_norm = feat_type.lower() - canonical_id = canonical_from_attr_values( - attr_map, - ["ID", "Name", "transcript_id", "gene_id", "gene", "Parent"], - target_lookup, - ) - canonical_parent = canonical_from_attr_values( - attr_map, - ["Parent", "ID", "Name", "transcript_id", "gene_id", "gene"], - target_lookup, - ) - parent_text = attr_map.get("Parent", "") - for parent_token in str(parent_text).split(","): - if parent_token in transcript_to_gene: - canonical_parent = transcript_to_gene[parent_token] - break - - is_target = False - - if feat_type_norm in {"mrna", "transcript"} and canonical_id: - is_target = True - stats["genes_found"].add(canonical_id) - stats["mrna_count"] += 1 - for alias_key in ("ID", "Name", "transcript_id"): - alias = attr_map.get(alias_key) - if not alias: - continue - for token in str(alias).split(","): - if token: - transcript_to_gene[token] = canonical_id - elif feat_type_norm in {"cds", "utr", "exon"} and canonical_parent: - is_target = True - if feat_type_norm == "cds": - stats["cds_count"] += 1 - elif feat_type_norm == "utr": - stats["utr_count"] += 1 - - if is_target: - f_out.write('\t'.join(parts) + '\n') - stats["filtered_lines"] += 1 - - stats["genes_found"] = len(stats["genes_found"]) - return True, stats - -def gff3_to_tsv( - gff3_file: Path, - output_tsv: Path, - tree_order: List[str], - target_gene_ids: Optional[List[str]] = None, -) -> Tuple[bool, Dict]: - """将过滤后的 GFF3 转换为 TSV，并把 feature 归一到 scope/tree 成员 ID。""" - data = [] - stats = { - "total_features": 0, - "genes_with_structure": set(), - "total_cds": 0, - "total_utr": 0, - "exon_counts": {}, - "gene_lengths": {} - } - target_lookup = build_target_id_lookup(target_gene_ids or tree_order) - transcript_to_gene: Dict[str, str] = {} - seen_features: set[str] = set() - - with open_text_auto(gff3_file) as f: - for line in f: - parts = line.strip().split('\t') - if len(parts) != 9: - continue - - chrom, source, feat_type, start, end, score, strand, phase, attr = parts - feat_type_norm = feat_type.lower() - attr_map = parse_gff3_attributes(attr) - - if feat_type_norm in {"mrna", "transcript"}: - gene_id = canonical_from_attr_values( - attr_map, - ["ID", "Name", "transcript_id", "geneID", "gene_id", "gene", "Parent"], - target_lookup, - ) if target_lookup else strip_gff_id(attr_map.get("ID") or attr_map.get("Name") or attr_map.get("transcript_id") or attr_map.get("Parent")) - if gene_id: - for alias_key in ("ID", "Name", "transcript_id"): - alias = attr_map.get(alias_key) - if not alias: - continue - for token in str(alias).split(","): - if token: - transcript_to_gene[token] = gene_id - else: - gene_id = canonical_from_attr_values( - attr_map, - ["Parent", "ID", "Name", "transcript_id", "geneID", "gene_id", "gene"], - target_lookup, - ) if target_lookup else strip_gff_id(attr_map.get("Parent") or attr_map.get("ID")) - parent_text = attr_map.get("Parent", "") - for parent_token in str(parent_text).split(","): - if parent_token in transcript_to_gene: - gene_id = transcript_to_gene[parent_token] - break - - if not gene_id: - continue - - start_num = int(start) - end_num = int(end) - feature_key = f"{gene_id}|{feat_type}|{start_num}|{end_num}|{strand}|{chrom}" - if feature_key in seen_features: - continue - seen_features.add(feature_key) - - data.append({ - 'gene_id': gene_id, - 'feature': feat_type, - 'start': start_num, - 'end': end_num, - 'strand': strand, - 'chrom': chrom - }) - - stats["total_features"] += 1 - - if feat_type == "CDS": - stats["genes_with_structure"].add(gene_id) - stats["total_cds"] += 1 - stats["exon_counts"][gene_id] = stats["exon_counts"].get(gene_id, 0) + 1 - elif feat_type == "UTR": - stats["genes_with_structure"].add(gene_id) - stats["total_utr"] += 1 - - if feat_type in {"CDS", "UTR"}: - if gene_id not in stats["gene_lengths"]: - stats["gene_lengths"][gene_id] = {"min": start_num, "max": end_num} - else: - stats["gene_lengths"][gene_id]["min"] = min(stats["gene_lengths"][gene_id]["min"], start_num) - stats["gene_lengths"][gene_id]["max"] = max(stats["gene_lengths"][gene_id]["max"], end_num) - - if not data: - return False, stats - - stats["genes_with_structure"] = len(stats["genes_with_structure"]) - - for gene_id, lengths in stats["gene_lengths"].items(): - stats["gene_lengths"][gene_id] = lengths["max"] - lengths["min"] + 1 - - gene_list = sorted(set(d['gene_id'] for d in data)) - - if tree_order: - order_dict = {name: i for i, name in enumerate(tree_order)} - gene_list = sorted(gene_list, key=lambda x: order_dict.get(x, 999999)) - - gene_to_y = {g: i+1 for i, g in enumerate(gene_list)} - - with open(output_tsv, 'w') as f: - f.write('gene_id\tfeature\tstart\tend\tstrand\tchrom\ty_pos\n') - for d in data: - y_pos = gene_to_y.get(d['gene_id'], 0) - f.write(f"{d['gene_id']}\t{d['feature']}\t{d['start']}\t{d['end']}\t{d['strand']}\t{d['chrom']}\t{y_pos}\n") - - return True, stats - -def calculate_structure_metrics(stats: Dict, gene_count: int) -> Dict: - """计算结构相关的 metrics，并记录结构长度 QC。""" - genes_with_structure = min(int(stats.get("genes_with_structure", 0) or 0), max(gene_count, 0)) - metrics = { - "genes_total": gene_count, - "genes_with_structure": genes_with_structure, - "structure_coverage_rate": round(genes_with_structure / max(gene_count, 1), 4), - "total_cds_features": stats.get("total_cds", 0), - "total_utr_features": stats.get("total_utr", 0), - "avg_exon_count_per_gene": 0.0, - "avg_gene_length_bp": 0.0, - "avg_cds_length_bp": 0.0, - "median_gene_length_bp": 0.0, - "max_gene_length_bp": 0, - "long_gene_outlier_threshold_bp": 0, - "long_gene_outlier_count": 0, - "top_longest_gene_structures": [], - "long_gene_outliers": [], - } - - exon_counts = stats.get("exon_counts", {}) - gene_lengths = stats.get("gene_lengths", {}) - - if exon_counts: - metrics["avg_exon_count_per_gene"] = round(sum(exon_counts.values()) / len(exon_counts), 2) - - if gene_lengths: - lengths = sorted(int(v) for v in gene_lengths.values() if int(v) > 0) - metrics["avg_gene_length_bp"] = round(sum(lengths) / len(lengths), 2) - metrics["avg_cds_length_bp"] = round(metrics["avg_exon_count_per_gene"] * 100, 2) - mid = len(lengths) // 2 - if len(lengths) % 2: - median_len = lengths[mid] - else: - median_len = (lengths[mid - 1] + lengths[mid]) / 2 - metrics["median_gene_length_bp"] = round(median_len, 2) - metrics["max_gene_length_bp"] = max(lengths) - threshold = int(max(10000, median_len * 5)) - metrics["long_gene_outlier_threshold_bp"] = threshold - ranked = sorted(gene_lengths.items(), key=lambda item: int(item[1]), reverse=True) - metrics["top_longest_gene_structures"] = [ - { - "gene_id": gene_id, - "length_bp": int(length), - "cds_count": int(exon_counts.get(gene_id, 0) or 0), - } - for gene_id, length in ranked[:10] - ] - metrics["long_gene_outliers"] = [ - item for item in metrics["top_longest_gene_structures"] - if int(item.get("length_bp", 0)) >= threshold - ] - metrics["long_gene_outlier_count"] = sum(1 for _gene_id, length in gene_lengths.items() if int(length) >= threshold) - - return metrics - - -def run_r_plotting(tree_file: Path, structure_tsv: Path, output_prefix: Path, - rscript_path: Optional[Path] = None, cpu: int = 4, - subfamily_file: Optional[Path] = None, - project_root: Optional[Path] = None) -> Dict: - """运行 R 绘图脚本""" - if rscript_path is None: - rscript_path = Path(__file__).parent.parent.parent / "scripts" / "plot_structure.R" - - if not rscript_path.exists(): - return {"success": False, "error": f"R script not found: {rscript_path}"} - - rscript_exe = Path("/home/xzq/miniconda3/envs/test4py311/bin/Rscript") - if not rscript_exe.exists(): - rscript_exe = Path("Rscript") - - cmd = [ - str(rscript_exe), str(rscript_path), - "--tree", str(tree_file), - "--structure", str(structure_tsv), - "--output", str(output_prefix) - ] - - # 传递项目根目录，让 R 脚本自动推断 job_id 和 subfamily - if project_root and project_root.exists(): - cmd.extend(["--root", str(project_root)]) - - # 如果指定了亚家族文件，传递给 R 脚本 - if subfamily_file and subfamily_file.exists(): - cmd.extend(["--subfamily", str(subfamily_file)]) - - print(f"[CMD] {' '.join(cmd)}") - - start_time = datetime.now() - try: - result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=600) - end_time = datetime.now() - - success = result.returncode == 0 - - output_files = [] - for ext in ['.png', '.pdf']: - f = Path(f"{output_prefix}{ext}") - if f.exists(): - output_files.append(str(f)) - - return { - "success": success and len(output_files) > 0, - "returncode": result.returncode, - "stdout": result.stdout[-2000:], - "stderr": result.stderr[-2000:], - "output_files": output_files, - "elapsed_sec": (end_time - start_time).total_seconds() - } - - except subprocess.TimeoutExpired: - return {"success": False, "error": "R plotting timeout (>600s)", "elapsed_sec": 600} - except Exception as e: - return {"success": False, "error": str(e), "elapsed_sec": 0} - - -def generate_report(prefix: str, metrics: Dict, parameters: Dict, - output_file: Path, artifacts: Dict, plot_success: bool) -> None: - """生成 structure_report.md 报告""" - report_lines = [ - f"# Structure Analysis Report: {prefix}", - "", - f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", - f"**Status:** {'Success' if plot_success else 'Partial Success (Plotting Failed)'}", - "", - "## Summary", - "", - f"- **Total Genes:** {metrics['genes_total']}", - f"- **Genes with Structure:** {metrics['genes_with_structure']}", - f"- **Structure Coverage:** {metrics['structure_coverage_rate']*100:.1f}%", - f"- **Total CDS Features:** {metrics['total_cds_features']}", - f"- **Total UTR Features:** {metrics['total_utr_features']}", - "", - "## Structure Statistics", - "", - f"- **Average Exons per Gene:** {metrics['avg_exon_count_per_gene']}", - f"- **Average Gene Length:** {metrics['avg_gene_length_bp']:,.0f} bp", - f"- **Median Gene Length:** {metrics.get('median_gene_length_bp', 0):,.0f} bp", - f"- **Maximum Gene Length:** {metrics.get('max_gene_length_bp', 0):,.0f} bp", - f"- **Average CDS Length (est.):** {metrics['avg_cds_length_bp']:,.0f} bp", - "", - "## Structure Length QC", - "", - f"- **Long-gene outlier threshold:** {metrics.get('long_gene_outlier_threshold_bp', 0):,.0f} bp (max of 10 kb or 5x median gene span)", - f"- **Long-gene outliers:** {metrics.get('long_gene_outlier_count', 0)}", - "", - ] - top_longest = metrics.get("top_longest_gene_structures") or [] - if top_longest: - report_lines.extend([ - "| Rank | Gene ID | Structure span (bp) | CDS count |", - "| --- | --- | ---: | ---: |", - ]) - for idx, item in enumerate(top_longest[:10], start=1): - report_lines.append( - f"| {idx} | `{item.get('gene_id', '')}` | {int(item.get('length_bp', 0)):,} | {int(item.get('cds_count', 0))} |" - ) - report_lines.append("") - if metrics.get("long_gene_outliers"): - report_lines.append("**QC note:** One or more gene structures exceed the long-gene threshold. Inspect the source GFF3 before biological interpretation.") - report_lines.append("") - else: - report_lines.append("**QC note:** No gene structure exceeds the long-gene threshold; visually long tracks are usually due to real intron spacing relative to compact genes.") - report_lines.append("") - report_lines.extend([ - "## Parameters", - "", - f"- **GFF3 Source:** {parameters.get('gff3_source', 'N/A')}", - f"- **GFF3 Naming:** {parameters.get('gff3_naming_convention', 'N/A')}", - f"- **Subfamily File:** {parameters.get('subfamily_file', 'Not provided')}", - f"- **CPU Cores:** {parameters.get('cpu', 'N/A')}", - f"- **Tree File:** {parameters.get('tree_file', 'N/A')}", - f"- **Gene ID File:** {parameters.get('gene_id_file', 'N/A')}", - "", - "## Output Files", - "" - ]) - - for key, artifact in artifacts.items(): - if isinstance(artifact, dict): - report_lines.append(f"- **{key}:** `{artifact.get('path', 'N/A')}` ({artifact.get('type', 'unknown')})") - - report_lines.extend([ - "", - "## Visualization", - "" - ]) - - if plot_success: - report_lines.append("Gene structure visualization was successfully generated with three panels:") - report_lines.append("1. **Phylogenetic Tree** - Gene relationships with subfamily coloring") - report_lines.append("2. **Subfamily Bars** - Visual connection between tree and structure") - report_lines.append("3. **Gene Structure** - CDS (green) and UTR (orange) architecture") - else: - report_lines.append("**Warning:** Gene structure visualization failed. See logs for details.") - - report_lines.extend([ - "", - "---", - "*Generated by Gene Family Analysis Pipeline - Structure Stage*" - ]) - - with open(output_file, 'w') as f: - f.write('\n'.join(report_lines)) - - print(f"[INFO] Report generated: {output_file}") - - -def write_manifest(manifest: Dict, manifest_file: Path) -> None: - """写入 manifest 文件""" - with open(manifest_file, 'w') as f: - json.dump(manifest, f, indent=2) - print(f"[INFO] Manifest: {manifest_file}") - - -def write_failure_manifest(workspace: Path, final_dir: Path, prefix: str, - job_id: str, run_id: str, start_time: datetime, - error_msg: str, partial_artifacts: Dict = None) -> None: - """写入失败状态的 manifest""" - end_time = datetime.now() - - manifest = { - "manifest_version": "1.0", - "job_id": job_id, - "stage": "structure", - "run_id": run_id, - "prefix": prefix, - "timestamp": end_time.isoformat(), - "status": "failed", - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - "elapsed_sec": (end_time - start_time).total_seconds(), - "cmd": [str(Path(__file__).name)], - "workspace_root": str(workspace.resolve()), - "workspace": str(final_dir.resolve()), - "input_dir": str(workspace / "input"), - "artifacts": partial_artifacts or {}, - "parameters": { - "stage": "structure", - "error": error_msg - }, - "metrics": { - "error_type": "execution_failed" - } - } - - manifest_file = final_dir / f"{prefix}.structure.manifest.json" - write_manifest(manifest, manifest_file) - print(f"[ERROR] Failure manifest written: {manifest_file}") - - -def main(): - parser = argparse.ArgumentParser(description='Structure stage - Gene structure visualization') - parser.add_argument('--prefix', required=True, help='Analysis prefix (e.g., Ft.SPL)') - parser.add_argument('--workspace', required=True, help='Workspace directory') - parser.add_argument('--gff3', help='Input GFF3 file (auto-detect if not specified, naming: {species_short}_final.gff3)') - parser.add_argument('--scope-file', help='Gene-family scope JSON from resolve_scope.py') - parser.add_argument('--analysis-route', default='', help='Explicit analysis route identity for manifest/status') - parser.add_argument('--member-set-id', default='', help='Explicit member-set identity for manifest/status') - parser.add_argument('--tree', help='Phylogeny tree file; auto-detects scoped phylogeny manifest when omitted') - parser.add_argument('--subfamily', help='Subfamily classification file (explicit path; auto-detect only uses confirmed_subfamily.tsv)') - parser.add_argument('--cpu', type=int, default=4, help='CPU cores for processing') - parser.add_argument('--log-file', help='Log file path') - parser.add_argument('--run-id', help='Run ID (auto-generated if not provided)') - - args = parser.parse_args() - - run_id = args.run_id or str(uuid.uuid4())[:8] - job_id = Path(args.workspace).name - - start_time = datetime.now() - ws = Path(args.workspace) - - # 1. 创建标准目录结构（契约要求） - structure_dir = ws / "structure" / "analysis" - structure_dir.mkdir(parents=True, exist_ok=True) - final_dir = ws / "final" - final_dir.mkdir(parents=True, exist_ok=True) - - scope_data: Optional[Dict] = None - scope_file: Optional[Path] = None - if args.scope_file: - scope_file = Path(args.scope_file) - if not scope_file.exists(): - print(f"[ERROR] Scope file not found: {args.scope_file}") - write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, f"Scope file not found: {args.scope_file}") - sys.exit(1) - scope_data = read_scope(scope_file) - scope_id = str((scope_data or {}).get("scope_id") or "").strip() - output_prefix = user_output_prefix_for( - "structure", - args.prefix, - scope_data or {}, - analysis_route=args.analysis_route, - member_set_id=args.member_set_id, - ) if scope_id else args.prefix - - # 2. 解析输入文件路径（支持自动查找和命名规范） - try: - if scope_data: - gff3_sources = resolve_scope_gff3_paths(scope_data, ws=ws) - gff3_file = gff3_sources[0][1] - else: - gff3_file = resolve_gff3_path(ws, args.prefix, args.gff3) - species_short_for_gff = args.prefix.split('.')[0] if '.' in args.prefix else args.prefix - gff3_sources = [(species_short_for_gff, gff3_file)] - except FileNotFoundError as e: - print(f"[ERROR] {e}") - write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, str(e)) - sys.exit(1) - - # 亚家族文件（可选） - subfamily_file = resolve_subfamily_file(ws, args.prefix, args.subfamily, output_prefix=output_prefix, scope_data=scope_data) - - # 其他必需输入 - phase = "subfamily_evidence_required" if args.analysis_route == "subfamily_evidence" else "post_confirm_required" - try: - resolved_members = resolve_effective_members( - workspace=ws, - final_dir=final_dir, - input_dir=current_input_dir(ws), - prefix=args.prefix, - phase=phase, # type: ignore[arg-type] - require="id", - member_set_id=args.member_set_id, - ) - gene_id_file = resolved_members.path_id or Path("") - print(f"[INFO] Gene ID file from {resolved_members.source_kind}: {gene_id_file}") - except MemberResolutionError as e: - if args.analysis_route == "subfamily_evidence": - error_msg = f"Complete identify member review before Structure subfamily evidence: {e}" - else: - error_msg = f"Complete subfamily confirmation before regular Structure analysis: {e}" - print(f"[ERROR] {error_msg}") - write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, error_msg) - sys.exit(1) - try: - tree_file = resolve_scope_tree_file(ws, args.prefix, scope_data, args.tree) - except FileNotFoundError as e: - print(f"[ERROR] {e}") - write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, str(e)) - sys.exit(1) - if scope_data: - target_species_set = {str(item) for item in scope_data.get("target_species", []) if str(item).strip()} - tree_ids = [] - for item in read_tree_order(tree_file): - if not item: - continue - if "|" in item: - species_code, gene_code = item.split("|", 1) - if target_species_set and species_code not in target_species_set: - continue - tree_ids.append(gene_code) - else: - tree_ids.append(item) - if tree_ids: - gene_id_file = structure_dir / f"{output_prefix}.scope_members.id" - with gene_id_file.open("w", encoding="utf-8") as handle: - for gene_id in tree_ids: - handle.write(gene_id + "\n") - - # 验证其他输入 - missing = [] - if not gene_id_file.exists(): - missing.append(f"Gene ID file: {gene_id_file}") - - if missing: - error_msg = "Missing required input files:\n" + "\n".join(missing) - print(f"[ERROR] {error_msg}") - write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, error_msg) - sys.exit(1) - - # 提取物种简写用于报告 - species_short = args.prefix.split('.')[0] if '.' in args.prefix else args.prefix - gff3_naming = ( - "scope target GFF3: " + ", ".join(f"{species}:{path.name}" for species, path in gff3_sources) - if scope_data else f"{species_short}_final.gff3" - ) - - print(f"[INFO] Structure stage started for {args.prefix}") - print(f"[INFO] Job ID: {job_id}, Run ID: {run_id}") - print(f"[INFO] GFF3 naming convention: {gff3_naming}") - print(f"[INFO] GFF3 source: {gff3_file}") - if len(gff3_sources) > 1: - for species, source_path in gff3_sources: - print(f"[INFO] Scope GFF3 source: {species} -> {source_path}") - - # 3. 读取输入数据 - gene_ids = read_gene_ids(gene_id_file) - print(f"[INFO] {len(gene_ids)} genes to analyze") - - tree_order = read_tree_order(tree_file) - print(f"[INFO] Tree order: {len(tree_order)} tips") - - # 4. 过滤 GFF3（中间文件放入 structure/analysis/） - structure_gff3 = structure_dir / f"{output_prefix}.structure.filtered.gff3" - filter_success, filter_stats = filter_scope_gff3(gff3_sources, gene_ids, structure_gff3, structure_dir, args.prefix) - - if not filter_success: - error_msg = "Failed to filter GFF3 file" - print(f"[ERROR] {error_msg}") - write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, error_msg) - sys.exit(1) - - print(f"[INFO] Filtered GFF3: {structure_gff3}") - print(f"[INFO] - Total lines: {filter_stats['total_lines']}") - print(f"[INFO] - Filtered lines: {filter_stats['filtered_lines']}") - print(f"[INFO] - Genes found: {filter_stats['genes_found']}") - - # 5. 转换为 TSV（中间文件放入 structure/analysis/） - structure_tsv = structure_dir / f"{output_prefix}.structure.tsv" - convert_success, convert_stats = gff3_to_tsv(structure_gff3, structure_tsv, tree_order, gene_ids) - - if not convert_success: - error_msg = "Failed to convert GFF3 to TSV (no valid features found)" - print(f"[ERROR] {error_msg}") - write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, error_msg) - sys.exit(1) - - print(f"[INFO] Structure TSV: {structure_tsv}") - print(f"[INFO] - Genes with structure: {convert_stats['genes_with_structure']}") - print(f"[INFO] - Total CDS: {convert_stats['total_cds']}") - print(f"[INFO] - Total UTR: {convert_stats['total_utr']}") - - # 6. 计算结构 metrics - structure_metrics = calculate_structure_metrics(convert_stats, len(gene_ids)) - print(f"[INFO] Metrics calculated:") - for k, v in structure_metrics.items(): - print(f" - {k}: {v}") - - # 7. R 绘图（输出到 final/） - output_path_prefix = final_dir / f"{output_prefix}.structure" - project_root = Path(__file__).parent.parent.parent # bbav5 根目录 - plot_result = run_r_plotting( - tree_file, - structure_tsv, - output_path_prefix, - cpu=args.cpu, - subfamily_file=subfamily_file, - project_root=project_root - ) - - plot_success = plot_result.get("success", False) - if not plot_success: - print(f"[WARNING] R plotting failed: {plot_result.get('error', plot_result.get('stderr', 'unknown'))}") - - print(f"[INFO] Plot files: {plot_result.get('output_files', [])}") - - # 8. 准备 artifacts - end_time = datetime.now() - artifacts = {} - - # 中间文件 - if structure_gff3.exists(): - artifacts["structure_gff3_filtered"] = { - "artifact": "structure_gff3_filtered", - "path": str(structure_gff3.relative_to(ws)), - "full_path": display_path(structure_gff3), - "type": "gff3", - "size": structure_gff3.stat().st_size, - "sha256": compute_file_hash(structure_gff3), - "description": "Filtered GFF3 with only target genes (intermediate)" - } - - if structure_tsv.exists(): - artifacts["structure_tsv"] = { - "artifact": "structure_tsv", - "path": str(structure_tsv.relative_to(ws)), - "full_path": display_path(structure_tsv), - "type": "tsv", - "size": structure_tsv.stat().st_size, - "description": "Gene structure data in TSV format (intermediate)" - } - - # 最终产物 - for img_file in plot_result.get("output_files", []): - img_path = Path(img_file) - if img_path.exists(): - key = f"structure_plot_{img_path.suffix[1:]}" - artifacts[key] = { - "artifact": "structure_plot", - "path": str(img_path.relative_to(ws)), - "full_path": display_path(img_path), - "type": "image", - "format": img_path.suffix[1:], - "size": img_path.stat().st_size, - "sha256": compute_file_hash(img_path), - "description": f"Gene structure visualization ({img_path.suffix[1:].upper()})" - } - - # 9. 生成报告 - report_file = final_dir / f"{output_prefix}.structure_report.md" - - parameters = { - "cpu": args.cpu, - "prefix": args.prefix, - "output_prefix": output_prefix, - "stage": "structure", - "job_id": job_id, - "run_id": run_id, - "gff3_source": str(gff3_file), - "gff3_sources": [{"species": species, "path": str(path)} for species, path in gff3_sources], - "gff3_naming_convention": gff3_naming, - "gff3_source_sha256": compute_file_hash(gff3_file), - "subfamily_file": str(subfamily_file) if subfamily_file else None, - "tree_file": str(tree_file), - "gene_id_file": str(gene_id_file), - "gene_count": len(gene_ids), - "tree_tip_count": len(tree_order), - "Rscript_path": str(Path(__file__).parent.parent.parent / "scripts" / "plot_structure.R"), - "tool_versions": get_tool_versions(), - "timestamp_start": start_time.isoformat(), - "timestamp_end": end_time.isoformat(), - "workspace_root": display_path(ws), - "input_dir": str(ws / "input"), - "structure_analysis_dir": str(structure_dir), - "final_dir": str(final_dir) - } - - generate_report(output_prefix, structure_metrics, parameters, report_file, artifacts, plot_success) - - if report_file.exists(): - artifacts["structure_report"] = { - "artifact": "structure_report", - "path": str(report_file.relative_to(ws)), - "full_path": display_path(report_file), - "type": "report", - "format": "markdown", - "size": report_file.stat().st_size, - "sha256": compute_file_hash(report_file), - "description": "Human-readable analysis report" - } - - # 10. 确定状态 - status = "success" if plot_success else "partial_success" - - # 11. 构建完整 manifest - manifest = { - "manifest_version": "1.0", - "job_id": job_id, - "stage": "structure", - "run_id": run_id, - "prefix": args.prefix, - "output_prefix": output_prefix, - "timestamp": end_time.isoformat(), - "status": status, - "start_time": start_time.isoformat(), - "end_time": end_time.isoformat(), - "elapsed_sec": (end_time - start_time).total_seconds(), - "cmd": [ - str(Path(__file__).name), - "--prefix", args.prefix, - "--workspace", str(ws), - "--gff3", str(gff3_file), - "--cpu", str(args.cpu), - "--run-id", run_id - ] + (["--scope-file", str(scope_file)] if scope_file else []) + (["--subfamily", str(subfamily_file)] if subfamily_file else []), - "workspace_root": display_path(ws), - "workspace": display_path(structure_dir), - "input_dir": str(ws / "input"), - "artifacts": artifacts, - "parameters": parameters, - "metrics": structure_metrics - } - if scope_data: - manifest["scope_id"] = scope_data.get("scope_id", "") - manifest["scope_mode"] = scope_data.get("scope_mode", "") - manifest["scope_file"] = str(scope_file) - manifest["target_species"] = scope_data.get("target_species", []) - manifest["reference_species"] = scope_data.get("reference_species", []) - manifest["comparative_species"] = scope_data.get("comparative_species", []) - manifest["selected_species"] = scope_data.get("selected_species", []) - manifest["scope"] = { - "family": scope_data.get("family", ""), - "prefix": scope_data.get("prefix", ""), - "scope_id": scope_data.get("scope_id", ""), - "scope_mode": scope_data.get("scope_mode", ""), - "target_species": scope_data.get("target_species", []), - "reference_species": scope_data.get("reference_species", []), - "comparative_species": scope_data.get("comparative_species", []), - "selected_species": scope_data.get("selected_species", []), - } - context = build_stage_context( - stage="structure", - prefix=args.prefix, - scope_data=scope_data or {}, - scope_file=scope_file or "", - output_prefix=output_prefix, - subfamily_file=subfamily_file, - analysis_route=args.analysis_route, - member_set_id=args.member_set_id, - ) - manifest["tree_source"] = str(tree_file) - manifest["gene_id_file"] = str(gene_id_file) - manifest["gff3_source"] = str(gff3_file) - apply_stage_context(manifest, context) - - manifest_file = final_dir / f"{output_prefix}.structure.manifest.json" - write_manifest(manifest, manifest_file) - if scope_data and scope_data.get("scope_id"): - write_legacy_manifest_copy(manifest_file, args.prefix, str(scope_data.get("scope_id") or ""), "structure") - - # 12. 完成 - if status == "success": - print(f"[INFO] Structure stage completed successfully") - else: - print(f"[WARNING] Structure stage completed with partial success (plotting failed)") - - print("\n=== OUTPUT_FILES ===") - for k, v in artifacts.items(): - if isinstance(v, dict) and v.get("type") == "image": - print(f"IMAGE: {v['full_path']}") - - sys.exit(0 if status == "success" else 1) - - -if __name__ == "__main__": - main() diff --git a/bioflow_kernel/gfa_core/mas/agents/diagnosis.py.orig b/bioflow_kernel/gfa_core/mas/agents/diagnosis.py.orig deleted file mode 100755 index cb2c231..0000000 --- a/bioflow_kernel/gfa_core/mas/agents/diagnosis.py.orig +++ /dev/null @@ -1,4545 +0,0 @@ -""" -DiagnosisAgent V3 - 错误诊断与自动修复 -集成Knowledge查询、自动修复、前端通知和全局迭代计数 -""" -import re -import json -import shutil -import shlex -import gzip -import os -import subprocess -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple -from datetime import datetime -from .base import AgentMessage, AgentState, BaseAgent -from gfa_core.mas.llm import LLMManager -from gfa_core.mas.stage_catalog import STAGE_DEPENDENCIES as CATALOG_STAGE_DEPENDENCIES -from gfa_core.project_paths import PROJECT_ROOT -from gfa_core.ppi_provenance import ppi_diagnosis_code - - -STAGE_SCRIPT_MAP = { - "identify": "run_identify.py", - "phylogeny": "run_phylogeny.py", - "motif": "run_motif.py", - "domain": "run_domain.py", - "structure": "run_structure.py", - "promoter": "run_promoter.py", - "synteny": "run_synteny.py", - "interspecies_synteny": "run_interspecies_synteny.py", - "karyotype": "run_karyotype.py", - "combo": "run_combo.py", - "ppi": "run_ppi.py", - "expression": "run_expression.py", - "pep": "run_pep.py", - "rnaseq_qc": "run_rnaseq_qc.py", - "rnaseq_align": "run_rnaseq_align.py", - "rnaseq_quant": "run_rnaseq_quant.py", - "rnaseq_de": "run_rnaseq_de.py", - "rnaseq_enrich": "run_rnaseq_enrich_v2.py", - "rnaseq_go_enrich": "run_rnaseq_enrich.py", - "rnaseq_kegg_enrich": "run_rnaseq_enrich.py", - "rnaseq_fgsea": "run_rnaseq_fgsea.py", - "annotation": "run_annotation.py", - "annotation_go": "run_annotation.py", - "annotation_kegg": "run_annotation.py", - "rnaseq_wgcna": "run_rnaseq_wgcna.py", - "rnaseq_candidate_family": "run_rnaseq_candidate_family.py", - "rnaseq_sample_qc": "run_rnaseq_sample_qc.py", - "genome_prepare": "run_genome_prepare.py", - "genome_prepare_v2": "run_genome_prepare.py", - "ortholog": "run_ortholog.py", - "ncbi_fetch_sequences": "run_ncbi_fetch_sequences.py", - "sequence_extract_by_id": "run_sequence_extract_by_id.py", -} -RAP_TRANSCRIPT_RE = re.compile(r"^(Os\d{2})t(\d+)(?:-\d+)?$", re.IGNORECASE) -KNOWN_COPY_PATCH_CODES = { - "python_import_path_error", - "protein_fasta_stop_markers", - "r_table_comment_char", - "success_with_zero_alignment_rate", - "success_with_zero_qc_reads", - "fgsea_rank_gene2ko_id_mismatch", - "wgcna_matrix_comment_char", - "quant_fpkm_tpm_comment_char", -} -RUNTIME_REPAIR_CODES = { - "protein_fasta_stop_markers", - "identify_pfam_remote_404", - "identify_pfam_partial_fetch", - "identify_hmm_model_mismatch", - "pfam_remote_404", - "identify_missing_hmm_or_pfam", - "ncbi_family_reference_link_missing", - "manifest_missing", - "manifest_missing_stage_incomplete", - "ortholog_intermediate_not_materialized", - "genome_prepare_final_gff_empty", - "genome_prepare_pep_looks_like_genome", - "genome_prepare_cds_looks_like_genome", - "gene_gff_id_overlap_low", -} -REPAIR_POLICY_BY_CODE = { - "protein_fasta_stop_markers": ("auto", "protein_stop_marker_clean"), - "genome_prepare_final_gff_empty": ("auto", "restore_final_gff_from_source"), - "genome_prepare_pep_looks_like_genome": ("auto", "extract_pep_from_genome_gff"), - "genome_prepare_cds_looks_like_genome": ("auto", "extract_cds_from_genome_gff"), - "gene_gff_id_overlap_low": ("report_only", "write_id_overlap_report"), - "identify_pfam_remote_404": ("auto", "prepare_local_pfam_hmm"), - "identify_pfam_partial_fetch": ("auto", "prepare_local_pfam_hmm"), - "identify_hmm_model_mismatch": ("auto", "prepare_local_pfam_hmm"), - "pfam_remote_404": ("auto", "prepare_local_pfam_hmm"), - "ncbi_family_reference_link_missing": ("auto", "create_family_reference_pep_link"), - "hmm_or_pfam_missing": ("confirm", "provide_hmm_or_required_pfams"), - "identify_missing_hmm_or_pfam": ("auto", "prepare_local_pfam_hmm"), - "manifest_missing_stage_incomplete": ("auto", "recover_manifest_from_final_artifacts"), - "manifest_missing": ("auto", "recover_manifest_from_final_artifacts"), - "ortholog_intermediate_not_materialized": ("auto", "materialize_ortholog_from_intermediate"), - "manifest_failed": ("report_only", "inspect_stage_log"), - "stage_runtime_same_file_copy": ("dev_fix", "guard_same_file_copy"), - "expression_missing_gene_ids": ("report_only", "provide_gene_family_member_ids"), - "expression_missing_sample_txt": ("report_only", "provide_or_convert_sample_table"), - "rnaseq_auto_expression_missing": ("report_only", "rerun_rnaseq_de_or_expression_rnaseq_auto"), - "rnaseq_auto_expression_failed": ("report_only", "inspect_expression_log"), - "rnaseq_context_missing": ("dev_fix", "align_rnaseq_manifest_context"), - "stage_identity_contract_mismatch": ("dev_fix", "align_manifest_identity_with_scope_contract"), - "combo_artifact_scope_mismatch": ("dev_fix", "align_combo_output_naming_with_scope_contract"), - "domain_scoped_identify_guard": ("report_only", "align_domain_backend_guard_with_scope_contract"), - "upstream_genome_prepare_partial_success": ("report_only", "repair_or_exclude_failed_species"), - "plantcare_id_mismatch": ("report_only", "rerun_plantcare_for_current_promoters"), - "multi_target_single_species_stage": ("report_only", "split_target_scope_per_species"), - "kaks_scope_missing_reference": ("dev_fix", "plan_kaks_with_target_plus_reference_scope"), - "interspecies_pair_no_hits": ("report_only", "inspect_proteome_or_relax_similarity_threshold"), - "go_foreground_background_id_mismatch": ("confirm", "use_auto_annotation_background"), - "id_fasta_mismatch": ("report_only", "convert_gene_or_transcript_ids_before_sequence_extraction"), - "blast_binding_or_output_failure": ("report_only", "check_extension_stage_query_reference_binding"), - "misplaced_outputs": ("dev_fix", "fix_extension_wrapper_output_directory"), - "hmmer_no_hits_or_threshold": ("report_only", "inspect_hmmer_hits_and_thresholds"), - "hmmer_postprocess_empty": ("dev_fix", "fix_hmmer_postprocess_parser"), -} -ISSUE_LAYER_BY_CODE = { - "multi_target_single_species_stage": "planning", - "kaks_scope_missing_reference": "planning", - "hmm_or_pfam_missing": "input", - "identify_missing_hmm_or_pfam": "input", - "identify_pfam_partial_fetch": "tool_runtime", - "identify_hmm_model_mismatch": "tool_runtime", - "protein_fasta_stop_markers": "input_repair", - "missing_final_member_protein": "artifact_status", - "manifest_missing": "artifact_status", - "manifest_missing_stage_incomplete": "artifact_status", - "ortholog_intermediate_not_materialized": "artifact_status", - "manifest_failed": "artifact_status", - "stage_runtime_same_file_copy": "execution", - "expression_missing_gene_ids": "input", - "expression_missing_sample_txt": "input", - "rnaseq_auto_expression_missing": "execution", - "rnaseq_auto_expression_failed": "execution", - "rnaseq_context_missing": "artifact_status", - "stage_identity_contract_mismatch": "artifact_status", - "combo_artifact_scope_mismatch": "artifact_status", - "domain_scoped_identify_guard": "execution", - "success_with_empty_metrics": "artifact_status", - "success_with_zero_alignment_rate": "artifact_status", - "success_with_zero_qc_reads": "artifact_status", - "tool_version_format": "tool_runtime", - "pfam_remote_404": "tool_runtime", - "identify_pfam_remote_404": "tool_runtime", - "interspecies_pair_no_hits": "tool_runtime", - "id_mismatch": "input_contract", - "plantcare_id_mismatch": "input_contract", - "gene_gff_id_overlap_low": "input_contract", - "go_foreground_background_id_mismatch": "input_contract", - "id_fasta_mismatch": "input_contract", - "blast_binding_or_output_failure": "execution", - "misplaced_outputs": "execution", - "hmmer_no_hits_or_threshold": "tool_runtime", - "hmmer_postprocess_empty": "execution", -} -ISSUE_LAYER_PRIORITY = { - "planning": 0, - "input_contract": 1, - "input": 2, - "tool_runtime": 3, - "artifact_status": 4, - "input_repair": 5, - "report": 6, -} -DIAGNOSIS_STAGE_COVERAGE = { - "genome_prepare": { - "preflight": ["config_exists", "declared_files_exist", "fasta_content_type", "gff_nonempty"], - "postflight": ["manifest_status", "species_result_status", "final_gff_nonempty", "pep_cds_content_type"], - "failure": ["log_root_cause", "runtime_repair_candidates"], - }, - "identify": { - "preflight": ["genome_prepare_quality", "target_protein_available", "hmm_or_required_pfams"], - "postflight": ["final_members_nonempty", "protein_stop_markers", "domain_check"], - "failure": ["hmm_pfam_missing", "target_content_type", "header_length"], - }, - "identify_refine": { - "preflight": ["curated_id_list", "protein_source", "id_overlap"], - "postflight": ["refined_members_nonempty", "refined_protein_nonempty"], - "failure": ["missing_ids", "id_overlap_zero"], - }, - "phylogeny": { - "preflight": ["final_member_protein", "protein_stop_markers", "sequence_count"], - "postflight": ["tree_nonempty", "plot_nonempty"], - "failure": ["mafft_iqtree_error", "protein_format"], - }, - "reference_classification": { - "preflight": ["phylogeny_tree", "reference_subfamily"], - "postflight": ["assignment_table", "subfamily_table"], - "failure": ["missing_phylogeny_tree", "missing_reference_subfamily"], - }, - "motif": { - "preflight": ["final_member_protein", "protein_stop_markers", "sequence_count"], - "postflight": ["meme_outputs", "motif_plot"], - "failure": ["meme_empty", "protein_format"], - }, - "domain": { - "preflight": ["final_member_protein", "pfam_database", "protein_stop_markers"], - "postflight": ["domain_table", "domain_plot"], - "failure": ["pfam_unavailable", "hmmscan_parse"], - }, - "structure": { - "preflight": ["final_members", "gff_overlap"], - "postflight": ["structure_outputs"], - "failure": ["id_overlap_zero", "tool_unavailable"], - }, - "promoter": { - "preflight": ["final_members", "genome", "gff_overlap"], - "postflight": ["promoter_fasta", "cis_element_outputs"], - "failure": ["gff_id_mismatch", "genome_missing"], - }, - "synteny": { - "preflight": ["whole_genome_pep", "gff_nonempty", "family_ids", "id_overlap"], - "postflight": ["anchors", "plots"], - "failure": ["mcscanx_empty", "gff_id_mismatch"], - }, - "interspecies_synteny": { - "preflight": ["target_ref_pep", "target_ref_gff", "species_selection"], - "postflight": ["pairwise_anchors", "jcvi_plot", "tree_plot"], - "failure": ["missing_whole_proteome", "species_config_misread"], - }, - "karyotype": { - "preflight": ["gff_nonempty", "family_ids", "chromosome_assignment"], - "postflight": ["karyotype_plot", "unplaced_report"], - "failure": ["empty_gff", "all_unplaced"], - }, - "combo": { - "preflight": ["phylogeny", "motif", "domain", "structure"], - "postflight": ["combo_plot"], - "failure": ["missing_upstream_artifacts"], - }, - "ortholog": { - "preflight": ["target_family_pep", "reference_family_pep", "scope_reference_species"], - "postflight": ["ortholog_pairs", "ortholog_manifest", "intermediate_materialized"], - "failure": ["blast_failed", "intermediate_not_materialized", "post_processing_failed"], - }, - "ppi": { - "preflight": ["family_ids", "string_species_or_orthologs", "ppi_provenance_identity"], - "postflight": ["ppi_edges", "ppi_plot"], - "failure": [ - "string_unavailable", - "ortholog_missing", - "ppi_requires_ortholog_mapping", - "ppi_reference_data_missing", - "ppi_invalid_ortholog_mapping", - "ppi_string_id_mapping_low", - ], - }, - "expression": { - "preflight": ["expression_matrix", "sample_info", "family_ids_optional"], - "postflight": ["heatmap", "expression_table"], - "failure": ["matrix_id_mismatch", "sample_group_error"], - }, - "pep": { - "preflight": ["final_member_protein", "protein_stop_markers"], - "postflight": ["physicochemical_table", "feature_plots", "tool_availability"], - "failure": ["protein_format", "external_tool_unavailable"], - }, - "rnaseq_qc": { - "preflight": ["samples_yaml", "fastq_pairs", "duplicate_fastq"], - "postflight": ["clean_fastq", "qc_metrics"], - "failure": ["missing_fastq", "sample_config_error"], - }, - "rnaseq_align": { - "preflight": ["clean_fastq", "genome_index"], - "postflight": ["bam_nonempty", "alignment_rate"], - "failure": ["index_missing", "zero_alignment"], - }, - "rnaseq_quant": { - "preflight": ["bam_files", "gtf_gff", "featurecounts_ready"], - "postflight": ["count_matrix", "fpkm_tpm", "matrix_parse"], - "failure": ["featurecounts_error", "comment_char_parse"], - }, - "extension_stage": { - "preflight": ["publish_gate", "semantic_input_binding", "explicit_file_roles"], - "postflight": ["output_contract", "empty_outputs", "manifest_quality", "prefix_consistency"], - "failure": ["diagnosis_report", "stdout_stderr", "repair_candidate_gate"], - }, - "rnaseq_de": { - "preflight": ["count_matrix", "samples_yaml", "comparisons"], - "postflight": ["de_results", "significant_gene_counts"], - "failure": ["comparison_missing", "design_error"], - }, - "rnaseq_sample_qc": { - "preflight": ["count_or_vst_matrix", "sample_table"], - "postflight": ["pca", "correlation", "duplicate_samples"], - "failure": ["matrix_parse", "sample_mismatch"], - }, - "rnaseq_go_enrich": { - "preflight": ["de_results", "gene2go_or_annotation", "id_overlap"], - "postflight": ["go_results", "foreground_background_overlap"], - "failure": ["missing_gene2go", "id_overlap_zero"], - }, - "rnaseq_kegg_enrich": { - "preflight": ["de_results", "gene2ko_or_annotation", "id_overlap"], - "postflight": ["kegg_results", "foreground_background_overlap"], - "failure": ["missing_gene2ko", "id_overlap_zero"], - }, - "rnaseq_enrich": { - "preflight": ["de_results", "background_source", "go_kegg_background"], - "postflight": ["go_kegg_results", "id_overlap"], - "failure": ["missing_background", "id_overlap_zero"], - }, - "rnaseq_fgsea": { - "preflight": ["de_ranks", "gene2ko", "rank_background_overlap"], - "postflight": ["fgsea_results", "pathway_size"], - "failure": ["rank_gene2ko_id_mismatch", "kegg_species"], - }, - "rnaseq_wgcna": { - "preflight": ["expression_or_count_matrix", "sample_table"], - "postflight": ["module_count", "grey_ratio", "soft_threshold"], - "failure": ["matrix_parse", "sample_size"], - }, - "rnaseq_candidate_family": { - "preflight": ["de_results", "annotation_sources", "optional_enrichment_wgcna"], - "postflight": ["candidate_family_ranking", "evidence_matrix", "recommendation", "rag_queries", "rag_references"], - "failure": ["annotation_missing", "id_mapping"], - }, - "annotation": { - "preflight": ["protein_background", "tool_availability"], - "postflight": ["gene2go", "gene2ko", "annotation_coverage"], - "failure": ["interproscan_unavailable", "kofamscan_unavailable"], - }, - "annotation_go": { - "preflight": ["protein_background", "interproscan_availability"], - "postflight": ["gene2go", "annotation_go_manifest", "go_progress"], - "failure": ["interproscan_unavailable", "go_terms_empty"], - }, - "annotation_kegg": { - "preflight": ["protein_background", "kofamscan_availability"], - "postflight": ["gene2ko", "annotation_kegg_manifest", "kegg_progress"], - "failure": ["kofamscan_unavailable", "ko_hits_empty"], - }, -} - - -class DiagnosisAgent(BaseAgent): - """ - 诊断Agent V3 - - 功能： - 1. 分析错误日志，定位问题 - 2. 调用Knowledge查询修复方案 - 3. 自动修复常见问题 - 4. 生成前端通知消息 - 5. 集成全局迭代计数 - 6. 记录错误模式 - """ - name = "diagnosis" - - @staticmethod - def ppi_identity_diagnosis(payload: Dict[str, Any]) -> Dict[str, Any]: - provenance = payload.get("ppi_provenance") if isinstance(payload.get("ppi_provenance"), dict) else {} - selection_mode = str(provenance.get("selection_mode") or payload.get("selection_mode") or "unresolved") - code = str(provenance.get("diagnosis_code") or ppi_diagnosis_code(selection_mode, str(payload.get("status") or ""))) - return { - "code": code, - "selection_mode": selection_mode, - "selected_string_taxon_id": str(provenance.get("selected_string_taxon_id") or ""), - "snapshot": dict(provenance.get("string_snapshot") or {}), - } - role = "Error Diagnosis Specialist" - - # 常见错误模式及修复方案 - ERROR_PATTERNS = { - "missing_input_file": { - "patterns": [r"文件不存在", r"FileNotFoundError", r"No such file", r"找不到文件"], - "suggestion": "检查输入文件路径是否正确，确保文件已上传到input目录", - "auto_fix": False - }, - "hmm_not_found": { - "patterns": [r"required_pfams is empty", r"No HMM file", r"找不到HMM"], - "suggestion": "请上传HMM模型文件或指定--required-pfams参数", - "auto_fix": False - }, - "blast_db_error": { - "patterns": [r"BLAST Database error", r"makeblastdb failed"], - "suggestion": "Identify BLAST 固定用目标物种全蛋白 FASTA 建库；请优先检查目标全蛋白 FASTA 是否存在、非空且为有效蛋白FASTA格式，再检查参考家族蛋白 query 文件。", - "auto_fix": False - }, - "memory_error": { - "patterns": [r"MemoryError", r"out of memory", r"内存不足"], - "suggestion": "减少--cpu参数值或分批处理，建议使用--cpu 4", - "auto_fix": True, - "fix_action": "reduce_cpu" - }, - "rnaseq_library_strandness_error": { - "patterns": [r"library_strandness", r"链特异性", r"rna-strandness", r"featureCounts.*-s"], - "suggestion": "请在 samples.yaml 的 rnaseq.library_strandness 或每个 samples..library_strandness 中明确填写 non_stranded / forward / reverse；混合链特异性样本当前不能用单次 featureCounts 定量。", - "auto_fix": False - }, - "timeout_error": { - "patterns": [r"Timeout", r"timed out", r"超时"], - "suggestion": "增加超时时间或检查输入数据大小", - "auto_fix": False - }, - "conda_env_error": { - "patterns": [r"conda", r"Environment", r"找不到命令"], - "suggestion": "检查conda环境是否正确激活，确保工具已安装", - "auto_fix": False - }, - "permission_error": { - "patterns": [r"Permission denied", r"权限不足"], - "suggestion": "检查文件权限，确保有读写权限", - "auto_fix": False - }, - "invalid_parameter": { - "patterns": [r"unrecognized arguments", r"invalid parameter", r"参数错误"], - "suggestion": "检查参数名称是否正确，注意使用短横线而非下划线", - "auto_fix": True, - "fix_action": "fix_parameter_format" - }, - "tool_not_found": { - "patterns": [ - r"No such file or directory: 'mafft'", - r"No such file or directory: 'makeblastdb'", - r"No such file or directory: 'iqtree'", - r"No such file or directory: 'hmmsearch'", - r"command not found: mafft", - r"command not found: blastp", - r"command not found: iqtree", - r"command not found: hmmsearch", - r"RuntimeError: MAFFT失败.*No such file", - r"RuntimeError:.*失败.*No such file or directory" - ], - "suggestion": "生物信息学工具未安装或未添加到PATH。建议：1) 激活conda环境: conda activate test4py311 2) 安装缺失工具: conda install -c bioconda mafft blast iqtree hmmer", - "auto_fix": False, - "severity": "high" - }, - "dependency_failed": { - "patterns": [ - r"依赖阶段.*未完成", - r"derived_from.*manifest.*not found", - r"前置阶段.*未执行", - r"请先执行.*阶段", - r"requires.*stage.*not completed", - r"dependency.*not satisfied" - ], - "suggestion": "前置分析阶段未完成，系统将自动先执行依赖阶段", - "auto_fix": True, - "fix_action": "execute_dependencies", - "severity": "high" - }, - "go_foreground_background_id_mismatch": { - "patterns": [ - r"GO foreground/background ID mismatch", - r"no DE genes overlap uploaded gene2go background", - r"foreground.*background.*ID mismatch", - r"gene2go.*DE genes.*overlap=0" - ], - "suggestion": "当前 gene2go 背景与 DE 基因 ID 体系不一致。应提示用户确认本地自动注释，生成与当前蛋白/DE ID 匹配的 gene2go 后再做 GO 富集。", - "auto_fix": False, - "severity": "high" - } - } - - # 参数修复映射 - PARAMETER_FIXES = { - "dom_i_evalue_max": "dom-i-evalue-max", - "hmm_cov_min": "hmm-cov-min", - "blast_evalue": "blast-evalue", - "idEvalue": "id-evalue", - "idCoverage": "id-coverage" - } - - def __init__(self, llm_manager: LLMManager): - super().__init__(llm_manager) - self.error_history: List[Dict] = [] - self.auto_fix_count = 0 - - async def execute(self, state: AgentState, message: AgentMessage) -> AgentState: - """执行诊断""" - print(f"[DiagnosisAgent] 开始诊断...") - - error_info = message.payload.get("error", {}) - execution_result = message.payload.get("execution_result", {}) - - # 提取错误信息 - stage = error_info.get("stage") or execution_result.get("stage", "unknown") - error_message = error_info.get("message") or error_info.get("error") or execution_result.get("error") or execution_result.get("message") or "" - stderr = error_info.get("stderr") or execution_result.get("stderr", "") - stdout = error_info.get("stdout") or execution_result.get("stdout", "") - full_error = f"{error_message}\n{stderr}\n{stdout}" - - print(f"[DiagnosisAgent] 诊断阶段: {stage}") - - # 诊断错误 - diagnosis = await self._diagnose_error(stage, full_error, error_info) - - # 如果未匹配到已知模式，调用 Knowledge Agent 查询 - if diagnosis.get("error_type") == "unknown": - print(f"[DiagnosisAgent] 未知错误，尝试调用 Knowledge Agent...") - knowledge_result = await self._query_knowledge_for_error(full_error) - if knowledge_result: - diagnosis = knowledge_result - - # 获取当前迭代计数 - iteration_count = state.results.get("iteration_count", 0) - max_iterations = state.results.get("max_iterations", 3) - - # 检查是否达到最大迭代次数 - if iteration_count >= max_iterations: - diagnosis["can_auto_fix"] = False - diagnosis["suggestion"] = f"已达到最大迭代次数({max_iterations})，停止自动修复。" - diagnosis["stop_iteration"] = True - - # 尝试自动修复 - if diagnosis.get("can_auto_fix") and diagnosis.get("fix_action"): - fix_result = await self._attempt_auto_fix( - diagnosis["fix_action"], error_info, full_error, diagnosis - ) - diagnosis["fix_result"] = fix_result - - if fix_result and fix_result.get("success"): - self.auto_fix_count += 1 - iteration_count += 1 - state.results["iteration_count"] = iteration_count - state.results["auto_fix_count"] = self.auto_fix_count - diagnosis["iteration_count"] = iteration_count - - # 生成前端通知 - notification = self._generate_notification(stage, diagnosis, fix_result) - diagnosis["notification"] = notification - state.results["auto_fix_notification"] = notification - - # 更新状态 - state.results["diagnosis"] = diagnosis - state.results["can_auto_fix"] = diagnosis.get("can_auto_fix", False) - - state.messages.append({ - "role": "diagnosis", - "content": f"🔍 诊断完成: {diagnosis.get('error_type', 'unknown')}", - }) - - return state - - async def _diagnose_error(self, stage: str, error_text: str, error_info: Dict) -> Dict: - """诊断错误""" - # 尝试匹配已知错误模式 - matched_pattern = self._match_error_pattern(error_text) - - if matched_pattern: - error_type = matched_pattern["error_type"] - suggestion = matched_pattern["suggestion"] - can_auto_fix = matched_pattern.get("auto_fix", False) - fix_action = matched_pattern.get("fix_action") - - return { - "error_type": error_type, - "stage": stage, - "suggestion": suggestion, - "can_auto_fix": can_auto_fix, - "fix_action": fix_action, - "confidence": 0.9, - "error_text": error_text[:500] - } - - # 未知错误 - return { - "error_type": "unknown", - "stage": stage, - "suggestion": "无法自动诊断，请检查错误日志", - "can_auto_fix": False, - "requires_user_confirmation": True, - "confirmation_reason": "未知错误不能自动修复，需要用户查看日志后确认下一步。", - "confidence": 0.3, - "error_text": error_text[:500] - } - - def diagnose_job_stage( - self, - job_id: str, - stage: str, - prefix: str = "", - project_root: Path = PROJECT_ROOT, - prepare_patch_workspace: bool = False, - scope_id: str = "", - params: Optional[Dict[str, Any]] = None, - ) -> Dict[str, Any]: - """只读审计某个 job/stage 的产物、manifest、日志和输入配置。 - - 该方法不重跑阶段、不修改输入、不修改全局脚本。它只返回结构化问题、 - 用户确认需求以及可选的当前 job 修复副本工作区。 - """ - workspace = project_root / "workspace" / job_id - final_dir = workspace / "final" - input_dir = project_root / "input" / job_id - logs_dir = workspace / "logs" - issues: List[Dict[str, Any]] = [] - evidence: List[str] = [] - params = params or {} - stage_context = self._stage_context_from_params(stage, prefix, params) - scope_id = scope_id or self._scope_id_from_params(params) or str(stage_context.get("scope_id") or "") - - def add_issue( - code: str, - severity: str, - message: str, - recommendation: str, - *, - requires_confirmation: bool = True, - runtime_repair: bool = False, - safe_patch_candidate: bool = False, - blocks_downstream: bool = False, - evidence_items: Optional[List[str]] = None, - ) -> None: - layer = ISSUE_LAYER_BY_CODE.get(code, "report") - issues.append({ - "code": code, - "severity": severity, - "layer": layer, - "issue_layer": layer, - "message": message, - "recommendation": recommendation, - "requires_user_confirmation": requires_confirmation, - "runtime_repair": runtime_repair, - "safe_patch_candidate": safe_patch_candidate, - "blocks_downstream": blocks_downstream, - "evidence": evidence_items or [], - }) - - manifest = self._load_stage_manifest(final_dir, stage, prefix, scope_id=scope_id) - if manifest and not stage_context: - stage_context = manifest.get("stage_context") or { - "scope_id": manifest.get("scope_id", ""), - "scope_mode": manifest.get("scope_mode", ""), - "analysis_route": manifest.get("analysis_route", ""), - "member_set_id": manifest.get("member_set_id", ""), - "target_species": manifest.get("target_species", []), - "reference_species": manifest.get("reference_species", []), - "comparative_species": manifest.get("comparative_species", []), - } - if manifest: - evidence.append(f"manifest={manifest.get('_path')}") - status = manifest.get("status", "unknown") - if status not in {"success", "partial_success"}: - manifest_detail = str(manifest.get("error") or manifest.get("message") or "").strip() - log_text = self._latest_stage_log_text(logs_dir, stage, limit=6000) - log_lines = [] - if log_text: - for line in log_text.splitlines(): - stripped = line.strip() - if not stripped: - continue - if any(token in stripped for token in ["Traceback", "Error", "ERROR", "失败", "FileNotFoundError", "cannot", "Cannot", "missing", "未找到"]): - log_lines.append(stripped) - if not log_lines: - log_lines = [line.strip() for line in log_text.splitlines()[-8:] if line.strip()] - detail_items = [] - if manifest_detail: - detail_items.append(manifest_detail[:500]) - if log_lines: - detail_items.append("日志摘要: " + " | ".join(log_lines[-6:])[:900]) - add_issue( - "manifest_failed", - "blocking", - f"{stage} manifest 状态为 {status}", - "查看日志摘要定位根因；若是缺上游产物，应先补跑对应上游阶段后重试。", - blocks_downstream=True, - evidence_items=detail_items, - ) - else: - stage_dir = workspace / stage - has_intermediate = stage_dir.exists() and any(stage_dir.rglob("*")) - latest_log = self._latest_stage_log_text(logs_dir, stage, limit=8000) - root_cause_found = False - if "SameFileError" in latest_log or "are the same file" in latest_log: - add_issue( - "stage_runtime_same_file_copy", - "blocking", - f"{stage} 阶段脚本在收尾复制文件时把同一个文件复制到自身，导致运行崩溃。", - "这是阶段脚本的运行时保护缺失，不是用户输入文件或 manifest 收尾问题。应修复同文件 copy guard 后重新执行该阶段。", - requires_confirmation=False, - blocks_downstream=True, - evidence_items=[latest_log[-1600:] if latest_log else str(logs_dir)], - ) - root_cause_found = True - if "ModuleNotFoundError: No module named 'apps'" in latest_log: - add_issue( - "python_import_path_error", - "blocking", - f"{stage} 启动失败：阶段脚本无法导入项目模块 apps。", - "这是工程启动路径问题，不是用户输入文件问题。应在阶段脚本中加入项目根目录到 sys.path，或用模块方式从项目根目录执行。", - requires_confirmation=True, - safe_patch_candidate=True, - blocks_downstream=True, - evidence_items=[latest_log[-1200:]], - ) - root_cause_found = True - if "requires Gene_Family" in latest_log and "known_family" in latest_log or "family_pep requires Gene_Family" in latest_log: - add_issue( - "known_family_missing_gene_family", - "blocking", - f"{stage} 读取 config.xlsx 失败：known-family/family_pep 行缺少 Gene_Family。", - "应检查 config.xlsx 中已知家族 seed 文件行，例如 At.NAC.pep.fasta 应填写 Gene_Family=NAC、Species_Role=seed_reference、File_Scope=known_family。", - requires_confirmation=True, - blocks_downstream=True, - evidence_items=[latest_log[-1200:]], - ) - root_cause_found = True - if stage == "domain" and "Domain stage requires identify results" in latest_log: - add_issue( - "domain_scoped_identify_guard", - "blocking", - "domain 后端执行守卫要求 legacy identify manifest，但当前 scope 流程已有同 scope 的成员蛋白和系统发育输入。", - "这是后端执行层与 scope 契约不一致，不应按 manifest 缺失自动恢复；应修订 domain 后端守卫，使其接受 scope_file/同 scope phylogeny manifest 解析到的成员蛋白。", - requires_confirmation=False, - blocks_downstream=True, - evidence_items=[latest_log[-1200:]], - ) - root_cause_found = True - if stage == "kaks": - scope_data = self._scope_from_params(params) - ref_species = params.get("referenceSpecies") or params.get("reference_species") or params.get("refSpeciesList") or [] - if isinstance(ref_species, str): - ref_species = [item.strip() for item in re.split(r"[,;，；\s]+", ref_species) if item.strip()] - if not ref_species and scope_data: - ref_species = scope_data.get("reference_species") or [] - scope_mode = str(params.get("scopeMode") or params.get("scope_mode") or (scope_data.get("scope_mode") if scope_data else "") or "") - missing_ref_failure = "KaKs scope 中没有 reference_species" in latest_log or (bool(params) and not ref_species and scope_mode in {"", "target_only"}) - if missing_ref_failure: - add_issue( - "kaks_scope_missing_reference", - "blocking", - "Ka/Ks 是 target-vs-reference 分析，但本次 scope 没有 reference_species。", - "应在规划阶段使用 target_plus_reference scope，并从 config/用户参数中带入参考物种；若用户未指定，按项目默认参考物种处理。", - requires_confirmation=False, - blocks_downstream=True, - evidence_items=[latest_log[-1200:] if latest_log else json.dumps(params, ensure_ascii=False)[:1200]], - ) - root_cause_found = True - if stage == "ortholog": - from gfa_core.ortholog_materialize import ortholog_pairs_has_data - - ortholog_dir = workspace / "ortholog" - final_pairs = final_dir / f"{prefix}.ortholog_pairs.txt" if prefix else None - intermediate_pairs = sorted(ortholog_dir.glob("*_pairs.txt")) if ortholog_dir.exists() else [] - intermediate_with_data = [path for path in intermediate_pairs if ortholog_pairs_has_data(path)] - final_missing = ( - final_pairs is None - or not final_pairs.exists() - or final_pairs.stat().st_size == 0 - or not ortholog_pairs_has_data(final_pairs) - ) - if intermediate_with_data and final_missing: - add_issue( - "ortholog_intermediate_not_materialized", - "blocking", - "Ortholog 已在 ortholog/ 生成 BLAST 中间产物，但 final/ 映射文件与 manifest 缺失。", - "复用现有 intermediate pairs 执行 ortholog 收尾落盘（无需重跑 BLAST），完成后再运行 kaks/ppi。", - requires_confirmation=False, - runtime_repair=True, - blocks_downstream=True, - evidence_items=[str(path) for path in intermediate_with_data[:5]], - ) - root_cause_found = True - if "Ortholog post-processing failed" in latest_log: - add_issue( - "ortholog_post_processing_failed", - "blocking", - "Ortholog BLAST 已完成，但 backend 收尾落盘失败。", - "检查日志中的 post-processing 异常；可重跑 ortholog 阶段，系统会复用 ortholog/ 现有 pairs。", - blocks_downstream=True, - evidence_items=[latest_log[-1200:] if latest_log else str(logs_dir)], - ) - root_cause_found = True - if stage == "expression": - gene_id_candidates = ( - list(input_dir.glob("*geneID*")) - + list(input_dir.glob("*.final_members.id")) - + list(final_dir.glob("*geneID*")) - + list(final_dir.glob("*.final_members.id")) - ) - if not gene_id_candidates: - add_issue( - "expression_missing_gene_ids", - "blocking", - "Expression(使用FPKM) 已找到/可使用表达矩阵，但缺少目标基因列表。", - "请先运行 Identify 生成 *.final_members.id，或上传要绘图的基因列表文件（*geneID* 或 *.final_members.id）。不要把全基因组 FPKM 矩阵直接当作基因家族成员列表。", - requires_confirmation=False, - blocks_downstream=True, - evidence_items=[str(input_dir), str(final_dir)], - ) - root_cause_found = True - sample_candidates = list(input_dir.glob("sample.txt")) + list(final_dir.glob("sample.txt")) - if not sample_candidates: - add_issue( - "expression_missing_sample_txt", - "warning" if root_cause_found else "blocking", - "Expression 阶段缺少 sample.txt 分组文件。", - "当前 job 有 samples.yaml 时可转换为 sample.txt；但基因家族绘图仍必须先提供目标基因列表。", - requires_confirmation=False, - blocks_downstream=not root_cause_found, - evidence_items=[str(input_dir / "samples.yaml")], - ) - root_cause_found = True - if has_intermediate and not (stage == "ortholog" and root_cause_found): - add_issue( - "manifest_missing_stage_incomplete", - "warning", - f"未找到 {stage} 阶段 manifest，但阶段目录已有中间产物。", - "该阶段可能正在运行或未完成收尾。不要中断进程；等待阶段结束后再做最终诊断。", - requires_confirmation=False, - blocks_downstream=False, - evidence_items=[str(stage_dir)], - ) - elif not root_cause_found: - add_issue( - "manifest_missing", - "blocking", - f"未找到 {stage} 阶段 manifest", - "先确认该阶段是否已执行；若已执行但未产出 manifest，应检查阶段日志。", - requires_confirmation=False, - blocks_downstream=True, - ) - - identity_status = self._add_stage_identity_status_issue(job_id, stage, prefix, params, manifest, add_issue, gate="diagnose") - if identity_status: - evidence.append(f"stage_identity_status={json.dumps(identity_status, ensure_ascii=False)[:1200]}") - - if stage == "rnaseq_qc": - self._audit_rnaseq_qc(input_dir, final_dir, manifest, add_issue) - elif stage == "rnaseq_align": - self._audit_rnaseq_align(final_dir, manifest, add_issue) - elif stage == "rnaseq_sample_qc": - self._audit_rnaseq_sample_qc(final_dir, logs_dir, manifest, add_issue) - elif stage == "rnaseq_quant": - self._audit_rnaseq_quant(final_dir, logs_dir, manifest, add_issue) - elif stage == "rnaseq_go_enrich": - self._audit_rnaseq_go_enrich(final_dir, manifest, add_issue) - elif stage == "rnaseq_fgsea": - self._audit_rnaseq_fgsea(final_dir, logs_dir, manifest, add_issue) - elif stage == "rnaseq_wgcna": - self._audit_rnaseq_wgcna(final_dir, logs_dir, manifest, add_issue) - else: - self._audit_generic_gene_family_stage(stage, final_dir, logs_dir, manifest, add_issue, prefix) - - issues = self._prioritize_issues(self._dedupe_issues(issues)) - highest = self._highest_severity(issues) - repair_plan = self._build_repair_plan(stage, issues) - primary_issue = issues[0] if issues else None - primary_code = str((primary_issue or {}).get("code") or "") - primary_layer = str( - (primary_issue or {}).get("layer") - or (primary_issue or {}).get("issue_layer") - or ISSUE_LAYER_BY_CODE.get(primary_code, "report") - ) - block_sidecar_repairs = ( - primary_layer == "planning" - or ( - primary_layer == "artifact_status" - and primary_code in {"manifest_missing", "manifest_missing_stage_incomplete", "manifest_failed"} - ) - ) - safe_patch_candidates = [ - issue for issue in issues - if issue.get("safe_patch_candidate") and issue.get("code") in KNOWN_COPY_PATCH_CODES - ] - if block_sidecar_repairs: - safe_patch_candidates = [] - needs_confirmation = repair_plan.get("requires_user_confirmation", False) - patch_workspace = None - if prepare_patch_workspace and safe_patch_candidates: - patch_workspace = self.prepare_repair_workspace( - job_id=job_id, - stage=stage, - issue=safe_patch_candidates[0], - project_root=project_root, - ) - ppi_identity_diagnosis = self.ppi_identity_diagnosis(manifest or params) if stage == "ppi" else {} - - return { - "stage": stage, - "job_id": job_id, - "prefix": prefix, - "stage_context": stage_context, - "scope_id": scope_id, - "status": "ok" if not issues else "needs_attention", - "severity": highest, - "primary_issue": primary_issue, - "requires_user_confirmation": needs_confirmation, - "issues": issues, - "evidence": evidence, - "repair_plan": repair_plan, - "safe_patch_candidates": [issue["code"] for issue in safe_patch_candidates], - "patch_workspace": patch_workspace, - "ppi_identity_diagnosis": ppi_identity_diagnosis, - "policy": { - "auto_modify_input": False, - "auto_modify_global_code": False, - "auto_rerun_stage": False, - "patch_scope": "job_reproducibility_copy_only", - }, - } - - def diagnose_stage_gate( - self, - job_id: str, - stage: str, - prefix: str = "", - gate: str = "preflight", - project_root: Path = PROJECT_ROOT, - params: Optional[Dict[str, Any]] = None, - ) -> Dict[str, Any]: - """Global QA gate for stage preflight/postflight/failure. - - This is intentionally job-local and deterministic: it reads manifests, - logs and files, assigns issue codes, and exposes runtime repair actions. - It never edits global source code. - """ - gate = gate if gate in {"preflight", "postflight", "failure"} else "preflight" - requested_stage = stage - stage = {"genome_prepare_v2": "genome_prepare"}.get(stage, stage) - workspace = project_root / "workspace" / job_id - final_dir = workspace / "final" - input_dir = project_root / "input" / job_id - logs_dir = workspace / "logs" - issues: List[Dict[str, Any]] = [] - evidence: List[str] = [] - - def add_issue( - code: str, - severity: str, - message: str, - recommendation: str, - *, - requires_confirmation: bool = False, - runtime_repair: bool = False, - safe_patch_candidate: bool = False, - blocks_downstream: bool = False, - evidence_items: Optional[List[str]] = None, - ) -> None: - layer = ISSUE_LAYER_BY_CODE.get(code, "report") - issues.append({ - "code": code, - "severity": severity, - "layer": layer, - "issue_layer": layer, - "message": message, - "recommendation": recommendation, - "requires_user_confirmation": requires_confirmation, - "runtime_repair": runtime_repair, - "safe_patch_candidate": safe_patch_candidate, - "blocks_downstream": blocks_downstream, - "evidence": evidence_items or [], - }) - - params = params or {} - stage_context = self._stage_context_from_params(stage, prefix, params) - scope_id = self._scope_id_from_params(params) or str(stage_context.get("scope_id") or "") - manifest = self._load_stage_manifest(final_dir, stage, prefix, scope_id=scope_id) - if stage == "genome_prepare" and gate == "preflight" and self._genome_prepare_manifest_stale(input_dir, manifest): - evidence.append("genome_prepare_previous_manifest_ignored=input_updated_after_failed_manifest") - manifest = None - if manifest and not stage_context: - stage_context = manifest.get("stage_context") or { - "scope_id": manifest.get("scope_id", ""), - "scope_mode": manifest.get("scope_mode", ""), - "analysis_route": manifest.get("analysis_route", ""), - "member_set_id": manifest.get("member_set_id", ""), - "target_species": manifest.get("target_species", []), - "reference_species": manifest.get("reference_species", []), - "comparative_species": manifest.get("comparative_species", []), - } - if gate in {"postflight", "failure"}: - base = self.diagnose_job_stage(job_id, stage, prefix, project_root, scope_id=scope_id, params=params) - for issue in base.get("issues", []): - issues.append(issue) - evidence.extend(base.get("evidence", [])) - if manifest: - evidence.append(f"manifest={manifest.get('_path')}") - - identity_status = self._add_stage_identity_status_issue(job_id, stage, prefix, params, manifest, add_issue, gate=gate) - if identity_status: - evidence.append(f"stage_identity_status={json.dumps(identity_status, ensure_ascii=False)[:1200]}") - - if params.get("source") == "methodology" or params.get("methodology"): - self._gate_saved_methodology(stage, final_dir, input_dir, params, add_issue, evidence) - - if stage == "genome_prepare": - self._gate_genome_prepare(input_dir, final_dir, logs_dir, manifest, gate, add_issue) - elif stage == "identify": - self._gate_identify(input_dir, final_dir, logs_dir, prefix, params, gate, add_issue) - elif stage in {"phylogeny", "motif", "domain", "pep"}: - self._gate_protein_stage(stage, final_dir, prefix, manifest, add_issue, params) - elif stage in {"structure", "promoter", "synteny", "karyotype"}: - self._gate_gff_member_stage(stage, final_dir, prefix, manifest, add_issue, params) - elif stage == "reference_classification": - self._gate_reference_classification(input_dir, final_dir, logs_dir, prefix, params, add_issue) - elif stage == "interspecies_synteny": - self._gate_interspecies_synteny(input_dir, final_dir, add_issue) - elif stage in {"rnaseq_qc", "rnaseq_align", "rnaseq_quant", "rnaseq_de", "rnaseq_sample_qc", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_enrich", "rnaseq_fgsea", "rnaseq_wgcna"}: - # Reuse the existing RNA-seq audits for both postflight and failure. - if gate != "preflight": - if stage == "rnaseq_qc": - self._audit_rnaseq_qc(input_dir, final_dir, manifest, add_issue) - elif stage == "rnaseq_align": - self._audit_rnaseq_align(final_dir, manifest, add_issue) - elif stage == "rnaseq_sample_qc": - self._audit_rnaseq_sample_qc(final_dir, logs_dir, manifest, add_issue) - elif stage == "rnaseq_quant": - self._audit_rnaseq_quant(final_dir, logs_dir, manifest, add_issue) - elif stage == "rnaseq_de": - self._audit_rnaseq_de_auto_expression(final_dir, logs_dir, manifest, add_issue) - elif stage in {"rnaseq_go_enrich", "rnaseq_enrich"}: - self._audit_rnaseq_go_enrich(final_dir, manifest, add_issue) - elif stage == "rnaseq_fgsea": - self._audit_rnaseq_fgsea(final_dir, logs_dir, manifest, add_issue) - elif stage == "rnaseq_wgcna": - self._audit_rnaseq_wgcna(final_dir, logs_dir, manifest, add_issue) - - if gate == "failure": - self._augment_failure_with_llm( - job_id=job_id, - stage=stage, - prefix=prefix, - input_dir=input_dir, - final_dir=final_dir, - logs_dir=logs_dir, - manifest=manifest, - params=params, - issues=issues, - evidence=evidence, - add_issue=add_issue, - ) - - issues = self._prioritize_issues(self._dedupe_issues(issues)) - highest = self._highest_severity(issues) - repair_plan = self._build_repair_plan(stage, issues) - primary_issue = issues[0] if issues else None - runtime_repairs = list(dict.fromkeys( - issue["code"] for issue in issues - if issue.get("runtime_repair") or issue.get("code") in RUNTIME_REPAIR_CODES - )) - blocking = any(issue.get("severity") == "blocking" or issue.get("blocks_downstream") for issue in issues) - ppi_identity_diagnosis = self.ppi_identity_diagnosis(manifest or params) if stage == "ppi" else {} - return { - "stage": stage, - "requested_stage": requested_stage, - "job_id": job_id, - "prefix": prefix, - "stage_context": stage_context, - "scope_id": scope_id, - "gate": gate, - "status": "pass" if not blocking else "blocked", - "severity": highest, - "primary_issue": primary_issue, - "coverage": DIAGNOSIS_STAGE_COVERAGE.get(stage, {}), - "issues": issues, - "evidence": evidence, - "repair_plan": repair_plan, - "requires_user_confirmation": repair_plan.get("requires_user_confirmation", False), - "runtime_repair_candidates": runtime_repairs, - "ppi_identity_diagnosis": ppi_identity_diagnosis, - "policy": { - "auto_modify_input": True, - "auto_modify_global_code": False, - "auto_rerun_stage": gate in {"preflight", "failure"}, - "patch_scope": "job_local_runtime_repair", - }, - } - - def _gate_saved_methodology(self, stage: str, final_dir: Path, input_dir: Path, params: Dict[str, Any], add_issue, evidence: List[str]) -> None: - family = str(params.get("geneFamily") or params.get("gene_family") or "").strip() - targets = [str(x).strip() for x in (params.get("targetSpecies") or params.get("target_species") or []) if str(x).strip()] - refs = [str(x).strip() for x in (params.get("referenceSpecies") or params.get("reference_species") or params.get("refSpeciesList") or []) if str(x).strip()] - if params.get("requiresConfirmedSubfamily") and stage not in {"reference_classification", "subfamily_confirm"}: - confirmed = [] - if final_dir.exists(): - for path in final_dir.glob("*confirmed_subfamily.tsv"): - name = path.name - if family and family not in name: - continue - if targets and not any(target in name for target in targets): - continue - if path.stat().st_size > 0: - confirmed.append(str(path)) - if not confirmed: - add_issue( - "methodology_confirmed_subfamily_missing", - "blocking", - "保存方法学复现链路要求先完成 confirmed subfamily，再执行确认后常规分析。", - "先执行 reference_classification 并在亚家族确认弹窗保存 confirmed_subfamily.tsv。", - blocks_downstream=True, - evidence_items=["missing=confirmed_subfamily.tsv", "missing=subfamily_confirm.manifest.json"], - ) - else: - evidence.append("methodology_confirmed_subfamily=" + confirmed[0]) - if stage == "expression" and (params.get("source") == "methodology" or params.get("methodology")): - matrix_files = [] - if input_dir.exists(): - for pattern in ("*TPM*", "*tpm*", "*FPKM*", "*fpkm*", "*expression*matrix*", "*expr*matrix*"): - matrix_files.extend(str(p) for p in input_dir.rglob(pattern) if p.is_file() and p.name != "config.xlsx") - geo_id = str(params.get("geoId") or params.get("geo_id") or "").strip() - if not matrix_files and not geo_id: - add_issue( - "methodology_expression_input_missing", - "blocking", - "保存方法学记录了组织表达来源，但当前 job 未提供表达矩阵，也未在 expression 参数中确认 GEO ID。", - "上传 TPM/FPKM/count 表达矩阵，或在 expression 弹窗中填写 GEO ID 后再执行；raw RNA-seq accession 仍需单独授权。", - blocks_downstream=True, - ) - if str(params.get("analysisRoute") or params.get("analysis_route") or "").startswith("raw_rnaseq"): - add_issue( - "methodology_raw_rnaseq_requires_user_consent", - "blocking", - "Raw RNA-seq accession 重分析需要用户明确授权，当前只作为 blocked 子流程记录。", - "获得授权并准备 FASTQ/samples.yaml 后再展开 RNA-seq 子流程。", - requires_confirmation=True, - blocks_downstream=True, - ) - if stage in {"reference_classification", "interspecies_synteny"} and not refs: - add_issue( - "methodology_reference_species_missing", - "blocking", - "保存方法学链路需要 referenceSpecies（当前 HDZIP 复现为 Si vs At），但参数中缺失参考物种。", - "在 config.xlsx 或执行参数中补齐 referenceSpecies/refSpeciesList，例如 At。", - blocks_downstream=True, - ) - - def apply_runtime_repairs( - self, - job_id: str, - stage: str, - prefix: str = "", - diagnosis: Optional[Dict[str, Any]] = None, - project_root: Path = PROJECT_ROOT, - *, - execute_heavy: bool = False, - ) -> Dict[str, Any]: - """Apply safe job-local runtime repairs, never global source patches.""" - diagnosis = diagnosis or self.diagnose_stage_gate(job_id, stage, prefix, "preflight", project_root) - issues = diagnosis.get("issues", []) - repair_plan = diagnosis.get("repair_plan") or self._build_repair_plan(stage, issues) - auto_codes = { - action.get("issue_code") - for action in repair_plan.get("actions", []) - if action.get("policy") == "auto" - } - if any(issue.get("code", "").startswith("genome_prepare_") for issue in issues) and self.is_stage_running(job_id, "genome_prepare"): - return { - "success": False, - "status": "running_stage_blocked", - "stage": stage, - "job_id": job_id, - "message": "genome_prepare 当前仍在运行，拒绝执行 job-local 修复以避免覆盖正在生成的产物。", - "results": [], - } - results = [] - extracted_species: set[str] = set() - for issue in issues: - code = issue.get("code") - if code not in auto_codes: - continue - if code == "protein_fasta_stop_markers": - results.append(self._apply_protein_stop_marker_fix(job_id, stage, prefix, issue, project_root)) - elif code == "gene_gff_id_overlap_low": - results.append(self._write_gene_gff_overlap_report(job_id, stage, prefix, issue, project_root)) - elif code == "genome_prepare_final_gff_empty": - results.append(self._repair_empty_final_gff(job_id, issue, project_root)) - elif code in {"genome_prepare_pep_looks_like_genome", "genome_prepare_cds_looks_like_genome"}: - species = str(self._issue_payload(issue).get("species") or "") - if species and species in extracted_species: - continue - results.append(self._repair_genome_prepare_extract_sequences(job_id, issue, project_root, execute_heavy=execute_heavy)) - if species: - extracted_species.add(species) - elif code in {"identify_pfam_remote_404", "pfam_remote_404", "identify_missing_hmm_or_pfam", "identify_pfam_partial_fetch", "identify_hmm_model_mismatch"}: - results.append(self._repair_identify_local_pfam_hmm(job_id, prefix, issue, project_root)) - elif code == "ncbi_family_reference_link_missing": - results.append(self._repair_ncbi_family_reference_link(job_id, issue, project_root)) - elif code in {"manifest_missing", "manifest_missing_stage_incomplete"}: - results.append(self._repair_missing_manifest_from_artifacts(job_id, stage, prefix, issue, project_root)) - elif code == "ortholog_intermediate_not_materialized": - results.append(self._repair_ortholog_materialize_from_intermediate(job_id, prefix, issue, project_root)) - changed = any(item.get("success") and item.get("changed", True) for item in results) - should_rerun = any(bool(item.get("should_rerun_stage")) for item in results) - return { - "success": all(item.get("success", False) for item in results) if results else True, - "status": "repaired" if changed else "no_repair_needed", - "stage": stage, - "job_id": job_id, - "repair_plan": repair_plan, - "results": results, - "should_rerun_stage": should_rerun, - "global_code_changed": False, - } - - def _build_repair_plan(self, stage: str, issues: List[Dict[str, Any]]) -> Dict[str, Any]: - """Normalize issue-specific fixes into a common repair contract. - - Policies: - - auto: safe job-local action; may be applied without user input. - - confirm: needs user confirmation or missing user parameter. - - report_only: diagnostic/report action only; no data mutation. - - dev_fix: global source change or tool installation required. - """ - actions: List[Dict[str, Any]] = [] - seen: set[tuple[str, str]] = set() - primary_issue = issues[0] if issues else {} - primary_code = str(primary_issue.get("code") or "") - primary_layer = str( - primary_issue.get("layer") - or primary_issue.get("issue_layer") - or ISSUE_LAYER_BY_CODE.get(primary_code, "report") - ) - sidecar_repairs_blocked = ( - primary_layer == "planning" - or ( - primary_layer == "artifact_status" - and primary_code in {"manifest_missing", "manifest_missing_stage_incomplete", "manifest_failed"} - ) - ) - for issue in issues: - code = str(issue.get("code") or "unknown") - policy, action = REPAIR_POLICY_BY_CODE.get(code, ("report_only", "review_issue")) - layer = str(issue.get("layer") or issue.get("issue_layer") or ISSUE_LAYER_BY_CODE.get(code, "report")) - if issue.get("runtime_repair") and code in RUNTIME_REPAIR_CODES and code != "gene_gff_id_overlap_low": - policy, action = REPAIR_POLICY_BY_CODE.get(code, ("auto", code)) - if issue.get("safe_patch_candidate") and code in KNOWN_COPY_PATCH_CODES and code not in RUNTIME_REPAIR_CODES: - policy, action = "dev_fix", "prepare_job_patch_then_upgrade_source" - if issue.get("requires_user_confirmation") and policy == "auto": - policy = "confirm" - if layer == "planning" or (sidecar_repairs_blocked and code != primary_code): - policy = "report_only" - key = (code, action) - if key in seen: - continue - seen.add(key) - actions.append({ - "issue_code": code, - "policy": policy, - "action": action, - "layer": layer, - "issue_layer": layer, - "stage": stage, - "severity": issue.get("severity", "warning"), - "message": issue.get("message", ""), - "recommendation": issue.get("recommendation", ""), - "safe_job_local": policy in {"auto", "report_only"}, - "requires_user_confirmation": policy == "confirm", - "global_code_change": policy == "dev_fix", - }) - policies = {action["policy"] for action in actions} - return { - "stage": stage, - "actions": actions, - "has_auto": "auto" in policies, - "has_confirm": "confirm" in policies, - "has_report_only": "report_only" in policies, - "has_dev_fix": "dev_fix" in policies, - "requires_user_confirmation": "confirm" in policies, - "auto_action_count": sum(1 for action in actions if action["policy"] == "auto"), - "confirm_action_count": sum(1 for action in actions if action["policy"] == "confirm"), - "dev_fix_count": sum(1 for action in actions if action["policy"] == "dev_fix"), - } - - def _add_stage_identity_status_issue( - self, - job_id: str, - stage: str, - prefix: str, - params: Dict[str, Any], - manifest: Optional[Dict[str, Any]], - add_issue, - *, - gate: str = "diagnose", - ) -> Optional[Dict[str, Any]]: - """Explain manifest-success/status-failure cases with the shared scope identity contract.""" - if not manifest or str(manifest.get("status") or "") not in {"success", "partial_success"}: - return None - try: - from gfa_core.mas.tools.stage_identity import resolve_stage_identity, get_stage_status_for_identity, explain_status_mismatch - - identity = resolve_stage_identity( - stage=stage, - prefix=prefix, - params={**(params or {}), "prefix": prefix}, - job_id=job_id, - ) - report = get_stage_status_for_identity(job_id=job_id, identity=identity, strict_identity=True) - if report.completed: - return report.to_dict() - evidence_items = [ - f"manifest_status={report.manifest_status}", - f"manifest_path={report.manifest_path}", - explain_status_mismatch(identity, report), - ] - if gate == "preflight": - # A successful manifest from a different scope route means the current - # request still needs to run. It must not block the run before the - # formal stage has a chance to create matching artifacts. - return report.to_dict() - add_issue( - "stage_identity_contract_mismatch", - "blocking", - f"{stage} manifest 已成功生成，但与当前 scope identity 契约不匹配。", - "检查并补齐 manifest 的 scope_id、analysis_route、member_set_id/stage_context；不要把该问题误判为未生成输出。", - requires_confirmation=True, - safe_patch_candidate=True, - blocks_downstream=True, - evidence_items=evidence_items, - ) - return report.to_dict() - except Exception as exc: - return {"status": "identity_status_unavailable", "error": str(exc)} - - def _stage_context_from_params(self, stage: str, prefix: str, params: Dict[str, Any]) -> Dict[str, Any]: - raw = params.get("stageContext") or params.get("stage_context") - if isinstance(raw, dict): - return raw - try: - from gfa_core.gene_family_stage_context import build_stage_context, scope_data_from_params - - scope_data = self._scope_from_params(params) or scope_data_from_params(prefix, params) - context = build_stage_context( - stage=stage, - prefix=prefix, - scope_data=scope_data, - scope_file=params.get("scopeFile") or params.get("scope_file") or "", - analysis_route=params.get("analysisRoute") or params.get("analysis_route") or "", - member_set_id=params.get("memberSetId") or params.get("member_set_id") or "", - ) - return context.to_dict() - except Exception: - return {} - - def prepare_repair_workspace( - self, - job_id: str, - stage: str, - issue: Dict[str, Any], - project_root: Path = PROJECT_ROOT, - ) -> Dict[str, Any]: - """为当前 job 准备修复副本目录，不修改副本内容、不执行副本。 - - 后续若用户确认，执行层可以在该副本上应用最小 patch 并用同样输入/输出契约测试。 - """ - script_name = STAGE_SCRIPT_MAP.get(stage) - if not script_name: - return {"success": False, "error": f"No script mapping for stage: {stage}"} - original_script = project_root / "apps" / "cli" / script_name - if not original_script.exists(): - return {"success": False, "error": f"Original script not found: {original_script}"} - - ts = datetime.now().strftime("%Y%m%d_%H%M%S") - patch_dir = project_root / "workspace" / job_id / "reproducibility" / "patches" / stage / ts - patch_dir.mkdir(parents=True, exist_ok=True) - patched_script = patch_dir / f"{original_script.stem}.patched{original_script.suffix}" - shutil.copy2(original_script, patched_script) - report = { - "stage": stage, - "job_id": job_id, - "created_at": datetime.now().isoformat(), - "repair_type": "contract_preserving_job_patch_workspace", - "issue": issue, - "original_script": str(original_script), - "patched_script": str(patched_script), - "input_changed": False, - "output_contract_changed": False, - "analysis_parameters_changed": False, - "global_code_changed": False, - "requires_user_approval_before_execution": True, - } - report_path = patch_dir / "patch_report.json" - report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") - return { - "success": True, - "patch_dir": str(patch_dir), - "patched_script": str(patched_script), - "patch_report": str(report_path), - } - - def apply_confirmed_fix( - self, - job_id: str, - stage: str, - prefix: str = "", - diagnosis: Optional[Dict[str, Any]] = None, - fix_data: Optional[Dict[str, Any]] = None, - project_root: Path = PROJECT_ROOT, - ) -> Dict[str, Any]: - """用户确认后应用 job 级修复副本流程。 - - 默认只准备副本和 patch 报告；只有 fix_data.execute_patch 为 True 时才执行。 - 执行时必须使用 patched script，且输入/输出路径由原命令或用户确认命令保持不变。 - """ - fix_data = fix_data or {} - diagnosis = diagnosis or self.diagnose_job_stage(job_id, stage, prefix, project_root) - action = fix_data.get("action") or "prepare_job_patch" - issues = diagnosis.get("issues", []) - issue_code = fix_data.get("issue_code") - issue = None - for item in issues: - if issue_code and item.get("code") == issue_code: - issue = item - break - if not issue_code and item.get("safe_patch_candidate") and item.get("code") in KNOWN_COPY_PATCH_CODES: - issue = item - break - if issue is None: - return { - "success": False, - "status": "blocked", - "message": "没有可安全副本修复的问题；需要用户手动处理或补充确认信息。", - "diagnosis": diagnosis, - } - - if issue.get("code") == "protein_fasta_stop_markers": - return self._apply_protein_stop_marker_fix( - job_id=job_id, - stage=stage, - prefix=prefix, - issue=issue, - project_root=project_root, - ) - - patch_ws = self.prepare_repair_workspace(job_id, stage, issue, project_root) - if not patch_ws.get("success"): - return patch_ws - - patch_result = self._apply_known_patch_to_copy( - issue_code=issue.get("code", ""), - patched_script=Path(patch_ws["patched_script"]), - ) - patch_ws["patch_application"] = patch_result - self._update_patch_report(Path(patch_ws["patch_report"]), {"patch_application": patch_result}) - - if action == "prepare_job_patch" or not fix_data.get("execute_patch", False): - return { - "success": True, - "status": "prepared", - "message": "已准备当前 job 的修复副本；尚未执行。", - "patch_workspace": patch_ws, - "requires_user_confirmation_before_execution": True, - } - - if self.is_stage_running(job_id, stage): - return { - "success": False, - "status": "running_stage_blocked", - "message": f"{job_id}/{stage} 当前仍在运行，拒绝执行修复副本以避免覆盖正在生成的产物。", - "patch_workspace": patch_ws, - } - - cmd_result = self._build_patched_command( - job_id=job_id, - stage=stage, - prefix=prefix, - patched_script=Path(patch_ws["patched_script"]), - fix_data=fix_data, - project_root=project_root, - ) - if not cmd_result.get("success"): - self._update_patch_report(Path(patch_ws["patch_report"]), {"execution": cmd_result}) - return { - "success": False, - "status": "needs_command", - "message": cmd_result.get("message", "无法构建修复副本执行命令"), - "patch_workspace": patch_ws, - } - - backup = self._backup_failed_run(job_id, stage, prefix, project_root) - execution = self._execute_patched_command( - cmd=cmd_result["cmd"], - cwd=project_root, - timeout=int(fix_data.get("timeout", 3600)), - ) - recovery = { - "used_patched_script": True, - "patched_script": patch_ws["patched_script"], - "original_script": str(project_root / "apps" / "cli" / STAGE_SCRIPT_MAP.get(stage, "")), - "patch_report": patch_ws["patch_report"], - "reason": issue.get("message"), - "issue_code": issue.get("code"), - "contract_preserved": True, - "input_changed": False, - "output_contract_changed": False, - "analysis_parameters_changed": False, - "backup": backup, - "executed_at": datetime.now().isoformat(), - } - manifest_update = self._annotate_recovery_manifest(job_id, stage, prefix, recovery, project_root) - execution_record = { - "command": cmd_result["cmd"], - "execution": execution, - "recovery": recovery, - "manifest_update": manifest_update, - } - self._update_patch_report(Path(patch_ws["patch_report"]), {"execution": execution_record}) - return { - "success": execution.get("returncode") == 0, - "status": "executed" if execution.get("returncode") == 0 else "execution_failed", - "message": "修复副本执行完成" if execution.get("returncode") == 0 else "修复副本执行失败", - "patch_workspace": patch_ws, - **execution_record, - } - - def is_stage_running(self, job_id: str, stage: str) -> bool: - """检查同一 job/stage 是否有正在运行的命令。""" - try: - proc = subprocess.run( - ["ps", "-eo", "pid,ppid,cmd"], - text=True, - capture_output=True, - check=False, - timeout=10, - ) - except Exception: - return False - script = STAGE_SCRIPT_MAP.get(stage, stage) - for line in proc.stdout.splitlines(): - if job_id in line and script in line: - if "ps -eo" not in line and "rg " not in line: - return True - return False - - def _scope_id_from_params(self, params: Dict[str, Any]) -> str: - raw = str(params.get("scope_id") or params.get("scopeId") or "").strip() - if raw: - return raw - scope_file = params.get("scopeFile") or params.get("scope_file") - if not scope_file: - return "" - try: - path = Path(str(scope_file)) - if not path.is_absolute(): - path = PROJECT_ROOT / path - data = json.loads(path.read_text(encoding="utf-8")) - return str(data.get("scope_id") or "").strip() - except Exception: - return "" - - def _scope_from_params(self, params: Dict[str, Any]) -> Dict[str, Any]: - scope_file = params.get("scopeFile") or params.get("scope_file") - if not scope_file: - return {} - try: - path = Path(str(scope_file)) - if not path.is_absolute(): - path = PROJECT_ROOT / path - data = json.loads(path.read_text(encoding="utf-8")) - return data if isinstance(data, dict) else {} - except Exception: - return {} - - def _manifest_path_value(self, raw: Any, final_dir: Path, manifest: Optional[Dict[str, Any]]) -> Optional[Path]: - if not raw: - return None - p = Path(str(raw)) - if p.is_absolute(): - return p - workspace = (manifest or {}).get("workspace_root") or (manifest or {}).get("workspace") - if workspace: - candidate = Path(str(workspace)) / p - if candidate.exists(): - return candidate - if str(p).startswith("final/"): - return final_dir / p.name - return PROJECT_ROOT / p - - def _artifact_path_from_manifest(self, manifest: Optional[Dict[str, Any]], final_dir: Path, keys: List[str]) -> Optional[Path]: - artifacts = (manifest or {}).get("artifacts", {}) - if not isinstance(artifacts, dict): - return None - for key in keys: - artifact = artifacts.get(key) - raw = None - if isinstance(artifact, dict): - raw = artifact.get("full_path") or artifact.get("path") or artifact.get("relpath_or_hint") or artifact.get("file") - elif isinstance(artifact, str): - raw = artifact - path = self._manifest_path_value(raw, final_dir, manifest) - if path and path.exists() and path.stat().st_size > 0: - return path - return None - - def _clean_fasta_candidate(self, protein_fasta: Path) -> Path: - clean_name = re.sub(r"(\.fa(?:sta)?)$", r".clean\1", protein_fasta.name, flags=re.IGNORECASE) - if clean_name == protein_fasta.name: - clean_name = protein_fasta.name + ".clean.fa" - return protein_fasta.with_name(clean_name) - - def _protein_fasta_for_prefix(self, final_dir: Path, prefix: str, manifest: Optional[Dict[str, Any]]) -> Optional[Path]: - candidates: List[Path] = [] - params = (manifest or {}).get("parameters", {}) - if isinstance(params, dict): - for key in ("protein_fasta", "protein_source", "scope_input_fasta", "proteins", "fasta"): - path = self._manifest_path_value(params.get(key), final_dir, manifest) - if path: - clean_path = self._clean_fasta_candidate(path) - if clean_path.exists() and clean_path.stat().st_mtime >= path.stat().st_mtime: - candidates.append(clean_path) - candidates.append(path) - artifacts = (manifest or {}).get("artifacts", {}) - if isinstance(artifacts, dict): - for key in ("final_members", "final_proteins", "protein_fasta", "proteins", "scope_input_fasta", "input_fasta"): - artifact = artifacts.get(key) - if isinstance(artifact, dict): - raw = artifact.get("full_path") or artifact.get("path") or artifact.get("relpath_or_hint") - if raw: - p = self._manifest_path_value(raw, final_dir, manifest) - if p: - candidates.append(p) - elif isinstance(artifact, str): - p = self._manifest_path_value(artifact, final_dir, manifest) - if p: - candidates.append(p) - if prefix: - candidates.append(final_dir / f"{prefix}.final_members.pep.clean.fa") - candidates.append(final_dir / f"{prefix}.final_members.pep.fa") - candidates.append(final_dir / f"{prefix}.final_members.pep.fasta") - candidates.extend(sorted(final_dir.glob("*final_members.pep.clean.fa"))) - candidates.extend(sorted(final_dir.glob("*final_members.pep.fa"))) - for path in candidates: - if path.exists() and path.is_file() and path.stat().st_size > 0: - return path - return None - - def _protein_stop_marker_stats(self, protein_fasta: Path) -> Dict[str, Any]: - stats = { - "protein_fasta": str(protein_fasta), - "sequence_count": 0, - "sequences_with_stop_markers": 0, - "stop_marker_count": 0, - } - current_changed = False - try: - with open(protein_fasta, "r", encoding="utf-8", errors="ignore") as handle: - for line in handle: - if line.startswith(">"): - if stats["sequence_count"] and current_changed: - stats["sequences_with_stop_markers"] += 1 - stats["sequence_count"] += 1 - current_changed = False - continue - count = line.count("*") - if count: - stats["stop_marker_count"] += count - current_changed = True - if stats["sequence_count"] and current_changed: - stats["sequences_with_stop_markers"] += 1 - except Exception as exc: - stats["error"] = str(exc) - return stats - - def _write_clean_protein_fasta(self, source: Path, clean_fasta: Path) -> Dict[str, Any]: - stats = self._protein_stop_marker_stats(source) - clean_fasta.parent.mkdir(parents=True, exist_ok=True) - with open(source, "r", encoding="utf-8", errors="ignore") as src, open(clean_fasta, "w", encoding="utf-8") as dst: - for line in src: - dst.write(line if line.startswith(">") else line.replace("*", "")) - stats.update({ - "clean_fasta": str(clean_fasta), - "created_at": datetime.now().isoformat(), - "policy": "job-local derived FASTA; original final_members.pep.fa is unchanged", - }) - clean_fasta.with_suffix(clean_fasta.suffix + ".repair.json").write_text( - json.dumps(stats, indent=2, ensure_ascii=False), - encoding="utf-8", - ) - return stats - - def _normalize_gene_id_for_match(self, value: str) -> str: - text = str(value or "").strip() - if "|" in text: - text = text.split("|")[-1] - for prefix in ("gene:", "transcript:", "CDS:", "mRNA:"): - if text.startswith(prefix): - text = text[len(prefix):] - text = re.sub(r"_[PT]\d+$", "", text) - text = re.sub(r"\.\d+$", "", text) - return text.upper() - - def _read_final_member_ids(self, final_dir: Path, prefix: str, manifest: Optional[Dict[str, Any]] = None) -> List[str]: - candidates = [] - params = (manifest or {}).get("parameters", {}) - if isinstance(params, dict): - for key in ("gene_id_file", "member_id_file"): - path = self._manifest_path_value(params.get(key), final_dir, manifest) - if path: - candidates.append(path) - artifact_path = self._artifact_path_from_manifest( - manifest, - final_dir, - ["final_members_id", "member_ids", "gene_ids"], - ) - if artifact_path: - candidates.append(artifact_path) - if prefix: - candidates.append(final_dir / f"{prefix}.final_members.id") - candidates.extend(sorted(final_dir.glob("*final_members.id"))) - for path in candidates: - if path.exists() and path.stat().st_size > 0: - return [line.strip() for line in path.read_text(encoding="utf-8", errors="ignore").splitlines() if line.strip()] - return [] - - def _scope_gff_paths(self, params: Dict[str, Any]) -> List[Path]: - scope = self._scope_from_params(params) - if not scope: - return [] - wanted = set(scope.get("selected_species") or scope.get("target_species") or []) - paths: List[Path] = [] - files = scope.get("files") if isinstance(scope.get("files"), dict) else {} - for species, by_type in files.items(): - if wanted and species not in wanted: - continue - if not isinstance(by_type, dict): - continue - for item in by_type.get("gff", []) or []: - if not isinstance(item, dict): - continue - raw = item.get("path") - if not raw: - continue - p = Path(str(raw)) - if not p.is_absolute(): - p = PROJECT_ROOT / p - if p.exists() and p.stat().st_size > 0: - paths.append(p) - return paths - - def _find_job_gff3(self, final_dir: Path, prefix: str, params: Optional[Dict[str, Any]] = None) -> Optional[Path]: - if isinstance(params, dict): - for key in ("gff3_source", "gff3", "gff_file"): - raw = params.get(key) - if raw: - path = Path(str(raw)) - if not path.is_absolute(): - path = PROJECT_ROOT / path - if path.exists() and path.stat().st_size > 0: - return path - gff3_sources = params.get("gff3_sources") - if isinstance(gff3_sources, list): - for item in gff3_sources: - if not isinstance(item, dict): - continue - raw = item.get("path") - if not raw: - continue - path = Path(str(raw)) - if not path.is_absolute(): - path = PROJECT_ROOT / path - if path.exists() and path.stat().st_size > 0: - return path - scoped_paths = self._scope_gff_paths(params or {}) - if scoped_paths: - return scoped_paths[0] - job_id = final_dir.parent.name - species = prefix.split(".", 1)[0] if prefix and "." in prefix else prefix - candidates: List[Path] = [] - if species: - candidates.extend([ - final_dir / f"{species}_final.gff3", - final_dir.parent / "input" / f"{species}_final.gff3", - PROJECT_ROOT / "input" / job_id / f"{species}_final.gff3", - ]) - candidates.extend(sorted(final_dir.glob("*.gff3"))) - candidates.extend(sorted((PROJECT_ROOT / "input" / job_id).glob("*.gff3")) if (PROJECT_ROOT / "input" / job_id).exists() else []) - for path in candidates: - if path.exists() and path.stat().st_size > 0: - return path - return None - - def _read_gff_gene_ids(self, gff3: Path, limit: int = 500000) -> set[str]: - ids: set[str] = set() - opener = gzip.open if str(gff3).lower().endswith(".gz") else open - with opener(gff3, "rt", encoding="utf-8", errors="ignore") as handle: - for idx, line in enumerate(handle): - if idx > limit: - break - if not line.strip() or line.startswith("#"): - continue - parts = line.rstrip("\n").split("\t") - if len(parts) < 9: - continue - if parts[2] not in {"gene", "mRNA", "transcript", "CDS"}: - continue - attrs = parts[8] - for item in attrs.split(";"): - if item.startswith("ID=") or item.startswith("Parent="): - raw = item.split("=", 1)[1].split(",", 1)[0] - norm = self._normalize_gene_id_for_match(raw) - if norm: - ids.add(norm) - return ids - - def _apply_protein_stop_marker_fix( - self, - job_id: str, - stage: str, - prefix: str, - issue: Dict[str, Any], - project_root: Path = PROJECT_ROOT, - ) -> Dict[str, Any]: - workspace = project_root / "workspace" / job_id - final_dir = workspace / "final" - manifest = self._load_stage_manifest(final_dir, stage, prefix) - payload = self._issue_payload(issue) - protein_fasta = Path(str(payload.get("protein_fasta"))).expanduser() if payload.get("protein_fasta") else None - if protein_fasta and not protein_fasta.is_absolute(): - protein_fasta = project_root / protein_fasta - if not protein_fasta or not protein_fasta.exists(): - protein_fasta = self._protein_fasta_for_prefix(final_dir, prefix, manifest) - if not protein_fasta: - return { - "success": False, - "status": "blocked", - "message": f"未找到可清理的蛋白 FASTA：final/{prefix}.final_members.pep.fa", - "issue": issue, - } - if protein_fasta.parent != final_dir: - clean_fasta = self._clean_fasta_candidate(protein_fasta) - else: - clean_fasta = final_dir / f"{prefix or protein_fasta.stem}.final_members.pep.clean.fa" - repair = self._write_clean_protein_fasta(protein_fasta, clean_fasta) - return { - "success": True, - "status": "prepared", - "message": "已生成去除 '*' stop marker 的 job-local 蛋白 FASTA。后续 phylogeny/motif/domain/pep 将优先使用该文件。", - "issue_code": issue.get("code"), - "repair": repair, - "input_changed": False, - "global_code_changed": False, - "requires_user_confirmation_before_execution": False, - } - - def _repair_backup_path(self, path: Path) -> Optional[Path]: - if not path.exists(): - return None - backup_dir = path.parent / "diagnosis_repair_backups" - backup_dir.mkdir(parents=True, exist_ok=True) - backup = backup_dir / f"{datetime.now():%Y%m%d_%H%M%S}_{path.name}" - shutil.copy2(path, backup) - return backup - - def _move_to_repair_backup(self, path: Path) -> Optional[Path]: - """Move an invalid input out of the way before writing a replacement. - - Moving is intentional here: sequence-repair outputs may be symlinks to - large source files. Writing through the symlink would modify the source. - """ - if not path.exists() and not path.is_symlink(): - return None - backup_dir = path.parent / "diagnosis_repair_backups" - backup_dir.mkdir(parents=True, exist_ok=True) - backup = backup_dir / f"{datetime.now():%Y%m%d_%H%M%S}_{path.name}" - shutil.move(str(path), str(backup)) - return backup - - def _issue_payload(self, issue: Dict[str, Any]) -> Dict[str, Any]: - for item in issue.get("evidence", []): - if not isinstance(item, str): - continue - text = item.strip() - if text.startswith("{") and text.endswith("}"): - try: - return json.loads(text) - except Exception: - continue - return {} - - def _write_gene_gff_overlap_report( - self, - job_id: str, - stage: str, - prefix: str, - issue: Dict[str, Any], - project_root: Path = PROJECT_ROOT, - ) -> Dict[str, Any]: - payload = self._issue_payload(issue) - workspace = project_root / "workspace" / job_id - report_dir = workspace / "final" / "diagnosis_reports" - report_dir.mkdir(parents=True, exist_ok=True) - report_path = report_dir / f"{prefix or stage}.{stage}.gene_gff_id_overlap.report.json" - report = { - "issue_code": issue.get("code"), - "stage": stage, - "prefix": prefix, - "created_at": datetime.now().isoformat(), - "diagnosis": payload or issue, - "policy": "job-local report only; no gene IDs or GFF attributes were modified", - "next_action": "Review unmatched ID examples and choose a mapping rule before changing IDs.", - } - report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8") - return { - "success": True, - "changed": False, - "status": "reported", - "issue_code": issue.get("code"), - "message": "已生成 GFF/成员 ID 重叠诊断报告；该问题不能静默改 ID。", - "report": str(report_path), - "global_code_changed": False, - } - - def _repair_identify_local_pfam_hmm( - self, - job_id: str, - prefix: str, - issue: Dict[str, Any], - project_root: Path = PROJECT_ROOT, - ) -> Dict[str, Any]: - """Prepare a job-local query HMM from config/family mapping/local Pfam resources.""" - input_dir = project_root / "input" / job_id - workspace = project_root / "workspace" / job_id - if not input_dir.exists(): - return { - "success": False, - "changed": False, - "status": "blocked", - "issue_code": issue.get("code"), - "message": f"input directory not found: {input_dir}", - } - try: - from apps.cli.run_identify import ( - _pfam_alias_to_accession, - build_required_pfam_hmm, - read_analysis_family, - read_required_pfams_from_config, - ) - except Exception as exc: - return { - "success": False, - "changed": False, - "status": "blocked", - "issue_code": issue.get("code"), - "message": f"无法导入 identify Pfam fallback 工具: {exc}", - } - - payload = self._issue_payload(issue) - required_pfams = str( - payload.get("required_pfams") - or payload.get("requiredPfams") - or payload.get("pfam") - or payload.get("pfams") - or "" - ).strip() - source = "issue_evidence" - if not required_pfams: - required_pfams = read_required_pfams_from_config(input_dir, workspace) - source = "config" - - family = "" - if not required_pfams: - parts = [part for part in str(prefix or "").split(".") if part] - if len(parts) >= 2: - family = parts[-1] - elif parts and parts[0].lower() not in {"multi", "target", "reference", "comparative"}: - family = parts[0] - if not family: - family = read_analysis_family(input_dir, workspace) - mapped = _pfam_alias_to_accession(family) if family else None - if mapped: - required_pfams = mapped - source = f"family_alias:{family}" - - if not required_pfams: - return { - "success": False, - "changed": False, - "status": "blocked", - "issue_code": issue.get("code"), - "message": "未能从 issue/config/prefix 推断 Required_Pfams；请在 config.xlsx 的 Analysis.Required_Pfams 填写 PFxxxxx，或上传 HMM。", - } - - cache_dir = input_dir / "genome_prepare" / "hmm_cache" - try: - hmm_path, queries, fetch_map = build_required_pfam_hmm(required_pfams, None, cache_dir) - except Exception as exc: - return { - "success": False, - "changed": False, - "status": "blocked", - "issue_code": issue.get("code"), - "required_pfams": required_pfams, - "source": source, - "message": f"本地 Pfam/HMM fallback 失败: {exc}", - } - - record = { - "issue_code": issue.get("code"), - "created_at": datetime.now().isoformat(), - "job_id": job_id, - "prefix": prefix, - "required_pfams": required_pfams, - "queries": queries, - "fetch_map": fetch_map, - "source": source, - "query_hmm": str(hmm_path), - "global_code_changed": False, - "next_action": "rerun identify with the same job; run_identify.py will discover this HMM cache before remote lookup.", - } - record_path = cache_dir / f"{Path(hmm_path).stem}.diagnosis_repair.json" - record_path.write_text(json.dumps(record, indent=2, ensure_ascii=False), encoding="utf-8") - return { - "success": True, - "changed": True, - "status": "prepared", - "issue_code": issue.get("code"), - "message": "已基于 config/family/Pfam 本地资源生成 identify 可用的 job-local 查询 HMM。", - "required_pfams": required_pfams, - "query_hmm": str(hmm_path), - "repair_record": str(record_path), - "input_changed": True, - "global_code_changed": False, - "should_rerun_stage": True, - "requires_user_confirmation_before_execution": False, - } - - def _repair_ncbi_family_reference_link( - self, - job_id: str, - issue: Dict[str, Any], - project_root: Path = PROJECT_ROOT, - ) -> Dict[str, Any]: - payload = self._issue_payload(issue) - protein_fasta = Path(str(payload.get("protein_fasta") or "")).expanduser() - expected = Path(str(payload.get("expected_reference_pep") or "")).expanduser() - if protein_fasta and not protein_fasta.is_absolute(): - protein_fasta = project_root / protein_fasta - if expected and not expected.is_absolute(): - expected = project_root / expected - if not protein_fasta.exists() or protein_fasta.stat().st_size == 0: - return { - "success": False, - "changed": False, - "status": "blocked", - "issue_code": issue.get("code"), - "message": f"NCBI protein FASTA not found or empty: {protein_fasta}", - } - if not expected: - parts = protein_fasta.name.split(".") - if len(parts) < 4: - return { - "success": False, - "changed": False, - "status": "blocked", - "issue_code": issue.get("code"), - "message": f"无法从文件名推断 Species/Family: {protein_fasta.name}", - } - expected = protein_fasta.parent / f"{parts[0]}.{parts[1]}.pep.fa" - expected.parent.mkdir(parents=True, exist_ok=True) - if expected.exists() and expected.stat().st_size > 0: - return { - "success": True, - "changed": False, - "status": "already_exists", - "issue_code": issue.get("code"), - "message": f"标准家族参考蛋白已存在: {expected}", - "reference_pep": str(expected), - } - try: - if expected.exists() or expected.is_symlink(): - expected.unlink() - os.symlink(protein_fasta.name, expected) - link_type = "symlink" - except Exception: - shutil.copy2(protein_fasta, expected) - link_type = "copy" - record = { - "issue_code": issue.get("code"), - "created_at": datetime.now().isoformat(), - "job_id": job_id, - "protein_fasta": str(protein_fasta), - "reference_pep": str(expected), - "link_type": link_type, - "global_code_changed": False, - } - record_path = expected.with_suffix(expected.suffix + ".diagnosis_repair.json") - record_path.write_text(json.dumps(record, indent=2, ensure_ascii=False), encoding="utf-8") - return { - "success": True, - "changed": True, - "status": "prepared", - "issue_code": issue.get("code"), - "message": f"已创建标准家族参考蛋白 {expected.name} ({link_type})。", - "protein_fasta": str(protein_fasta), - "reference_pep": str(expected), - "repair_record": str(record_path), - "input_changed": False, - "global_code_changed": False, - "requires_user_confirmation_before_execution": False, - } - - def _repair_missing_manifest_from_artifacts( - self, - job_id: str, - stage: str, - prefix: str, - issue: Dict[str, Any], - project_root: Path = PROJECT_ROOT, - ) -> Dict[str, Any]: - """Recover a minimal success manifest only when required final artifacts exist.""" - workspace = project_root / "workspace" / job_id - final_dir = workspace / "final" - final_dir.mkdir(parents=True, exist_ok=True) - manifest_stage = {"ncbi_fetch": "ncbi_fetch_sequences"}.get(stage, stage) - manifest_alias = "ncbi_fetch" if manifest_stage == "ncbi_fetch_sequences" else manifest_stage - manifest_path = final_dir / f"{prefix}.{manifest_alias}.manifest.json" if prefix else final_dir / f"{manifest_alias}.manifest.json" - if manifest_path.exists() and manifest_path.stat().st_size > 0: - return { - "success": True, - "changed": False, - "status": "already_exists", - "issue_code": issue.get("code"), - "message": f"Manifest 已存在: {manifest_path}", - "manifest_path": str(manifest_path), - } - if self.is_stage_running(job_id, manifest_stage): - return { - "success": False, - "changed": False, - "status": "running_stage_blocked", - "issue_code": issue.get("code"), - "message": f"{job_id}/{manifest_stage} 当前仍在运行，拒绝恢复 manifest。", - } - - if manifest_stage == "identify": - manifest = self._build_recovered_identify_manifest(job_id, prefix, workspace, final_dir) - elif manifest_stage == "ncbi_fetch_sequences": - manifest = self._build_recovered_ncbi_fetch_manifest(job_id, prefix, workspace, final_dir) - else: - return { - "success": False, - "changed": False, - "status": "unsupported_stage", - "issue_code": issue.get("code"), - "message": f"暂不支持 {manifest_stage} 的 manifest 自动恢复；需要先定义该阶段核心产物契约。", - } - - if not manifest: - return { - "success": False, - "changed": False, - "status": "insufficient_artifacts", - "issue_code": issue.get("code"), - "message": f"{manifest_stage} final 核心产物不足，不能安全恢复 manifest。", - } - - manifest["recovered_manifest"] = { - "recovered": True, - "issue_code": issue.get("code"), - "created_at": datetime.now().isoformat(), - "policy": "job-local manifest recovery from verified non-empty final artifacts", - "global_code_changed": False, - } - manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8") - return { - "success": True, - "changed": True, - "status": "recovered", - "issue_code": issue.get("code"), - "message": f"已根据 final 核心产物恢复 {manifest_stage} manifest。", - "manifest_path": str(manifest_path), - "input_changed": False, - "global_code_changed": False, - "requires_user_confirmation_before_execution": False, - } - - def _repair_ortholog_materialize_from_intermediate( - self, - job_id: str, - prefix: str, - issue: Dict[str, Any], - project_root: Path = PROJECT_ROOT, - ) -> Dict[str, Any]: - """Reuse ortholog/*_pairs.txt and run backend materialization without rerunning BLAST.""" - if self.is_stage_running(job_id, "ortholog"): - return { - "success": False, - "changed": False, - "status": "running_stage_blocked", - "issue_code": issue.get("code"), - "message": f"{job_id}/ortholog 当前仍在运行，拒绝执行 intermediate 落盘修复。", - } - - workspace = project_root / "workspace" / job_id - final_dir = workspace / "final" - ortholog_dir = workspace / "ortholog" - from gfa_core.ortholog_materialize import ortholog_pairs_has_data - - intermediate_pairs = [path for path in sorted(ortholog_dir.glob("*_pairs.txt")) if ortholog_pairs_has_data(path)] if ortholog_dir.exists() else [] - if not intermediate_pairs: - return { - "success": False, - "changed": False, - "status": "insufficient_artifacts", - "issue_code": issue.get("code"), - "message": "ortholog/ 下没有可复用的 *_pairs.txt，无法执行 intermediate 落盘修复。", - } - - scope_file = "" - if prefix: - scope_candidates = sorted(final_dir.glob(f"{prefix}.*.scope.json"), key=lambda p: p.stat().st_mtime, reverse=True) - for candidate in scope_candidates: - try: - scope_data = json.loads(candidate.read_text(encoding="utf-8")) - except Exception: - continue - if scope_data.get("reference_species") or scope_data.get("target_species"): - scope_file = str(candidate) - break - - try: - from apps.web.pipeline_backend import JobInfo, run_stage - - job = JobInfo( - job_id=job_id, - input_dir=str(project_root / "input" / job_id), - workspace_dir=str(workspace), - logs_dir=str(workspace / "logs"), - created_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - ) - msg, manifest_path = run_stage( - stage="ortholog", - job=job, - prefix=prefix, - scope_file=scope_file, - ) - except Exception as exc: - return { - "success": False, - "changed": False, - "status": "repair_failed", - "issue_code": issue.get("code"), - "message": f"ortholog intermediate 落盘修复失败: {exc}", - } - - if not manifest_path or not Path(manifest_path).exists() or str(msg).startswith("❌") or str(msg).startswith("⚠️"): - return { - "success": False, - "changed": False, - "status": "repair_failed", - "issue_code": issue.get("code"), - "message": msg or "ortholog intermediate 落盘修复未生成 manifest", - } - - return { - "success": True, - "changed": True, - "status": "recovered", - "issue_code": issue.get("code"), - "message": msg, - "manifest_path": str(manifest_path), - "input_changed": False, - "global_code_changed": False, - "requires_user_confirmation_before_execution": False, - "should_rerun_stage": False, - } - - def _build_recovered_identify_manifest( - self, - job_id: str, - prefix: str, - workspace: Path, - final_dir: Path, - ) -> Optional[Dict[str, Any]]: - if not prefix: - return None - pep = final_dir / f"{prefix}.final_members.pep.fa" - ids = final_dir / f"{prefix}.final_members.id" - if not pep.exists() or pep.stat().st_size == 0 or not ids.exists() or ids.stat().st_size == 0: - return None - sequence_count = self._count_fasta_records(pep) - id_count = self._count_nonempty_lines(ids) - if sequence_count <= 0 or id_count <= 0: - return None - artifacts = { - "final_members": self._manifest_artifact(workspace, pep, "fasta", artifact="final_members"), - "final_members_id": self._manifest_artifact(workspace, ids, "id_list", artifact="final_members_id"), - } - interpro = final_dir / f"{prefix}.final_members.interproscan.tsv" - interpro_status = final_dir / f"{prefix}.interproscan_domain_check.txt" - interpro_input = final_dir / f"{prefix}.final_members.interproscan.input.pep.fa" - if interpro.exists() and interpro.stat().st_size > 0: - artifacts["interproscan_domain_check"] = self._manifest_artifact(workspace, interpro, "tsv", artifact="interproscan_domain_check") - if interpro_status.exists() and interpro_status.stat().st_size > 0: - artifacts["interproscan_domain_check_status"] = self._manifest_artifact(workspace, interpro_status, "text", artifact="interproscan_domain_check_status") - if interpro_input.exists() and interpro_input.stat().st_size > 0: - artifacts["interproscan_clean_input"] = self._manifest_artifact(workspace, interpro_input, "fasta", artifact="interproscan_clean_input") - species_results = [] - for species_pep in sorted((workspace / "identify").glob("*/final/*.final_gfmembers.pep.fa")): - species = species_pep.name.split(".", 1)[0] - members = self._count_fasta_records(species_pep) - species_results.append({ - "species": species, - "file_prefix": species, - "input_file": "", - "final_members": members, - "final_pep_file": self._relpath(workspace, species_pep), - }) - artifacts[f"{species}_final_members"] = self._manifest_artifact( - workspace, - species_pep, - "fasta", - artifact=f"{species}_final_members", - extra={"species": species, "members": members}, - ) - return { - "manifest_version": "1.0", - "stage": "identify", - "status": "success", - "prefix": prefix, - "job_id": job_id, - "timestamp": datetime.now().isoformat(), - "created_at": datetime.now().isoformat(), - "completed_at": datetime.now().isoformat(), - "workspace_root": str(workspace.resolve()), - "workspace": str(final_dir.resolve()), - "species_count": len(species_results) or 1, - "species_results": species_results, - "metrics": { - "final_members": sequence_count, - "final_member_ids": id_count, - "sequence_count": sequence_count, - }, - "artifacts": artifacts, - "warnings": [ - "Manifest was recovered from final artifacts; original run parameters and elapsed time are unavailable." - ], - } - - def _build_recovered_ncbi_fetch_manifest( - self, - job_id: str, - prefix: str, - workspace: Path, - final_dir: Path, - ) -> Optional[Dict[str, Any]]: - if not prefix: - return None - protein = final_dir / f"{prefix}.ncbi_fetch.protein.fa" - cds = final_dir / f"{prefix}.ncbi_fetch.cds.fa" - mapping = final_dir / f"{prefix}.ncbi_fetch.mapping.tsv" - if not any(path.exists() and path.stat().st_size > 0 for path in [protein, cds, mapping]): - return None - artifacts: Dict[str, Any] = {} - if mapping.exists() and mapping.stat().st_size > 0: - artifacts["mapping"] = self._manifest_artifact(workspace, mapping, "tsv", artifact="mapping") - for key, suffix, typ in [ - ("unresolved", "unresolved.tsv", "tsv"), - ("ambiguous", "ambiguous.tsv", "tsv"), - ("report", "report.md", "markdown"), - ]: - path = final_dir / f"{prefix}.ncbi_fetch.{suffix}" - if path.exists() and path.stat().st_size > 0: - artifacts[key] = self._manifest_artifact(workspace, path, typ, artifact=key) - protein_records = self._count_fasta_records(protein) if protein.exists() else 0 - cds_records = self._count_fasta_records(cds) if cds.exists() else 0 - if protein.exists() and protein.stat().st_size > 0: - artifacts["protein_fasta"] = self._manifest_artifact(workspace, protein, "fasta", artifact="protein_fasta") - if cds.exists() and cds.stat().st_size > 0: - artifacts["cds_fasta"] = self._manifest_artifact(workspace, cds, "fasta", artifact="cds_fasta") - family_ref = final_dir / f"{prefix}.pep.fa" - if family_ref.exists() and family_ref.stat().st_size > 0: - artifacts["family_reference_pep"] = self._manifest_artifact(workspace, family_ref, "fasta", artifact="family_reference_pep") - status = "success" if protein_records or cds_records else "partial_success" - return { - "manifest_version": "1.0", - "stage": "ncbi_fetch_sequences", - "status": status, - "prefix": prefix, - "job_id": job_id, - "created_at": datetime.now().isoformat(), - "completed_at": datetime.now().isoformat(), - "parameters": {}, - "metrics": { - "protein_records": protein_records, - "cds_records": cds_records, - "mapping_rows": max(self._count_nonempty_lines(mapping) - 1, 0) if mapping.exists() else 0, - }, - "artifacts": artifacts, - "warnings": [ - "Manifest was recovered from final artifacts; original NCBI query parameters and elapsed time are unavailable." - ], - } - - def _manifest_artifact(self, workspace: Path, path: Path, typ: str, *, artifact: str = "", extra: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: - record = { - "artifact": artifact or path.stem, - "file": path.name, - "path": self._relpath(workspace, path), - "full_path": str(path.resolve()), - "relpath_or_hint": self._relpath(workspace, path), - "type": typ, - "size": path.stat().st_size if path.exists() else 0, - "sha256": self._sha256(path) if path.exists() else "", - } - if extra: - record.update(extra) - return record - - def _relpath(self, root: Path, path: Path) -> str: - try: - return str(path.relative_to(root)) - except Exception: - return str(path) - - def _sha256(self, path: Path) -> str: - import hashlib - - h = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - h.update(chunk) - return h.hexdigest() - - def _count_fasta_records(self, path: Path) -> int: - if not path.exists(): - return 0 - count = 0 - with path.open("r", encoding="utf-8", errors="ignore") as handle: - for line in handle: - if line.startswith(">"): - count += 1 - return count - - def _count_nonempty_lines(self, path: Path) -> int: - if not path.exists(): - return 0 - with path.open("r", encoding="utf-8", errors="ignore") as handle: - return sum(1 for line in handle if line.strip()) - - def _repair_empty_final_gff(self, job_id: str, issue: Dict[str, Any], project_root: Path = PROJECT_ROOT) -> Dict[str, Any]: - input_dir = project_root / "input" / job_id - payload_text = "\n".join(str(x) for x in issue.get("evidence", [])) - match = re.search(r"final_gff=([^\n]+)", payload_text) - final_gff = Path(match.group(1).strip()) if match else None - if not final_gff or not final_gff.exists(): - return {"success": False, "status": "blocked", "issue_code": issue.get("code"), "message": "未能定位空 final GFF。"} - species = self._species_from_path_name(final_gff) - candidates = self._candidate_source_gffs(input_dir, species) - if not candidates: - return {"success": False, "status": "blocked", "issue_code": issue.get("code"), "message": f"{species} 没有可用的非空原始 GFF。"} - source = candidates[0] - backup = self._repair_backup_path(final_gff) - shutil.copy2(source, final_gff) - record = { - "issue_code": issue.get("code"), - "species": species, - "source_gff": str(source), - "repaired_gff": str(final_gff), - "backup": str(backup) if backup else "", - "created_at": datetime.now().isoformat(), - "policy": "job-local runtime repair; global source code unchanged", - } - repair_dir = input_dir / "genome_prepare" - repair_dir.mkdir(parents=True, exist_ok=True) - (repair_dir / f"{species}.gff.repair.json").write_text( - json.dumps(record, indent=2, ensure_ascii=False), - encoding="utf-8", - ) - return {"success": True, "changed": True, "status": "repaired", "issue_code": issue.get("code"), "repair": record} - - def _repair_genome_prepare_extract_sequences( - self, - job_id: str, - issue: Dict[str, Any], - project_root: Path = PROJECT_ROOT, - *, - execute_heavy: bool = False, - ) -> Dict[str, Any]: - payload = self._issue_payload(issue) - species = str(payload.get("species") or "").strip() - if not species: - return {"success": False, "status": "blocked", "issue_code": issue.get("code"), "message": "无法从诊断证据中识别物种。"} - input_dir = project_root / "input" / job_id - genome_candidates = self._candidate_source_genomes(input_dir, species) - payload_genome = payload.get("genome") - if payload_genome: - payload_path = Path(str(payload_genome)) - genome_candidates.insert(0, payload_path if payload_path.is_absolute() else input_dir / payload_path) - genome = next((p for p in genome_candidates if p.exists() and p.stat().st_size > 0 and self._fasta_profile(p).get("looks_nucleotide")), None) - gff_candidates = self._candidate_source_gffs(input_dir, species) - payload_gff = payload.get("gff") - if payload_gff: - payload_path = Path(str(payload_gff)) - gff_candidates.insert(0, payload_path if payload_path.is_absolute() else input_dir / payload_path) - gff = next((p for p in gff_candidates if p.exists() and p.stat().st_size > 0), None) - if not genome or not gff: - return { - "success": False, - "status": "blocked", - "issue_code": issue.get("code"), - "message": f"{species} 缺少可用于 gffread 的 genome 或非空 GFF。", - "genome_candidates": [str(p) for p in genome_candidates], - "gff_candidates": [str(p) for p in gff_candidates], - } - if not execute_heavy: - return { - "success": True, - "changed": False, - "status": "ready", - "issue_code": issue.get("code"), - "message": "已确认可用 genome+GFF，可在确认后执行 gffread 提取 pep/cds。", - "species": species, - "genome": str(genome), - "gff": str(gff), - } - pep_out = input_dir / f"{species}.pep.fasta" - cds_out = input_dir / f"{species}.cds.fasta" - backups = [] - for path in (pep_out, cds_out): - backup = self._move_to_repair_backup(path) - if backup: - backups.append(str(backup)) - gffread = project_root / "software" / "gffread" - if not gffread.exists(): - gffread = Path("/home/xzq/miniconda3/envs/test4py311/bin/gffread") - cmd = [str(gffread) if gffread.exists() else "gffread", str(gff), "-g", str(genome), "-y", str(pep_out), "-x", str(cds_out)] - proc = subprocess.run(cmd, cwd=str(project_root), text=True, capture_output=True, check=False, timeout=3600) - record = { - "issue_code": issue.get("code"), - "species": species, - "cmd": cmd, - "returncode": proc.returncode, - "stdout_tail": proc.stdout[-2000:], - "stderr_tail": proc.stderr[-2000:], - "genome": str(genome), - "gff": str(gff), - "pep_out": str(pep_out), - "cds_out": str(cds_out), - "backups": backups, - "created_at": datetime.now().isoformat(), - "policy": "job-local runtime repair using gffread; global source code unchanged", - } - repair_dir = input_dir / "genome_prepare" - repair_dir.mkdir(parents=True, exist_ok=True) - (repair_dir / f"{species}.gffread_extract.repair.json").write_text( - json.dumps(record, indent=2, ensure_ascii=False), - encoding="utf-8", - ) - success = proc.returncode == 0 and pep_out.exists() and pep_out.stat().st_size > 0 and cds_out.exists() and cds_out.stat().st_size > 0 - return { - "success": success, - "changed": success, - "status": "repaired" if success else "repair_failed", - "issue_code": issue.get("code"), - "repair": record, - "should_rerun_stage": success, - } - - def _apply_known_patch_to_copy(self, issue_code: str, patched_script: Path) -> Dict[str, Any]: - """对副本应用已知低风险补丁。若全局脚本已修复，则记录 no-op。""" - text = patched_script.read_text(encoding="utf-8") - original = text - notes = [] - - if issue_code == "python_import_path_error": - marker = "# DiagnosisAgent patch: ensure project root is importable" - if marker in text: - notes.append("Patch already present: project root is added to sys.path.") - else: - lines = text.splitlines() - insert_at = 0 - if lines and lines[0].startswith("#!"): - insert_at = 1 - while insert_at < len(lines) and ( - not lines[insert_at].strip() - or lines[insert_at].startswith("#") - or lines[insert_at].startswith('"""') - or lines[insert_at].startswith("'''") - ): - insert_at += 1 - if insert_at > 20: - break - patch_lines = [ - marker, - "import sys as _diagnosis_sys", - "from pathlib import Path as _DiagnosisPath", - f"_DIAGNOSIS_PROJECT_ROOT = _DiagnosisPath({str(PROJECT_ROOT)!r})", - "if str(_DIAGNOSIS_PROJECT_ROOT) not in _diagnosis_sys.path:", - " _diagnosis_sys.path.insert(0, str(_DIAGNOSIS_PROJECT_ROOT))", - "", - ] - lines[insert_at:insert_at] = patch_lines - text = "\n".join(lines) + ("\n" if original.endswith("\n") else "") - notes.append("Inserted project root sys.path guard into patched script copy.") - elif issue_code == "r_table_comment_char": - old = 'read.table(kv$count_matrix, header = TRUE, row.names = 1, sep = "\\t", check.names = FALSE)' - new = 'read.table(kv$count_matrix, header = TRUE, row.names = 1, sep = "\\t", check.names = FALSE, quote = "", comment.char = "")' - if old in text: - text = text.replace(old, new) - notes.append("Disabled R comment parsing for count matrix reads.") - elif "comment.char = \"\"" in text: - notes.append("Patch already present: comment.char is disabled.") - elif issue_code == "success_with_zero_alignment_rate": - if "Overall alignment rate:" in text and "Total pairs:" in text: - notes.append("Patch already present: HISAT2 new-summary parser is supported.") - else: - notes.append("No automatic text patch template matched; manual patch required.") - elif issue_code == "success_with_zero_qc_reads": - if "Total read pairs processed:" in text and "Pairs written (passing filters):" in text: - notes.append("Patch already present: cutadapt paired-end parser is supported.") - else: - notes.append("No automatic text patch template matched; manual patch required.") - elif issue_code == "fgsea_rank_gene2ko_id_mismatch": - if "regexec(\"^(Os[0-9]{2})t([0-9]+)(-[0-9]+)?$\"" in text: - notes.append("Patch already present: RAP transcript IDs are normalized to RAP gene IDs for fgsea.") - elif "normalize_gene_id <- function(x)" in text and "x" in text: - notes.append( - "No direct source patch applied: fgsea R code is generated inside Python; " - "apply the global RAP normalization patch to run_rnaseq_fgsea.py and regenerate the stage script." - ) - else: - notes.append("No automatic text patch template matched; manual patch required.") - elif issue_code == "wgcna_matrix_comment_char": - old = 'read.table(path, header = TRUE, row.names = 1, sep = "\\t", check.names = FALSE)' - new = 'read.table(path, header = TRUE, row.names = 1, sep = "\\t",\n check.names = FALSE, quote = "", comment.char = "")' - if old in text: - text = text.replace(old, new) - notes.append("Disabled R comment parsing for WGCNA matrix reads.") - elif 'comment.char = ""' in text and "read_matrix <- function" in text: - notes.append("Patch already present: WGCNA matrix reads disable R comment parsing.") - else: - notes.append("No automatic text patch template matched; manual patch required.") - elif issue_code == "quant_fpkm_tpm_comment_char": - notes.append( - "The known patch is in scripts/calculate_fpkm_tpm_simple.R, not the copied Python runner. " - "Ensure count/GTF reads set quote = \"\" and comment.char = \"\" before rerunning quant." - ) - else: - notes.append(f"No known automatic patch template for issue: {issue_code}") - - changed = text != original - if changed: - patched_script.write_text(text, encoding="utf-8") - return { - "success": True, - "changed": changed, - "issue_code": issue_code, - "notes": notes, - } - - def _build_patched_command( - self, - job_id: str, - stage: str, - prefix: str, - patched_script: Path, - fix_data: Dict[str, Any], - project_root: Path, - ) -> Dict[str, Any]: - cmd = fix_data.get("cmd") or fix_data.get("command") - if isinstance(cmd, str): - cmd = shlex.split(cmd) - elif not isinstance(cmd, list): - manifest = self._load_stage_manifest(project_root / "workspace" / job_id / "final", stage, prefix) - cmd = manifest.get("cmd") if manifest else None - if not isinstance(cmd, list) or not cmd: - return { - "success": False, - "message": "缺少可执行命令。请在 fix_data.cmd 中提供原阶段命令，DiagnosisAgent 会替换为 patched script。", - } - - original_name = STAGE_SCRIPT_MAP.get(stage, "") - patched_cmd = [] - replaced = False - for part in cmd: - if original_name and (str(part).endswith(original_name) or str(part).endswith(f"apps/cli/{original_name}")): - patched_cmd.append(str(patched_script)) - replaced = True - else: - patched_cmd.append(str(part)) - if not replaced and patched_script.suffix == ".py": - # Conservative fallback for commands like ["python", "-m", ...] is not safe. - return { - "success": False, - "message": "原命令中未找到阶段脚本路径，无法保证使用 patched script 执行。", - "cmd": cmd, - } - return {"success": True, "cmd": patched_cmd} - - def _execute_patched_command(self, cmd: List[str], cwd: Path, timeout: int) -> Dict[str, Any]: - try: - proc = subprocess.run( - cmd, - cwd=str(cwd), - text=True, - capture_output=True, - check=False, - timeout=timeout, - ) - return { - "returncode": proc.returncode, - "stdout_tail": proc.stdout[-4000:], - "stderr_tail": proc.stderr[-4000:], - } - except subprocess.TimeoutExpired as exc: - return { - "returncode": 124, - "stdout_tail": (exc.stdout or "")[-4000:] if isinstance(exc.stdout, str) else "", - "stderr_tail": (exc.stderr or "")[-4000:] if isinstance(exc.stderr, str) else "", - "error": f"patched command timed out after {timeout}s", - } - except Exception as exc: - return {"returncode": 1, "error": str(exc), "stdout_tail": "", "stderr_tail": ""} - - def _backup_failed_run(self, job_id: str, stage: str, prefix: str, project_root: Path) -> Dict[str, Any]: - workspace = project_root / "workspace" / job_id - backup_dir = workspace / "reproducibility" / "failed_runs" / stage / datetime.now().strftime("%Y%m%d_%H%M%S") - backup_dir.mkdir(parents=True, exist_ok=True) - copied = [] - manifest = self._load_stage_manifest(workspace / "final", stage, prefix) - if manifest and manifest.get("_path"): - src = Path(manifest["_path"]) - if src.exists(): - dst = backup_dir / src.name - shutil.copy2(src, dst) - copied.append(str(dst)) - logs_dir = workspace / "logs" - logs = sorted(logs_dir.glob(f"*_{stage}.log"), key=lambda p: p.stat().st_mtime, reverse=True) if logs_dir.exists() else [] - if logs: - dst = backup_dir / logs[0].name - shutil.copy2(logs[0], dst) - copied.append(str(dst)) - return {"backup_dir": str(backup_dir), "copied": copied} - - def _annotate_recovery_manifest( - self, - job_id: str, - stage: str, - prefix: str, - recovery: Dict[str, Any], - project_root: Path, - ) -> Dict[str, Any]: - manifest = self._load_stage_manifest(project_root / "workspace" / job_id / "final", stage, prefix) - if not manifest or not manifest.get("_path"): - return {"success": False, "message": "No manifest found to annotate."} - path = Path(manifest.pop("_path")) - manifest["recovery"] = recovery - path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8") - return {"success": True, "manifest_path": str(path)} - - def _update_patch_report(self, report_path: Path, updates: Dict[str, Any]) -> None: - try: - data = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else {} - data.update(updates) - report_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8") - except Exception: - return - - def _open_text_maybe_gzip(self, path: Path): - if str(path).lower().endswith(".gz"): - return gzip.open(path, "rt", encoding="utf-8", errors="ignore") - return open(path, "r", encoding="utf-8", errors="ignore") - - def _fasta_profile(self, path: Path, max_bases: int = 50000) -> Dict[str, Any]: - profile = { - "path": str(path), - "exists": path.exists(), - "size": path.stat().st_size if path.exists() else 0, - "headers": 0, - "letters": 0, - "nucleotide_letters": 0, - "non_nucleotide_letters": 0, - "max_record_len": 0, - "first_header": "", - } - if not path.exists() or path.stat().st_size == 0: - return profile - current_len = 0 - try: - with self._open_text_maybe_gzip(path) as handle: - for raw in handle: - if raw.startswith(">"): - if current_len > profile["max_record_len"]: - profile["max_record_len"] = current_len - current_len = 0 - profile["headers"] += 1 - if not profile["first_header"]: - profile["first_header"] = raw.strip()[:120] - continue - seq = re.sub(r"[^A-Za-z*]", "", raw) - if not seq: - continue - current_len += len(seq) - upper = seq.upper().replace("*", "") - profile["letters"] += len(upper) - profile["nucleotide_letters"] += sum(1 for ch in upper if ch in {"A", "C", "G", "T", "U", "N"}) - profile["non_nucleotide_letters"] += sum(1 for ch in upper if ch not in {"A", "C", "G", "T", "U", "N"}) - if profile["letters"] >= max_bases: - break - if current_len > profile["max_record_len"]: - profile["max_record_len"] = current_len - except Exception as exc: - profile["error"] = str(exc) - letters = max(int(profile["letters"] or 0), 1) - profile["nucleotide_ratio"] = round(float(profile["nucleotide_letters"]) / letters, 4) - profile["non_nucleotide_ratio"] = round(float(profile["non_nucleotide_letters"]) / letters, 4) - profile["looks_nucleotide"] = profile["letters"] > 0 and profile["nucleotide_ratio"] >= 0.92 - profile["looks_protein"] = profile["letters"] > 0 and profile["non_nucleotide_ratio"] >= 0.08 - profile["looks_genome_like"] = bool(profile["looks_nucleotide"] and profile["max_record_len"] >= 5000) - return profile - - def _species_from_path_name(self, path: Path) -> str: - name = path.name - if "_final.gff" in name: - return name.split("_final.gff", 1)[0] - if "." in name: - return name.split(".", 1)[0] - return path.stem - - def _species_path_rank(self, path: Path, species: str) -> Tuple[int, int, str]: - species_l = species.lower() - name_l = path.name.lower() - parent_l = path.parent.name.lower() - score = 100 - if parent_l == species_l: - score = min(score, 0) - if name_l.startswith(f"{species_l}_final."): - score = min(score, 1) - if name_l.startswith(f"{species_l}."): - score = min(score, 2) - if name_l.startswith(f"{species_l}_") or name_l.startswith(f"{species_l}-"): - score = min(score, 3) - if species_l in name_l: - score = min(score, 5) - return (score, len(path.name), str(path)) - - def _dedupe_existing_paths(self, candidates: List[Path]) -> List[Path]: - unique: List[Path] = [] - seen = set() - for path in candidates: - try: - resolved = path.resolve() - except Exception: - resolved = path - if resolved in seen or not path.exists() or path.stat().st_size == 0: - continue - seen.add(resolved) - unique.append(path) - return unique - - def _candidate_source_gffs(self, input_dir: Path, species: str) -> List[Path]: - species_candidates: List[Path] = [] - all_candidates: List[Path] = [] - for base in [input_dir / species, input_dir]: - if base.exists(): - species_candidates.extend(sorted(base.glob(f"{species}_final.gff3"))) - species_candidates.extend(sorted(base.glob(f"{species}.gff"))) - species_candidates.extend(sorted(base.glob(f"{species}.gff3"))) - species_candidates.extend(sorted(base.glob(f"{species}*.gff"))) - species_candidates.extend(sorted(base.glob(f"{species}*.gff3"))) - all_candidates.extend(sorted(base.glob("*.gff"))) - all_candidates.extend(sorted(base.glob("*.gff3"))) - species_unique = self._dedupe_existing_paths(species_candidates) - if species_unique: - return sorted(species_unique, key=lambda p: self._species_path_rank(p, species)) - all_unique = self._dedupe_existing_paths(all_candidates) - if len(all_unique) == 1: - return all_unique - return [] - - def _candidate_source_genomes(self, input_dir: Path, species: str) -> List[Path]: - candidates: List[Path] = [] - suffixes = ["fasta", "fa", "fna", "fasta.gz", "fa.gz", "fna.gz"] - exact_names = [] - for suffix in suffixes: - exact_names.extend([ - f"{species}.genome.{suffix}", - f"{species}.genomic.{suffix}", - f"{species}.dna.{suffix}", - f"{species}.{suffix}", - ]) - patterns = [] - for suffix in suffixes: - patterns.extend([ - f"{species}*genome*.{suffix}", - f"{species}*genomic*.{suffix}", - f"{species}*dna*.{suffix}", - f"{species}*Chr*.{suffix}", - f"{species}*.{suffix}", - f"*{species}*genome*.{suffix}", - f"*{species}*genomic*.{suffix}", - f"*{species}*dna*.{suffix}", - f"*{species}*Chr*.{suffix}", - ]) - for base in [input_dir / species, input_dir]: - if not base.exists(): - continue - for name in exact_names: - candidates.append(base / name) - for pattern in patterns: - candidates.extend(sorted(base.glob(pattern))) - - blocked_tokens = ("pep", "protein", "cds", "transcript", "mrna") - unique = [ - path for path in self._dedupe_existing_paths(candidates) - if not any(token in path.name.lower() for token in blocked_tokens) - ] - return sorted(unique, key=lambda p: self._species_path_rank(p, species)) - - def _gate_genome_prepare(self, input_dir: Path, final_dir: Path, logs_dir: Path, manifest: Optional[Dict[str, Any]], gate: str, add_issue) -> None: - species_files: Dict[str, Dict[str, Path]] = {} - for path in sorted(input_dir.glob("*.pep.fasta")) + sorted(input_dir.glob("*.pep.fa")): - species_files.setdefault(self._species_from_path_name(path), {})["pep"] = path - for path in sorted(input_dir.glob("*.cds.fasta")) + sorted(input_dir.glob("*.cds.fa")): - species_files.setdefault(self._species_from_path_name(path), {})["cds"] = path - for path in sorted(input_dir.glob("*_final.gff3")): - species_files.setdefault(self._species_from_path_name(path), {})["gff"] = path - for path in ( - sorted(input_dir.glob("*.genome.fasta")) - + sorted(input_dir.glob("*.genome.fa")) - + sorted(input_dir.glob("*.genome.fna")) - + sorted(input_dir.glob("*.genome.fasta.gz")) - + sorted(input_dir.glob("*.genome.fa.gz")) - + sorted(input_dir.glob("*.genome.fna.gz")) - ): - species_files.setdefault(self._species_from_path_name(path), {})["genome"] = path - - metrics = (manifest or {}).get("metrics", {}) - for result in metrics.get("species_results", []) if isinstance(metrics, dict) else []: - if not isinstance(result, dict): - continue - species = str(result.get("species") or "").strip() - files = result.get("files") if isinstance(result.get("files"), dict) else {} - artifacts = result.get("artifacts") if isinstance(result.get("artifacts"), dict) else {} - if species: - species_files.setdefault(species, {}) - for key in ("pep", "cds", "gff", "genome"): - value = files.get(key) or (artifacts.get(key, {}) if isinstance(artifacts.get(key), dict) else {}).get("full_path") - if value and species: - species_files[species][key] = Path(value) - if species and result.get("success") is False: - add_issue( - "genome_prepare_species_failed", - "blocking", - f"genome_prepare 中物种 {species} 失败：{result.get('error') or 'unknown error'}", - "阻断依赖该物种的 identify/共线性等下游阶段；先修复该物种输入或从本次分析排除。", - blocks_downstream=True, - evidence_items=[json.dumps(result, ensure_ascii=False)[:1000]], - ) - - for species, files in sorted(species_files.items()): - gff = files.get("gff") or input_dir / f"{species}_final.gff3" - if gff.exists() and gff.stat().st_size == 0: - sources = self._candidate_source_gffs(input_dir, species) - add_issue( - "genome_prepare_final_gff_empty", - "blocking", - f"{species}_final.gff3 为空，下游结构、启动子、共线性和核型图会失败。", - "优先选择同物种非空原始 GFF 作为 job-local 运行级修复；修复后重跑 genome_prepare 或阻断下游。", - runtime_repair=bool(sources), - blocks_downstream=True, - evidence_items=[ - f"final_gff={gff}", - "candidate_gffs=" + json.dumps([str(p) for p in sources[:5]], ensure_ascii=False), - ], - ) - pep = files.get("pep") - if pep and pep.exists(): - profile = self._fasta_profile(pep) - if profile.get("looks_genome_like"): - payload = {"species": species, "pep": str(pep), "profile": profile} - if files.get("genome"): - payload["genome"] = str(files["genome"]) - if files.get("gff"): - payload["gff"] = str(files["gff"]) - add_issue( - "genome_prepare_pep_looks_like_genome", - "blocking", - f"{species} pep 文件看起来像基因组/核酸序列，不像蛋白序列。", - "不能把该文件作为蛋白输入；若有 genome+GFF，可用 gffread 在 job-local 重新提取 pep。", - runtime_repair=True, - blocks_downstream=True, - evidence_items=[json.dumps(payload, ensure_ascii=False)], - ) - cds = files.get("cds") - if cds and cds.exists(): - profile = self._fasta_profile(cds) - if profile.get("looks_genome_like"): - payload = {"species": species, "cds": str(cds), "profile": profile} - if files.get("genome"): - payload["genome"] = str(files["genome"]) - if files.get("gff"): - payload["gff"] = str(files["gff"]) - add_issue( - "genome_prepare_cds_looks_like_genome", - "warning", - f"{species} cds 文件像整段基因组而不是 CDS 集合。", - "若有 genome+GFF，可用 gffread 在 job-local 重新提取 CDS；否则标记该物种 CDS 不可用于 Ka/Ks。", - runtime_repair=True, - blocks_downstream=False, - evidence_items=[json.dumps(payload, ensure_ascii=False)], - ) - - def _gate_identify(self, input_dir: Path, final_dir: Path, logs_dir: Path, prefix: str, params: Dict[str, Any], gate: str, add_issue) -> None: - genome_prepare_manifests = sorted(final_dir.glob("*.genome_prepare.manifest.json")) - for manifest_path in genome_prepare_manifests: - try: - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - except Exception: - continue - if manifest.get("status") == "partial_success": - add_issue( - "upstream_genome_prepare_partial_success", - "blocking", - "上游 genome_prepare 为 partial_success，identify 不应静默继续使用有问题物种。", - "先运行 genome_prepare postflight 诊断，修复或排除失败物种后再鉴定。", - blocks_downstream=True, - evidence_items=[str(manifest_path)], - ) - self._gate_genome_prepare(input_dir, final_dir, logs_dir, manifest, "postflight", add_issue) - has_hmm = any(input_dir.glob("*.hmm")) or any((input_dir / "genome_prepare").glob("hmm_cache/*.hmm")) if (input_dir / "genome_prepare").exists() else any(input_dir.glob("*.hmm")) - required_pfams = str(params.get("requiredPfams") or params.get("required_pfams") or "").strip() - if gate == "preflight" and not has_hmm and not required_pfams: - add_issue( - "identify_missing_hmm_or_pfam", - "blocking", - "identify 缺少 HMM 文件或 required_pfams 参数。", - "上传 HMM，或提供 Pfam 号并确保 Pfam-A.hmm 可用。", - blocks_downstream=True, - ) - if gate == "failure": - log_text = self._latest_stage_log_text(logs_dir, "identify", limit=8000) - if re.search(r"HTTPError|HTTP Error\s+404|urllib\.error\.HTTPError", log_text, re.IGNORECASE) and re.search(r"pfam|hmm|xfam|interpro", log_text, re.IGNORECASE): - add_issue( - "identify_pfam_remote_404", - "blocking", - "Identify 下载 Pfam/HMM 远程资源时遇到 HTTP 404。", - "这通常不是用户缺少 protein 输入，而是 family 名称未规范化为 PF accession、远程 Pfam URL 变化，或本地 HMM 缓存未优先使用。应优先把 SPL/SBP 等家族名映射到 PF03110，使用本地缓存 HMM；没有缓存时再访问远程。", - blocks_downstream=True, - evidence_items=[log_text[-1200:]], - ) - if gate in {"postflight", "failure"}: - for status_path in sorted(final_dir.glob("*interproscan_domain_check.txt")): - text = status_path.read_text(encoding="utf-8", errors="ignore") if status_path.exists() else "" - if "status: failed" not in text: - continue - if "Java not found" in text or "broken symbolic link" in text or "unsupported reparse tag" in text: - add_issue( - "interproscan_java_unavailable", - "warning", - "Identify 已完成，但 InterProScan domain check 因 Java PATH/坏链接失败，成员确认表缺少结构域证据。", - "修复 interproscanpy311 的 JAVA_HOME/PATH，使 java 指向 $CONDA_PREFIX/lib/jvm/bin/java；然后重跑 identify 或补跑 InterProScan/domain_review。", - blocks_downstream=False, - evidence_items=[str(status_path), text[-1200:]], - ) - break - - def _gate_protein_stage( - self, - stage: str, - final_dir: Path, - prefix: str, - manifest: Optional[Dict[str, Any]], - add_issue, - params: Optional[Dict[str, Any]] = None, - ) -> None: - protein_fasta = self._protein_fasta_for_prefix(final_dir, prefix or (manifest or {}).get("prefix", ""), manifest) - if not protein_fasta: - add_issue( - "missing_final_member_protein", - "blocking", - f"{stage} 缺少 final_members protein FASTA。", - "先完成 identify/identify_refine，或提供可用成员蛋白文件。", - blocks_downstream=True, - ) - return - profile = self._fasta_profile(protein_fasta) - if profile.get("looks_nucleotide") and not profile.get("looks_protein"): - add_issue( - "protein_input_looks_nucleotide", - "blocking", - f"{stage} 的蛋白输入看起来像核酸序列。", - "不能继续建树/MEME/domain/PEP；应回到 genome_prepare/identify 修复蛋白来源。", - blocks_downstream=True, - evidence_items=[json.dumps(profile, ensure_ascii=False)], - ) - stop_stats = self._protein_stop_marker_stats(protein_fasta) - if stop_stats.get("stop_marker_count", 0) > 0: - clean_candidate = protein_fasta.with_name(protein_fasta.name.replace(".final_members.pep.fa", ".final_members.pep.clean.fa")) - clean_stats = self._protein_stop_marker_stats(clean_candidate) if clean_candidate.exists() else {} - if ( - clean_candidate.exists() - and clean_candidate.stat().st_mtime >= protein_fasta.stat().st_mtime - and clean_stats.get("stop_marker_count", 1) == 0 - ): - return - add_issue( - "protein_fasta_stop_markers", - "warning", - f"蛋白 FASTA 含 {stop_stats['stop_marker_count']} 个 '*' stop marker。", - "可自动生成 job-local clean FASTA，原始文件不变。", - runtime_repair=True, - evidence_items=[json.dumps(stop_stats, ensure_ascii=False)], - ) - - def _gate_gff_member_stage( - self, - stage: str, - final_dir: Path, - prefix: str, - manifest: Optional[Dict[str, Any]], - add_issue, - params: Optional[Dict[str, Any]] = None, - ) -> None: - params = params or {} - if stage in {"synteny", "ppi", "pep"}: - self._gate_protein_stage(stage, final_dir, prefix, manifest, add_issue, params) - effective_prefix = prefix or (manifest or {}).get("prefix", "") - member_ids = self._read_final_member_ids(final_dir, effective_prefix, manifest) - gff3 = self._find_job_gff3(final_dir, effective_prefix, params) - if not member_ids: - add_issue("missing_final_member_ids", "blocking", f"{stage} 缺少 final_members.id。", "先完成 identify/identify_refine。", blocks_downstream=True) - return - if not gff3: - add_issue("missing_nonempty_gff", "blocking", f"{stage} 缺少非空 GFF3。", "先修复 genome_prepare 的 GFF 输出。", blocks_downstream=True) - - def _gate_interspecies_synteny(self, input_dir: Path, final_dir: Path, add_issue) -> None: - peps = [p for p in list(input_dir.glob("*.pep.fasta")) + list(input_dir.glob("*.pep.fa")) if p.exists() and p.stat().st_size > 0] - gffs = [p for p in list(input_dir.glob("*_final.gff3")) + list(input_dir.glob("*.gff3")) + list(input_dir.glob("*.gff")) if p.exists() and p.stat().st_size > 0] - if len(peps) < 2: - add_issue("interspecies_missing_whole_proteomes", "blocking", "种间共线性至少需要两个物种的全基因组蛋白 FASTA。", "检查 config.xlsx 中 whole_genome pep 行。", blocks_downstream=True) - if len(gffs) < 2: - add_issue("interspecies_missing_gffs", "blocking", "种间共线性至少需要两个物种的非空 GFF。", "检查 genome_prepare 是否产生非空 *_final.gff3。", blocks_downstream=True) - - def _gate_reference_classification( - self, - input_dir: Path, - final_dir: Path, - logs_dir: Path, - prefix: str, - params: Dict[str, Any], - add_issue, - ) -> None: - scope_id = self._scope_id_from_params(params) - analysis_route = str(params.get("analysisRoute") or params.get("analysis_route") or "").strip() - target_species = params.get("targetSpecies") or params.get("target_species") or [] - comparative_species = params.get("comparativeSpecies") or params.get("comparative_species") or [] - reference_species = params.get("referenceSpecies") or params.get("reference_species") or [] - if isinstance(target_species, str): - target_species = [item.strip() for item in re.split(r"[,;，；\s]+", target_species) if item.strip()] - if isinstance(comparative_species, str): - comparative_species = [item.strip() for item in re.split(r"[,;，；\s]+", comparative_species) if item.strip()] - if isinstance(reference_species, str): - reference_species = [item.strip() for item in re.split(r"[,;，；\s]+", reference_species) if item.strip()] - classifiable_species = [] - for item in [*target_species, *comparative_species]: - if item and item not in classifiable_species: - classifiable_species.append(item) - family_for_identity = str(params.get("geneFamily") or "").strip() - if not family_for_identity and "." in prefix: - family_for_identity = prefix.split(".")[-1] - selected_species_codes = {str(item).strip().lower() for item in [*classifiable_species, *reference_species] if str(item).strip()} - if family_for_identity and family_for_identity.lower() in selected_species_codes: - add_issue( - "scope_family_species_code_conflict", - "blocking", - f"reference_classification 的 geneFamily={family_for_identity} 与物种代码冲突。", - "请以 config.xlsx 的 Analysis.Family 作为 family，并确认 prefix 形如 Lc.SPL 而不是 Lc.At。", - blocks_downstream=True, - ) - if analysis_route == "subfamily_prediction" and not reference_species: - add_issue( - "subfamily_missing_reference_species", - "blocking", - "亚家族预测缺少 referenceSpecies，不能降级为 target_only。", - "请在 config.xlsx 中设置 seed_reference/known_family reference，或在参数中显式提供 referenceSpecies。", - blocks_downstream=True, - ) - per_target_reference_route = ( - analysis_route == "subfamily_prediction" - and bool(classifiable_species) - and bool(reference_species) - ) - if per_target_reference_route: - # In this route reference_classification owns per-species target/comparative+reference - # phylogeny/evidence generation. Requiring the parent aggregate tree here - # would reintroduce meaningless multi-species+reference plots. - phylo_manifest = None - else: - phylo_manifest = self._load_stage_manifest(final_dir, "phylogeny", prefix, scope_id=scope_id) - tree_candidates: List[Path] = [] - if phylo_manifest: - artifacts = phylo_manifest.get("artifacts") or {} - tree_artifact = artifacts.get("tree") if isinstance(artifacts, dict) else None - if isinstance(tree_artifact, dict): - for key in ("full_path", "path"): - value = tree_artifact.get(key) - if value: - p = Path(str(value)) - tree_candidates.append(p if p.is_absolute() else PROJECT_ROOT / p) - tree_candidates.append(final_dir / f"{prefix}.tree.nwk") - if (not per_target_reference_route) and not any(p.exists() and p.stat().st_size > 0 for p in tree_candidates): - latest_phylogeny_log = self._latest_stage_log_text(logs_dir, "phylogeny", limit=1200) - evidence_items = [str(p) for p in tree_candidates] - if latest_phylogeny_log: - evidence_items.append(latest_phylogeny_log[-1200:]) - add_issue( - "missing_phylogeny_tree", - "blocking", - "reference_classification 缺少同一分析范围的 phylogeny 树文件。", - "普通 reference_classification 需要同 scope 的 phylogeny；亚家族支线会按每个 target/comparative 物种分别生成 物种+reference bootstrap 树。", - blocks_downstream=True, - evidence_items=evidence_items, - ) - - reference_file = str(params.get("referenceSubfamilyFile") or params.get("reference_subfamily_file") or "").strip() - reference_prefix = str(params.get("referencePrefix") or params.get("reference_prefix") or "At").strip() or "At" - family = str(params.get("geneFamily") or "").strip() - if not family and "." in prefix: - family = prefix.split(".")[-1] - subfamily_candidates: List[Path] = [] - if reference_file: - p = Path(reference_file) - subfamily_candidates.append(p if p.is_absolute() else PROJECT_ROOT / p) - if family: - for root in (final_dir, input_dir): - subfamily_candidates.extend([ - root / f"{reference_prefix}.{family}.subfamily", - root / f"{reference_prefix}.{family}.subfamily.tsv", - root / f"{reference_prefix}.{family}.geneID.subfamily", - ]) - subfamily_candidates.extend(list(final_dir.glob("*subfamily*")) + list(input_dir.glob("*subfamily*"))) - if not any(p.exists() and p.stat().st_size > 0 for p in subfamily_candidates): - add_issue( - "missing_reference_subfamily", - "blocking", - "reference_classification 缺少参考物种亚家族标注文件。", - "请提供参考物种亚家族文件，例如 At.SPL.subfamily，或在参数中指定 referenceSubfamilyFile。", - blocks_downstream=True, - evidence_items=[str(p) for p in subfamily_candidates[:12]], - ) - - def _load_stage_manifest(self, final_dir: Path, stage: str, prefix: str = "", scope_id: str = "") -> Optional[Dict[str, Any]]: - stage_aliases = { - "ncbi_fetch_sequences": ["ncbi_fetch_sequences", "ncbi_fetch"], - "sequence_extract_by_id": ["sequence_extract_by_id", "sequence_extract"], - } - aliases = stage_aliases.get(stage, [stage]) - if scope_id and prefix: - try: - from gfa_core.gene_family_scope import sanitize_token - - scoped_token = sanitize_token(scope_id) - except Exception: - scoped_token = re.sub(r"[^A-Za-z0-9_.-]+", "-", scope_id).strip(".-_") - for alias in aliases: - path = final_dir / f"{prefix}.{scoped_token}.{alias}.manifest.json" - if path.exists(): - try: - data = json.loads(path.read_text(encoding="utf-8")) - data["_path"] = str(path) - return data - except Exception: - continue - patterns = [] - for alias in aliases: - if prefix: - patterns.append(f"{prefix}.{alias}.manifest.json") - patterns.extend([f"*.{alias}.manifest.json", f"*{alias}*.manifest.json"]) - for pattern in patterns: - for path in sorted(final_dir.glob(pattern), key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True): - try: - data = json.loads(path.read_text(encoding="utf-8")) - manifest_stage = data.get("stage") - if scope_id and str(data.get("scope_id") or "") != scope_id: - continue - if manifest_stage == stage or manifest_stage in aliases or any(alias in path.name for alias in aliases): - data["_path"] = str(path) - return data - except Exception: - continue - return None - - def _genome_prepare_manifest_stale(self, input_dir: Path, manifest: Optional[Dict[str, Any]]) -> bool: - """Ignore old failed genome_prepare manifests after config/input changes.""" - if not manifest: - return False - if str(manifest.get("status") or "").lower() == "success": - return False - raw_path = str(manifest.get("_path") or "") - if not raw_path: - return False - manifest_path = Path(raw_path) - if not manifest_path.exists(): - return False - manifest_mtime = manifest_path.stat().st_mtime - for path in input_dir.rglob("*"): - if not path.is_file(): - continue - try: - rel_parts = path.relative_to(input_dir).parts - except Exception: - rel_parts = () - if rel_parts and rel_parts[0] in {"final", "genome_prepare"}: - continue - if path.stat().st_mtime > manifest_mtime: - return True - return False - - def _audit_rnaseq_qc(self, input_dir: Path, final_dir: Path, manifest: Optional[Dict[str, Any]], add_issue) -> None: - config = self._find_rnaseq_config(input_dir) - duplicates = self._find_duplicate_fastq_pairs(config, input_dir) if config else [] - if duplicates: - add_issue( - "duplicate_fastq_pair", - "blocking", - "RNA-seq 样本配置中存在多个样本复用同一对 FASTQ。", - "暂停流程，确认样本配置。可以提供一键命名/路径修订，但必须由用户确认正确文件。", - blocks_downstream=True, - evidence_items=duplicates, - ) - - metrics = (manifest or {}).get("metrics", {}) - artifacts = (manifest or {}).get("artifacts", {}) - if (manifest or {}).get("status") == "success" and artifacts: - if metrics.get("total_clean_reads", 0) == 0 or metrics.get("total_raw_reads", 0) == 0: - add_issue( - "success_with_zero_qc_reads", - "warning", - "QC 产物存在且 manifest 为 success，但 reads 统计为 0。", - "这通常是日志解析兼容问题。可在当前 job 修复副本中修正解析器并用已有 cutadapt 日志验证。", - requires_confirmation=True, - safe_patch_candidate=True, - evidence_items=[ - f"total_raw_reads={metrics.get('total_raw_reads')}", - f"total_clean_reads={metrics.get('total_clean_reads')}", - f"artifact_count={len(artifacts)}", - ], - ) - - def _audit_rnaseq_align(self, final_dir: Path, manifest: Optional[Dict[str, Any]], add_issue) -> None: - metrics = (manifest or {}).get("metrics", {}) - bams = list(final_dir.glob("*.sorted.bam")) - logs = list(final_dir.glob("*_hisat2.log")) - if (manifest or {}).get("status") == "success" and bams and metrics.get("average_alignment_rate", 0) == 0: - add_issue( - "success_with_zero_alignment_rate", - "warning", - "比对 BAM/HISAT2 日志存在且 manifest 为 success，但平均比对率为 0。", - "这通常是 HISAT2 日志格式解析问题。可在当前 job 修复副本中修正解析器并用现有 hisat2.log 验证。", - requires_confirmation=True, - safe_patch_candidate=True, - evidence_items=[ - f"bam_count={len(bams)}", - f"hisat2_log_count={len(logs)}", - f"average_alignment_rate={metrics.get('average_alignment_rate')}", - ], - ) - - def _audit_rnaseq_sample_qc(self, final_dir: Path, logs_dir: Path, manifest: Optional[Dict[str, Any]], add_issue) -> None: - if (manifest or {}).get("status") == "failed": - error = str((manifest or {}).get("error", "")) - log_text = self._latest_stage_log_text(logs_dir, "rnaseq_sample_qc") - if "did not have" in log_text and "elements" in log_text: - add_issue( - "r_table_comment_char", - "warning", - "样本 QC 读取 count matrix 失败，R read.table 可能把基因 ID 中的 # 当成注释。", - "可在当前 job 修复副本中关闭 R 表格读取的 comment.char，并使用原 count matrix 验证。", - requires_confirmation=True, - safe_patch_candidate=True, - evidence_items=[error[:300], log_text[-500:]], - ) - - if "number of samples and the number of model coefficients are equal" in log_text or "no replicates to estimate the dispersion" in log_text: - add_issue( - "rnaseq_sample_qc_condition_scope", - "blocking", - "样本 QC 的分组 scope 可能把重复编号写进 condition，导致每个 condition 只有 1 个样本并触发 DESeq2 设计矩阵失败。", - "将 sample config 中的 condition 规范为生物条件（例如 Peel.1/Peel.2/Peel.3 -> Peel），样本 QC 使用 blind VST；差异分析前仍需人工确认真实分组。", - requires_confirmation=False, - safe_patch_candidate=True, - blocks_downstream=True, - evidence_items=[error[:300], log_text[-700:]], - ) - - corr = final_dir / f"{(manifest or {}).get('prefix', '')}.rnaseq_sample_qc.sample_correlation.tsv" - if not corr.exists(): - candidates = sorted(final_dir.glob("*.rnaseq_sample_qc.sample_correlation.tsv")) - corr = candidates[0] if candidates else corr - high_pairs = self._find_high_cross_sample_correlations(corr) - if high_pairs: - add_issue( - "near_duplicate_samples", - "blocking", - "样本相关性矩阵发现近乎重复样本，可能是样本复用、标签错误或重复上传。", - "暂停下游解释，人工确认样本配置和 FASTQ 来源；不能由 agent 自动改样本语义。", - blocks_downstream=True, - evidence_items=high_pairs[:10], - ) - - def _audit_rnaseq_quant(self, final_dir: Path, logs_dir: Path, manifest: Optional[Dict[str, Any]], add_issue) -> None: - log_text = self._latest_stage_log_text(logs_dir, "rnaseq_quant", limit=8000) - if "DESeq2 FPKM/TPM calculation failed" not in log_text: - return - if re.search(r"line\s+\d+\s+did not have\s+\d+\s+elements", log_text, re.IGNORECASE): - add_issue( - "quant_fpkm_tpm_comment_char", - "warning", - "rnaseq_quant 主计数矩阵成功，但 FPKM/TPM 计算失败并回退，R 读取 count/GTF 时可能把基因 ID 中的 # 当成注释。", - "全局脚本应关闭 R 表格读取的 quote/comment.char；修复后用相同 count matrix 和 GTF 重新计算 FPKM/TPM。", - requires_confirmation=True, - safe_patch_candidate=True, - blocks_downstream=False, - evidence_items=[log_text[-1000:]], - ) - - def _audit_rnaseq_go_enrich(self, final_dir: Path, manifest: Optional[Dict[str, Any]], add_issue) -> None: - metrics = (manifest or {}).get("metrics", {}) - if (manifest or {}).get("status") != "success": - return - - overlap = metrics.get("go_foreground_background_overlap") - zero_overlap = False - if isinstance(overlap, dict) and overlap: - zero_overlap = any( - isinstance(item, dict) - and int(item.get("foreground_genes", 0) or 0) > 0 - and int(item.get("annotated_background_genes", 0) or 0) > 0 - and int(item.get("overlap_genes", 0) or 0) == 0 - for item in overlap.values() - ) - - if not overlap and int(metrics.get("go_enriched_terms", 0) or 0) == 0: - zero_overlap = self._go_has_zero_foreground_background_overlap(final_dir) - - if zero_overlap: - add_issue( - "go_foreground_background_id_mismatch", - "blocking", - "GO 富集前景 DE 基因与 gene2go 背景基因没有交集，通常是 gene/transcript ID 体系不一致。", - "富集脚本应自动执行安全 ID 规范化；若仍为 0 交集，应阻断下游解释并向用户展示前景/背景 ID 示例。", - requires_confirmation=True, - safe_patch_candidate=True, - blocks_downstream=True, - evidence_items=[ - f"go_enriched_terms={metrics.get('go_enriched_terms')}", - f"annotated_background_genes={metrics.get('annotated_background_genes')}", - f"overlap={json.dumps(overlap, ensure_ascii=False)[:500]}", - ], - ) - - def _audit_rnaseq_fgsea(self, final_dir: Path, logs_dir: Path, manifest: Optional[Dict[str, Any]], add_issue) -> None: - metrics = (manifest or {}).get("metrics", {}) - params = (manifest or {}).get("parameters", {}) - species_name = str(params.get("species_name", "") or "") - kegg_organism = str(params.get("kegg_organism", "") or "") - if species_name and any(ord(ch) > 127 for ch in species_name) and not kegg_organism: - add_issue( - "kegg_species_name_non_ascii", - "warning", - "fgsea 参数中的物种名含非 ASCII 字符，可能导致 KEGG organism 自动推断失败。", - "请用户确认物种名是否应为 'Oryza sativa' 或直接填写 KEGG organism 'osa'；agent 不应静默修改实验物种参数。", - requires_confirmation=True, - safe_patch_candidate=False, - blocks_downstream=False, - evidence_items=[f"species_name={species_name}", "kegg_organism is empty"], - ) - log_text = self._latest_stage_log_text(logs_dir, "rnaseq_fgsea", limit=8000) - failed = (manifest or {}).get("status") == "failed" - zero_mapping = int(metrics.get("genes_with_pathway_in_ranks", 0) or 0) == 0 - suspicious_log = "No KEGG pathway gene sets passed min/max size filters" in log_text or "fgsea failed" in str((manifest or {}).get("error", "")) - if not (failed or zero_mapping or suspicious_log): - return - - overlap = self._fgsea_de_gene2ko_overlap(final_dir) - if overlap and int(overlap.get("raw_overlap", 0)) == 0 and int(overlap.get("normalized_overlap", 0)) > 0: - add_issue( - "fgsea_rank_gene2ko_id_mismatch", - "blocking", - "fgsea 排名基因与 gene2ko 背景原始 ID 没有交集，但安全 ID 规范化后可恢复交集。", - "fgsea 脚本应自动执行与 GO/KEGG 富集一致的 ID 规范化；用户确认后可用当前 job 修复副本验证，再决定是否升级全局代码。", - requires_confirmation=True, - safe_patch_candidate=True, - blocks_downstream=True, - evidence_items=[ - f"raw_overlap={overlap.get('raw_overlap')}", - f"normalized_overlap={overlap.get('normalized_overlap')}", - f"de_file={overlap.get('de_file')}", - f"gene2ko={overlap.get('gene2ko')}", - f"foreground_examples={overlap.get('foreground_examples')}", - f"background_examples={overlap.get('background_examples')}", - ], - ) - elif failed: - add_issue( - "fgsea_failed_needs_review", - "blocking", - "fgsea 阶段失败，但未能确认是可自动修复的 ID 规范化问题。", - "展示日志尾部和输入 ID 示例，由用户确认后再进入副本修复或参数调整；不要静默重跑。", - requires_confirmation=True, - safe_patch_candidate=False, - blocks_downstream=True, - evidence_items=[str((manifest or {}).get("error", ""))[:500], log_text[-1000:]], - ) - - def _audit_rnaseq_wgcna(self, final_dir: Path, logs_dir: Path, manifest: Optional[Dict[str, Any]], add_issue) -> None: - if (manifest or {}).get("status") != "failed": - return - log_text = self._latest_stage_log_text(logs_dir, "rnaseq_wgcna", limit=8000) - if re.search(r"line\s+\d+\s+did not have\s+\d+\s+elements", log_text, re.IGNORECASE): - add_issue( - "wgcna_matrix_comment_char", - "blocking", - "WGCNA 读取表达矩阵失败，R read.table 很可能把基因 ID 中的 # 当成注释导致行列数不一致。", - "WGCNA 脚本应关闭矩阵读取的 quote/comment.char；确认后可用当前 job 修复副本验证，再决定是否升级全局代码。", - requires_confirmation=True, - safe_patch_candidate=True, - blocks_downstream=True, - evidence_items=[log_text[-1000:]], - ) - - def _audit_generic_gene_family_stage(self, stage: str, final_dir: Path, logs_dir: Path, manifest: Optional[Dict[str, Any]], add_issue, prefix: str = "") -> None: - log_text = self._latest_stage_log_text(logs_dir, stage) - if stage == "identify" and isinstance(manifest, dict): - hmm_models = manifest.get("hmm_models") if isinstance(manifest.get("hmm_models"), dict) else {} - missing_hmms = hmm_models.get("missing_requested_pfams") or [] - hmm_status = str(hmm_models.get("status") or "").lower() - if missing_hmms or hmm_status == "failed": - missing_text = ",".join(str(x) for x in missing_hmms) if missing_hmms else "unknown" - add_issue( - "identify_pfam_partial_fetch", - "blocking", - f"Identify 请求的 Pfam/HMM 模型未全部加载：{missing_text}。", - "重新构建 job-local query HMM，并验证 requested Pfams 与 loaded ACC 完全一致后再重跑 identify。", - blocks_downstream=True, - evidence_items=[json.dumps(hmm_models, ensure_ascii=False)[:1200]], - ) - text = log_text - if manifest: - try: - text = (text + "\n" + json.dumps(manifest, ensure_ascii=False))[-12000:] - except Exception: - pass - generic_patterns = [ - ("id_mismatch", r"No GFF IDs matched|No .*IDs matched|ID.*不匹配|无法匹配.*ID", "ID 匹配失败，可能是 FASTA/GFF 命名体系不一致。"), - ("plantcare_upload_timeout", r"Timeout waiting for PlantCARE result|No PlantCARE result returned|No result file found", "PlantCARE 未返回本次任务结果，可能是上传 FASTA 超过官网大小限制或外部队列/邮件失败。"), - ("plantcare_id_mismatch", r"PlantCARE result IDs do not match the extracted promoter FASTA IDs", "PlantCARE 结果文件与本次提取的启动子 FASTA ID 完全不重叠。"), - ("multi_target_single_species_stage", r"一次只能分析一个目标物种|only.*one target species|single target", "单物种阶段收到了多个 target species，需要按目标物种拆分执行。"), - ("interspecies_pair_no_hits", r"BLAST failed or produced no hits|produced no hits", "种间同源搜索没有产生有效 hits，可能是蛋白文件、压缩格式、阈值或物种距离问题。"), - ("empty_required_output", r"No successful results|produced no hits|output.*empty|结果为空", "核心输出为空，需要检查输入、ID 转换或阈值。"), - ("tool_version_format", r"line \d+ did not have|unexpected format|parse.*failed", "工具输出或输入表格式与解析器不兼容。"), - ("hmm_or_pfam_missing", r"NO_HMM_FILE_ERROR|必须上传至少一个HMM文件|required_pfams is empty|No HMM file|找不到HMM|Pfam database not found|Missing Pfam index", "基因家族鉴定缺少 HMM 模型或 Pfam 参数/数据库索引。"), - ("pfam_remote_404", r"HTTPError|HTTP Error\s+404|urllib\.error\.HTTPError", "远程 Pfam/HMM 资源访问失败，可能是 URL 变化或家族名没有规范化为 PF accession。"), - ] - for code, pattern, message in generic_patterns: - search_text = text - if (manifest or {}).get("status") == "success": - # Runtime error signatures must be backed by runtime logs. - # Successful manifests may contain descriptive strings that match - # generic failure regexes without representing a failed output. - search_text = log_text - if code == "plantcare_upload_timeout" and stage != "promoter": - continue - if code == "multi_target_single_species_stage": - # This issue must come from an actual runtime error/log line. - # Successful multi-target identify manifests legitimately contain - # artifact descriptions like "Single target species final ...". - if stage == "identify": - continue - search_text = log_text - if re.search(pattern, search_text, re.IGNORECASE): - if code == "hmm_or_pfam_missing": - recommendation = "Identify 需要二选一：上传家族 HMM 文件，或提供 required_pfams（如 PF03110）并确保 Pfam-A.hmm/索引可用。" - elif code == "pfam_remote_404": - recommendation = "优先使用本地 Pfam/HMM 缓存；把家族名/结构域别名规范化为 PF accession，例如 SPL/SBP -> PF03110，再重试。" - elif code == "id_mismatch": - recommendation = "先生成 ID 匹配报告，确认 final_members.id、蛋白 FASTA header 与 GFF3 gene/mRNA/CDS ID 的对应关系；不要静默改 ID。" - elif code == "plantcare_upload_timeout": - recommendation = "检查 promoter/analysis/gene.upstream.changeID.fasta 大小；若超过约 100 KB，应按上传字节大小分批提交 PlantCARE 并合并结果，而不是只延长 timeout。" - elif code == "plantcare_id_mismatch": - recommendation = "不要复用其他物种或其他 gene set 的 PlantCARE tab；应使用本次生成的 promoter FASTA 重新提交 PlantCARE，或上传与本次 FASTA header 匹配的 tab。" - elif code == "multi_target_single_species_stage": - recommendation = "该阶段应自动拆成每个 target species 单独执行，并写回原 multi-target scope 的聚合 manifest；若仍失败，检查 scope 拆分逻辑。" - elif code == "interspecies_pair_no_hits": - recommendation = "先确认 target/reference 全蛋白 FASTA 已物化为 plain FASTA，避免 DIAMOND/BLAST 链路直接处理 .gz 造成空结果；再检查序列有效性、evalue 阈值，必要时记录该物种对为 partial_success。" - else: - recommendation = "先展示错误点和候选修复；若属于解析兼容问题，可确认使用当前 job 修复副本验证。" - is_identify_root_cause = stage == "identify" and code in { - "hmm_or_pfam_missing", - "pfam_remote_404", - } - add_issue( - code, - "blocking" if is_identify_root_cause else "warning", - message, - recommendation, - requires_confirmation=(code == "hmm_or_pfam_missing"), - safe_patch_candidate=(code == "tool_version_format"), - blocks_downstream=is_identify_root_cause, - evidence_items=[text[-800:]], - ) - - metrics = (manifest or {}).get("metrics", {}) - artifacts = (manifest or {}).get("artifacts", {}) - effective_prefix = (manifest or {}).get("prefix") or prefix - if stage == "combo" and manifest and str(manifest.get("scope_id") or "").strip(): - try: - from gfa_core.mas.tools.stage_status import get_stage_status - - status = get_stage_status( - job_id=str(manifest.get("job_id") or final_dir.parent.name), - prefix=str(effective_prefix), - stage="combo", - workspace_root=final_dir.parent.parent, - scope_id=str(manifest.get("scope_id") or ""), - ) - scope_errors = [err for err in (status.errors or []) if "scope mismatch" in str(err).lower()] - if scope_errors: - add_issue( - "combo_artifact_scope_mismatch", - "blocking", - "Combo 组合图已生成，但产物文件名没有携带完整 scope token，结果校验无法确认它属于本次 target/reference 范围。", - "这是输出命名与 scoped 契约不一致的问题；应让 combo manifest 指向完整 scope 命名的 PDF/PNG，或同步放宽校验规则。", - requires_confirmation=False, - safe_patch_candidate=False, - blocks_downstream=True, - evidence_items=scope_errors, - ) - except Exception as exc: - add_issue( - "combo_status_audit_failed", - "warning", - "Combo 结果状态复核失败，无法确认 scoped artifact 是否满足契约。", - "检查 stage_status 工具和 combo manifest 的 scope_id/artifacts 字段。", - evidence_items=[str(exc)], - ) - if stage in {"ncbi_fetch_sequences", "ncbi_fetch"}: - self._audit_ncbi_family_reference_link(final_dir, effective_prefix, add_issue) - protein_fasta = self._protein_fasta_for_prefix(final_dir, effective_prefix, manifest) - if protein_fasta and stage in {"identify", "phylogeny", "motif", "domain", "pep", "ppi"}: - stop_stats = self._protein_stop_marker_stats(protein_fasta) - if stop_stats.get("stop_marker_count", 0) > 0: - clean_candidate = protein_fasta.with_name(protein_fasta.name.replace(".final_members.pep.fa", ".final_members.pep.clean.fa")) - clean_stats = self._protein_stop_marker_stats(clean_candidate) if clean_candidate.exists() else {} - clean_is_ready = ( - clean_candidate.exists() - and clean_candidate.stat().st_mtime >= protein_fasta.stat().st_mtime - and clean_stats.get("stop_marker_count", 1) == 0 - ) - if not clean_is_ready: - add_issue( - "protein_fasta_stop_markers", - "warning", - f"蛋白 FASTA 含有 {stop_stats['stop_marker_count']} 个 '*' stop marker，涉及 {stop_stats['sequences_with_stop_markers']} 条序列。", - "生成 job-local clean FASTA，保留原始文件不变；下游 MAFFT/MEME/hmmscan/PEP 使用 clean FASTA。", - safe_patch_candidate=True, - evidence_items=[json.dumps(stop_stats, ensure_ascii=False)], - ) - if stage in {"structure", "promoter", "synteny", "karyotype"}: - params = (manifest or {}).get("parameters", {}) - member_ids = self._read_final_member_ids(final_dir, effective_prefix, manifest) - gff3 = self._find_job_gff3(final_dir, effective_prefix, params if isinstance(params, dict) else None) - if member_ids and gff3: - member_norm = {self._normalize_gene_id_for_match(item) for item in member_ids} - gff_norm = self._read_gff_gene_ids(gff3) - overlap = member_norm & gff_norm - ratio = len(overlap) / max(len(member_norm), 1) - if ratio < 0.8: - evidence = { - "final_member_ids": len(member_norm), - "gff_ids": len(gff_norm), - "overlap": len(overlap), - "overlap_ratio": round(ratio, 4), - "gff3": str(gff3), - "example_unmatched": sorted(member_norm - gff_norm)[:10], - } - add_issue( - "gene_gff_id_overlap_low", - "warning" if overlap else "blocking", - f"final_members.id 与 GFF3 的规范化 ID 重叠率偏低：{len(overlap)}/{len(member_norm)}。", - "这通常是蛋白ID/转录本ID/基因ID体系不一致。应先确认映射规则，再决定是否生成修订后的成员ID或规范化GFF；系统不会静默改ID。", - requires_confirmation=False, - safe_patch_candidate=False, - blocks_downstream=(overlap == 0), - evidence_items=[json.dumps(evidence, ensure_ascii=False)], - ) - metric_required_stages = { - "identify", "phylogeny", "motif", "domain", "structure", "promoter", - "synteny", "interspecies_synteny", "karyotype", "ppi", "expression", "pep", - } - if stage in metric_required_stages and (manifest or {}).get("status") == "success" and artifacts and self._metrics_look_empty(metrics): - if stage == "identify" and self._identify_manifest_has_member_metrics(manifest or {}): - return - add_issue( - "success_with_empty_metrics", - "warning", - f"{stage} 产物存在且 manifest 为 success，但关键 metrics 看起来为空或为 0。", - "需要检查 manifest 统计逻辑；如果只是统计解析问题，可确认用当前 job 修复副本验证。", - safe_patch_candidate=True, - evidence_items=[f"metrics={json.dumps(metrics, ensure_ascii=False)[:500]}", f"artifact_count={len(artifacts)}"], - ) - - def _audit_ncbi_family_reference_link(self, final_dir: Path, prefix: str, add_issue) -> None: - candidates: List[Path] = [] - if prefix: - candidates.extend([ - final_dir / f"{prefix}.ncbi_fetch.protein.fa", - final_dir / f"{prefix}.ncbi_fetch.protein.fasta", - ]) - candidates.extend(sorted(final_dir.glob("*.ncbi_fetch.protein.fa"))) - candidates.extend(sorted(final_dir.glob("*.ncbi_fetch.protein.fasta"))) - seen: set[str] = set() - for protein_fa in candidates: - try: - key = str(protein_fa.resolve()) - except Exception: - key = str(protein_fa) - if key in seen or not protein_fa.exists() or protein_fa.stat().st_size == 0: - continue - seen.add(key) - parts = protein_fa.name.split(".") - if len(parts) < 4 or parts[2] != "ncbi_fetch": - continue - species, family = parts[0], parts[1] - expected = final_dir / f"{species}.{family}.pep.fa" - if expected.exists() and expected.stat().st_size > 0: - continue - add_issue( - "ncbi_family_reference_link_missing", - "warning", - f"NCBI 下载蛋白已存在，但缺少标准家族参考名 {expected.name}。", - "创建 job-local 软链接/副本，统一为 {Species}.{Family}.pep.fa，供 identify/phylogeny/reference classification 复用。", - requires_confirmation=False, - safe_patch_candidate=True, - blocks_downstream=False, - evidence_items=[json.dumps({ - "protein_fasta": str(protein_fa), - "expected_reference_pep": str(expected), - "species": species, - "family": family, - }, ensure_ascii=False)], - ) - - def _audit_rnaseq_de_auto_expression(self, final_dir: Path, logs_dir: Path, manifest: Optional[Dict[str, Any]], add_issue) -> None: - if not manifest or manifest.get("status") != "success": - return - prefix = str(manifest.get("prefix") or "").strip() - expression_manifest = final_dir / f"{prefix}.expression.manifest.json" if prefix else None - if expression_manifest is None: - matches = sorted(final_dir.glob("*.expression.manifest.json")) - expression_manifest = matches[0] if matches else None - if not manifest.get("rnaseq_scope_id"): - add_issue( - "rnaseq_context_missing", - "warning", - "rnaseq_de manifest 缺少 rnaseq_scope_id，后续表达图、富集和 WGCNA 难以稳定归属到同一 RNA-seq 实验。", - "使用当前 rnaseq context 逻辑重跑 rnaseq_de，或修复 manifest context 后再刷新状态。", - safe_patch_candidate=True, - ) - if not expression_manifest or not expression_manifest.exists(): - add_issue( - "rnaseq_auto_expression_missing", - "warning", - "rnaseq_de 已成功，但未找到自动触发生成的 Expression(使用FPKM) manifest。", - "检查 rnaseq_de 后端自动触发日志；必要时使用 run_expression.py --mode rnaseq_auto 基于真实 FPKM/TPM/VST 重跑。", - safe_patch_candidate=True, - evidence_items=[f"expected_expression_manifest={expression_manifest}"], - ) - return - try: - expr_manifest = json.loads(expression_manifest.read_text(encoding="utf-8")) - except Exception as exc: - add_issue( - "rnaseq_auto_expression_failed", - "warning", - "Expression(使用FPKM) manifest 存在但无法解析。", - "查看 expression 日志并重跑 rnaseq_auto 表达绘图。", - safe_patch_candidate=True, - evidence_items=[f"manifest={expression_manifest}", f"parse_error={exc}"], - ) - return - if expr_manifest.get("status") != "success": - add_issue( - "rnaseq_auto_expression_failed", - "warning", - "rnaseq_de 自动触发的 Expression(使用FPKM) 阶段未成功。", - "查看 expression 日志并使用完整真实 FPKM/TPM/VST 数据重跑 rnaseq_auto。", - safe_patch_candidate=True, - evidence_items=[f"manifest={expression_manifest}", f"status={expr_manifest.get('status')}", f"error={expr_manifest.get('error', '')}"], - ) - - def _go_has_zero_foreground_background_overlap(self, final_dir: Path) -> bool: - gene2go = self._find_gene2go(final_dir) - if not gene2go: - return False - background_raw, background_norm = self._read_gene2go_gene_sets(gene2go) - if not background_raw: - return False - for de_file in sorted(final_dir.glob("*.rnaseq_de.*.DE_genes.txt")): - foreground_raw, foreground_norm = self._read_de_gene_sets(de_file) - if not foreground_raw: - continue - raw_overlap = len(foreground_raw & background_raw) - normalized_overlap = len(foreground_norm & background_norm) - if raw_overlap == 0 and normalized_overlap >= 0: - return True - return False - - def _find_gene2go(self, final_dir: Path) -> Optional[Path]: - preferred = final_dir / "gene2go.txt" - if preferred.exists(): - return preferred - matches = sorted(final_dir.glob("*gene2go*.txt")) - return matches[0] if matches else None - - def _find_gene2ko(self, final_dir: Path) -> Optional[Path]: - preferred = final_dir / "gene2ko.txt" - if preferred.exists(): - return preferred - matches = sorted(final_dir.glob("*gene2ko*.txt")) - return matches[0] if matches else None - - def _fgsea_de_gene2ko_overlap(self, final_dir: Path) -> Dict[str, Any]: - gene2ko = self._find_gene2ko(final_dir) - de_files = sorted(final_dir.glob("*.rnaseq_de.*.DE_results.txt")) - if not gene2ko or not de_files: - return {} - background_raw, background_norm = self._read_gene2ko_gene_sets(gene2ko) - if not background_raw: - return {} - best: Dict[str, Any] = {} - for de_file in de_files: - foreground_raw, foreground_norm = self._read_de_gene_sets(de_file) - if not foreground_raw: - continue - raw_overlap = len(foreground_raw & background_raw) - normalized_overlap = len(foreground_norm & background_norm) - item = { - "de_file": str(de_file), - "gene2ko": str(gene2ko), - "foreground_genes": len(foreground_raw), - "background_genes": len(background_raw), - "raw_overlap": raw_overlap, - "normalized_overlap": normalized_overlap, - "foreground_examples": sorted(foreground_raw)[:5], - "background_examples": sorted(background_raw)[:5], - } - if not best or normalized_overlap > int(best.get("normalized_overlap", -1)): - best = item - if raw_overlap == 0 and normalized_overlap > 0: - return item - return best - - def _read_gene2go_gene_sets(self, path: Path) -> Tuple[set, set]: - raw_genes = set() - normalized_genes = set() - try: - for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): - if not line.strip() or line.startswith("#"): - continue - gene = re.split(r"[\t, ]+", line.strip())[0] - raw_genes.add(gene) - normalized_genes.add(self._normalize_enrichment_gene_id(gene)) - except Exception: - return set(), set() - raw_genes.discard("") - normalized_genes.discard("") - return raw_genes, normalized_genes - - def _read_gene2ko_gene_sets(self, path: Path) -> Tuple[set, set]: - raw_genes = set() - normalized_genes = set() - try: - for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): - if not line.strip() or line.startswith("#"): - continue - gene = re.split(r"[\t, ]+", line.strip())[0] - raw_genes.add(gene) - normalized_genes.add(self._normalize_enrichment_gene_id(gene)) - except Exception: - return set(), set() - raw_genes.discard("") - normalized_genes.discard("") - return raw_genes, normalized_genes - - def _read_de_gene_sets(self, path: Path) -> Tuple[set, set]: - raw_genes = set() - normalized_genes = set() - try: - for idx, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines()): - if idx == 0: - continue - if not line.strip() or line.startswith("#"): - continue - gene = line.split("\t", 1)[0].strip() - normalized = self._normalize_enrichment_gene_id(gene) - if normalized and normalized.lower() not in {"gene", "gene_id", "geneid", "id", "x"}: - raw_genes.add(gene) - normalized_genes.add(normalized) - except Exception: - return set(), set() - return raw_genes, normalized_genes - - def _normalize_enrichment_gene_id(self, gene_id: str) -> str: - value = (gene_id or "").strip().strip('"').lstrip(">") - value = re.sub(r"^[a-z][a-z0-9]{2,4}:", "", value) - value = re.split(r"\s+", value, maxsplit=1)[0] - previous = None - suffix_patterns = [ - re.compile(r"(?i)([._-](?:mrna|rna|transcript|cds|protein|pep|peptide)\d*[A-Za-z]*)$"), - re.compile(r"(?i)([._-][tp]\d+[A-Za-z]?)$"), - re.compile(r"(?i)([._-][rp][A-Z])$"), - re.compile(r"(\.\d+)$"), - ] - while value and value != previous: - previous = value - for pattern in suffix_patterns: - value = pattern.sub("", value) - match = RAP_TRANSCRIPT_RE.match(value) - if match: - return f"{match.group(1)}g{match.group(2)}" - return value - - def _find_rnaseq_config(self, input_dir: Path) -> Optional[Path]: - for pattern in ("samples*.yaml", "samples*.yml", "*sample*.yaml", "*sample*.yml"): - matches = sorted(input_dir.glob(pattern)) - if matches: - return matches[0] - return None - - def _find_duplicate_fastq_pairs(self, config_path: Path, input_dir: Path) -> List[str]: - try: - import yaml - cfg = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} - except Exception: - return [] - samples = cfg.get("samples") or {} - seen: Dict[Tuple[str, str], str] = {} - duplicates = [] - for sample, value in samples.items(): - if isinstance(value, dict): - r1 = value.get("r1") or value.get("read1") or "" - r2 = value.get("r2") or value.get("read2") or "" - elif isinstance(value, list) and len(value) >= 2: - r1, r2 = value[0], value[1] - else: - continue - p1 = str((input_dir / r1).resolve()) if r1 and not Path(str(r1)).is_absolute() else str(r1) - p2 = str((input_dir / r2).resolve()) if r2 and not Path(str(r2)).is_absolute() else str(r2) - key = (p1, p2) - if key in seen: - duplicates.append(f"{seen[key]} 与 {sample} 复用: {Path(p1).name} / {Path(p2).name}") - else: - seen[key] = str(sample) - return duplicates - - def _latest_stage_log_text(self, logs_dir: Path, stage: str, limit: int = 4000) -> str: - if not logs_dir.exists(): - return "" - logs = sorted(logs_dir.glob(f"*_{stage}.log"), key=lambda p: p.stat().st_mtime, reverse=True) - if not logs: - return "" - try: - text = logs[0].read_text(encoding="utf-8", errors="ignore") - return text[-limit:] - except Exception: - return "" - - def _find_high_cross_sample_correlations(self, path: Path, threshold: float = 0.999) -> List[str]: - if not path.exists(): - return [] - try: - lines = path.read_text(encoding="utf-8", errors="ignore").splitlines() - if not lines: - return [] - header = lines[0].split("\t")[1:] - pairs = [] - for row in lines[1:]: - parts = row.split("\t") - sample = parts[0] - for other, value in zip(header, parts[1:]): - if sample >= other: - continue - try: - corr = float(value) - except Exception: - continue - if corr >= threshold: - pairs.append(f"{sample} vs {other}: correlation={corr}") - return pairs - except Exception: - return [] - - def _metrics_look_empty(self, metrics: Any) -> bool: - if not isinstance(metrics, dict) or not metrics: - return True - numeric_values = [] - for value in metrics.values(): - if isinstance(value, (int, float)): - numeric_values.append(value) - return bool(numeric_values) and all(value == 0 for value in numeric_values) - - def _identify_manifest_has_member_metrics(self, manifest: Dict[str, Any]) -> bool: - species_results = manifest.get("species_results") - if not isinstance(species_results, list): - return False - for item in species_results: - if not isinstance(item, dict): - continue - final_members = item.get("final_members") - try: - if int(final_members) > 0: - return True - except Exception: - pass - review = item.get("domain_review") - if isinstance(review, dict): - try: - if int(review.get("default_keep_count") or 0) > 0: - return True - except Exception: - pass - return False - - def _dedupe_issues(self, issues: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - deduped: List[Dict[str, Any]] = [] - seen: set[Tuple[str, str]] = set() - for issue in issues: - key = (str(issue.get("code") or ""), str(issue.get("message") or "")) - if key in seen: - continue - seen.add(key) - deduped.append(issue) - return deduped - - def _prioritize_issues(self, issues: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Put actionable root causes before derived manifest symptoms.""" - if not issues: - return [] - root_cause_codes = { - "hmm_or_pfam_missing", - "identify_missing_hmm_or_pfam", - "pfam_remote_404", - "identify_pfam_remote_404", - "identify_pfam_partial_fetch", - "identify_hmm_model_mismatch", - "python_import_path_error", - "stage_runtime_same_file_copy", - "known_family_missing_gene_family", - "id_mismatch", - "plantcare_id_mismatch", - "multi_target_single_species_stage", - "interspecies_pair_no_hits", - "genome_prepare_final_gff_empty", - "genome_prepare_pep_looks_like_genome", - "genome_prepare_cds_looks_like_genome", - "domain_scoped_identify_guard", - "kaks_scope_missing_reference", - } - has_specific_root = any(str(issue.get("code") or "") in root_cause_codes for issue in issues) - if has_specific_root: - issues = [ - issue for issue in issues - if str(issue.get("code") or "") not in {"manifest_missing", "manifest_missing_stage_incomplete"} - ] - severity_weight = {"blocking": 0, "warning": 1, "info": 2, "ok": 3} - code_weight = { - "hmm_or_pfam_missing": 0, - "identify_missing_hmm_or_pfam": 0, - "pfam_remote_404": 1, - "identify_pfam_remote_404": 1, - "identify_pfam_partial_fetch": 1, - "identify_hmm_model_mismatch": 1, - "python_import_path_error": 2, - "stage_runtime_same_file_copy": 2, - "known_family_missing_gene_family": 3, - "id_mismatch": 4, - "plantcare_id_mismatch": 5, - "multi_target_single_species_stage": 6, - "interspecies_pair_no_hits": 7, - "domain_scoped_identify_guard": 8, - "manifest_failed": 80, - "manifest_missing": 90, - "manifest_missing_stage_incomplete": 91, - } - return sorted( - issues, - key=lambda issue: ( - ISSUE_LAYER_PRIORITY.get( - str( - issue.get("layer") - or issue.get("issue_layer") - or ISSUE_LAYER_BY_CODE.get(str(issue.get("code") or ""), "report") - ), - 50, - ), - code_weight.get(str(issue.get("code") or ""), 50), - severity_weight.get(str(issue.get("severity") or "warning"), 1), - ), - ) - - def _highest_severity(self, issues: List[Dict[str, Any]]) -> str: - order = {"ok": 0, "info": 1, "warning": 2, "blocking": 3} - highest = "ok" - for issue in issues: - sev = issue.get("severity", "warning") - if order.get(sev, 2) > order.get(highest, 0): - highest = sev - return highest - - def _augment_failure_with_llm( - self, - *, - job_id: str, - stage: str, - prefix: str, - input_dir: Path, - final_dir: Path, - logs_dir: Path, - manifest: Optional[Dict[str, Any]], - params: Dict[str, Any], - issues: List[Dict[str, Any]], - evidence: List[str], - add_issue, - ) -> None: - """Use an LLM only for failure-log interpretation, after deterministic checks. - - Deterministic gates still own file/manifest facts and safe repairs. The LLM - is a bounded classifier for unknown or under-specified runtime failures; it - cannot mark a safe patch candidate or modify data/code. - """ - if str(os.environ.get("GF_DIAGNOSIS_LLM_ENABLED", "1")).lower() in {"0", "false", "no", "off"}: - return - log_text = self._latest_stage_log_text(logs_dir, stage, limit=6000) - if len(log_text.strip()) < 40: - return - if any(str(issue.get("source") or "") == "llm" for issue in issues): - return - - deterministic_codes = [ - str(issue.get("code") or "") - for issue in issues - if issue.get("code") - ] - # If a highly specific deterministic root cause already exists, do not - # spend LLM calls. Generic manifest/output symptoms still get LLM context. - generic_codes = { - "manifest_missing", - "manifest_failed", - "manifest_missing_stage_incomplete", - "empty_required_output", - "tool_version_format", - "success_with_empty_metrics", - } - specific_blockers = [ - code for code in deterministic_codes - if code not in generic_codes - and not code.startswith("protein_fasta_stop_markers") - ] - if specific_blockers and str(os.environ.get("GF_DIAGNOSIS_LLM_FORCE", "0")).lower() not in {"1", "true", "yes", "on"}: - return - - self._last_llm_failure_skip_reason = "" - llm_issue = self._llm_failure_issue( - job_id=job_id, - stage=stage, - prefix=prefix, - log_text=log_text, - params=params, - manifest=manifest, - input_summary=self._summarize_dir_for_llm(input_dir), - final_summary=self._summarize_dir_for_llm(final_dir), - deterministic_issues=issues, - ) - if not llm_issue: - reason = str(getattr(self, "_last_llm_failure_skip_reason", "") or "").strip() - if reason: - evidence.append(f"llm_failure_diagnosis=skipped:{reason[:240]}") - return - add_issue( - llm_issue["code"], - llm_issue["severity"], - llm_issue["message"], - llm_issue["recommendation"], - requires_confirmation=True, - safe_patch_candidate=False, - blocks_downstream=llm_issue["severity"] == "blocking", - evidence_items=llm_issue.get("evidence", []), - ) - if issues: - issues[-1]["source"] = "llm" - issues[-1]["confidence"] = llm_issue.get("confidence") - issues[-1]["suspected_layer"] = llm_issue.get("suspected_layer", "unknown") - issues[-1]["suggested_rule"] = llm_issue.get("suggested_rule", {}) - evidence.append("llm_failure_diagnosis=attempted") - self._remember_llm_diagnosis_case(job_id, stage, prefix, log_text, llm_issue) - - def _summarize_dir_for_llm(self, path: Path, limit: int = 40) -> List[Dict[str, Any]]: - if not path.exists(): - return [] - rows: List[Dict[str, Any]] = [] - try: - files = sorted([p for p in path.rglob("*") if p.is_file()], key=lambda p: p.stat().st_mtime, reverse=True) - except Exception: - return [] - for item in files[:limit]: - try: - rows.append({ - "path": str(item.relative_to(path)), - "size": item.stat().st_size, - }) - except Exception: - continue - return rows - - def _llm_failure_issue( - self, - *, - job_id: str, - stage: str, - prefix: str, - log_text: str, - params: Dict[str, Any], - manifest: Optional[Dict[str, Any]], - input_summary: List[Dict[str, Any]], - final_summary: List[Dict[str, Any]], - deterministic_issues: List[Dict[str, Any]], - ) -> Optional[Dict[str, Any]]: - try: - from gfa_core.mas.config import config as mas_config - from gfa_core.mas.llm import LLMManager, LLMMessage - - llm_manager = self.llm_manager or LLMManager(config=mas_config.load_llm_config()) - manifest_digest = {} - if isinstance(manifest, dict): - manifest_digest = { - "status": manifest.get("status"), - "stage": manifest.get("stage"), - "prefix": manifest.get("prefix"), - "error": str(manifest.get("error") or manifest.get("message") or "")[:1000], - "path": manifest.get("_path"), - } - context = { - "job_id": job_id, - "stage": stage, - "prefix": prefix, - "params": {k: v for k, v in (params or {}).items() if k not in {"openai_api_key", "api_key", "password"}}, - "manifest": manifest_digest, - "deterministic_issues": [ - { - "code": item.get("code"), - "severity": item.get("severity"), - "message": item.get("message"), - } - for item in deterministic_issues[-8:] - ], - "input_files_recent": input_summary[:30], - "final_files_recent": final_summary[:30], - "log_tail": log_text[-4000:], - } - prompt = ( - "你是 bbav5 基因家族/RNA-seq 流水线的失败诊断器。" - "规则层已经做过文件、manifest 和基础日志检查；你只负责解释失败日志中规则未覆盖或不够具体的根因。" - "不要编造不存在的文件；证据必须来自给定 JSON。" - "不要提出直接修改全局代码的自动动作，只能给出候选根因和人工/工程修订建议。" - "请只返回一个 JSON 对象，字段如下：" - "root_cause_category, severity(info|warning|blocking), suspected_layer(input|parameter|environment|external_resource|code_compatibility|unknown), " - "message, recommendation, evidence(list), confidence(0-1), suggested_rule{issue_code, pattern_keywords, generalization}。\n\n" - f"诊断上下文 JSON:\n{json.dumps(context, ensure_ascii=False, indent=2)}" - ) - response = llm_manager.call( - [LLMMessage(role="user", content=prompt)], - temperature=0.1, - max_tokens=1200, - ) - parsed = self._parse_llm_json(response.content) - if not parsed: - return None - category = self._sanitize_issue_token(parsed.get("root_cause_category") or parsed.get("suggested_rule", {}).get("issue_code") or "unknown_failure") - severity = str(parsed.get("severity") or "warning").lower() - if severity not in {"info", "warning", "blocking"}: - severity = "warning" - confidence = parsed.get("confidence", 0.5) - try: - confidence = max(0.0, min(float(confidence), 1.0)) - except Exception: - confidence = 0.5 - message = str(parsed.get("message") or f"LLM 推断 {stage} 失败根因：{category}").strip()[:500] - recommendation = str(parsed.get("recommendation") or "请结合日志和产物检查该候选根因。").strip()[:800] - evidence_values = parsed.get("evidence") if isinstance(parsed.get("evidence"), list) else [] - suggested_rule = parsed.get("suggested_rule") if isinstance(parsed.get("suggested_rule"), dict) else {} - return { - "code": f"llm_{category}", - "severity": severity, - "message": message, - "recommendation": recommendation, - "evidence": [str(item)[:500] for item in evidence_values[:5]], - "confidence": confidence, - "suspected_layer": str(parsed.get("suspected_layer") or "unknown"), - "suggested_rule": suggested_rule, - } - except Exception as exc: - self._last_llm_failure_skip_reason = str(exc) - print(f"[DiagnosisAgent] LLM failure diagnosis skipped: {exc}") - return None - - def _parse_llm_json(self, text: str) -> Optional[Dict[str, Any]]: - raw = str(text or "").strip() - if not raw: - return None - if raw.startswith("```"): - raw = re.sub(r"^```(?:json)?", "", raw, flags=re.IGNORECASE).strip() - raw = re.sub(r"```$", "", raw).strip() - try: - data = json.loads(raw) - return data if isinstance(data, dict) else None - except Exception: - pass - match = re.search(r"\{.*\}", raw, re.DOTALL) - if not match: - return None - try: - data = json.loads(match.group(0)) - return data if isinstance(data, dict) else None - except Exception: - return None - - def _sanitize_issue_token(self, value: Any) -> str: - token = re.sub(r"[^A-Za-z0-9_]+", "_", str(value or "").strip().lower()).strip("_") - return token[:80] or "unknown_failure" - - def _remember_llm_diagnosis_case(self, job_id: str, stage: str, prefix: str, log_text: str, issue: Dict[str, Any]) -> None: - try: - memory_dir = PROJECT_ROOT / "memory_storage" - memory_dir.mkdir(parents=True, exist_ok=True) - signature = re.sub(r"\s+", " ", log_text[-800:]).strip()[:800] - record = { - "created_at": datetime.now().isoformat(), - "job_id": job_id, - "stage": stage, - "prefix": prefix, - "log_signature": signature, - "issue": issue, - } - path = memory_dir / "diagnosis_cases.jsonl" - with path.open("a", encoding="utf-8") as handle: - handle.write(json.dumps(record, ensure_ascii=False) + "\n") - except Exception as exc: - print(f"[DiagnosisAgent] Failed to remember LLM diagnosis case: {exc}") - - def _match_error_pattern(self, error_text: str) -> Optional[Dict]: - """匹配错误模式""" - error_lower = error_text.lower() - - priority_order = [ - "tool_not_found", - "dependency_failed", - "go_foreground_background_id_mismatch", - "memory_error", - "invalid_parameter", - "conda_env_error", - "hmm_not_found", - "blast_db_error", - "missing_input_file", - "permission_error", - "timeout_error" - ] - - for error_type in priority_order: - if error_type in self.ERROR_PATTERNS: - pattern_info = self.ERROR_PATTERNS[error_type] - for pattern in pattern_info["patterns"]: - if re.search(pattern, error_text, re.IGNORECASE): - return { - "error_type": error_type, - "suggestion": pattern_info["suggestion"], - "auto_fix": pattern_info.get("auto_fix", False), - "fix_action": pattern_info.get("fix_action"), - "severity": pattern_info.get("severity", "medium") - } - - # 未匹配到已知模式，返回None，由调用方决定是否查询Knowledge - return None - - async def _query_knowledge_for_error(self, error_text: str) -> Optional[Dict]: - """调用 Knowledge Agent 查询错误解决方案""" - try: - from .knowledge import create_knowledge_agent - from gfa_core.mas.llm import LLMManager - from .base import AgentMessage, AgentState - import uuid - - print(f"[DiagnosisAgent] 未知错误，调用 Knowledge Agent 查询...") - - # 创建 Knowledge Agent - llm_manager = LLMManager() - knowledge_agent = create_knowledge_agent(llm_manager) - - # 构建查询 - query = f"基因家族分析错误：{error_text[:200]}，如何解决？" - - message = AgentMessage( - sender="diagnosis", - receiver="knowledge", - message_type="query", - payload={"query": query, "mode": "consultation"} - ) - - state = AgentState( - session_id=f"diagnosis_{uuid.uuid4().hex[:8]}", - results={}, - messages=[] - ) - - # 执行查询 - result_state = await knowledge_agent.execute(state, message) - answer = result_state.results.get("knowledge_response", "") - - if answer and len(answer) > 20: - print(f"[DiagnosisAgent] Knowledge Agent 返回解决方案") - return { - "error_type": "unknown", - "suggestion": answer[:500], - "auto_fix": False, - "severity": "medium", - "source": "knowledge" - } - except Exception as e: - print(f"[DiagnosisAgent] Knowledge 查询失败: {e}") - - return None - - async def _attempt_auto_fix(self, fix_action: str, error_info: Dict, - error_text: str, diagnosis: Dict) -> Dict: - """尝试自动修复""" - if fix_action == "reduce_cpu": - return self._fix_reduce_cpu(error_info) - elif fix_action == "fix_parameter_format": - return self._fix_parameter_format(error_info) - elif fix_action == "execute_dependencies": - return await self._fix_execute_dependencies(error_info, error_text) - else: - return {"success": False, "message": f"Unknown fix action: {fix_action}"} - - def _fix_reduce_cpu(self, error_info: Dict) -> Dict: - """修复：减少CPU使用""" - current_params = error_info.get("params", {}) - current_cpu = current_params.get("cpu", 8) - new_cpu = max(1, current_cpu // 2) - - return { - "success": True, - "action": "reduce_cpu", - "changes": {"cpu": {"from": current_cpu, "to": new_cpu}}, - "message": f"将CPU从{current_cpu}减少到{new_cpu}", - "new_params": {**current_params, "cpu": new_cpu} - } - - def _fix_parameter_format(self, error_info: Dict) -> Dict: - """修复：参数格式""" - current_params = error_info.get("params", {}) - fixed_params = {} - changes = {} - - for key, value in current_params.items(): - if key in self.PARAMETER_FIXES: - new_key = self.PARAMETER_FIXES[key] - fixed_params[new_key] = value - changes[key] = {"to": new_key} - else: - fixed_params[key] = value - - return { - "success": True, - "action": "fix_parameter_format", - "changes": changes, - "message": f"修复了{len(changes)}个参数名称", - "new_params": fixed_params - } - - async def _fix_execute_dependencies(self, error_info: Dict, error_text: str) -> Dict: - """修复：自动执行依赖阶段""" - current_params = error_info.get("params", {}) - stage = error_info.get("stage", "unknown") - - # 从错误文本中提取缺失的依赖阶段 - import re - missing_deps = [] - dep_patterns = [ - r"请先执行\s*(\w+)\s*阶段", - r"requires\s*(\w+)\s*stage", - r"依赖阶段\s*(\w+)\s*未完成", - r"(\w+)\s*stage\s*not completed" - ] - - for pattern in dep_patterns: - matches = re.findall(pattern, error_text, re.IGNORECASE) - missing_deps.extend(matches) - - # 根据主规划目录推断依赖，避免诊断链路与 PlanningAgentV2 分叉。 - if stage in CATALOG_STAGE_DEPENDENCIES and not missing_deps: - missing_deps = list(CATALOG_STAGE_DEPENDENCIES[stage]) - - return { - "success": True, - "action": "execute_dependencies", - "changes": {"added_stages": missing_deps}, - "message": f"需要前置执行依赖阶段: {', '.join(missing_deps)}", - "new_params": { - **current_params, - "execute_dependencies_first": True, - "missing_dependencies": missing_deps - } - } - - def _generate_notification(self, stage: str, diagnosis: Dict, fix_result: Dict) -> str: - """生成前端通知消息""" - error_type = diagnosis.get("error_type", "unknown") - action = fix_result.get("action", "") - changes = fix_result.get("changes", {}) - - notification_parts = [ - f"🔧 自动修复通知", - f"阶段: {stage}", - f"错误类型: {error_type}", - f"修复操作: {action}", - ] - - if changes: - notification_parts.append("参数变更:") - for key, change in changes.items(): - if isinstance(change, dict) and "from" in change and "to" in change: - notification_parts.append(f" - {key}: {change['from']} → {change['to']}") - elif isinstance(change, dict) and "to" in change: - notification_parts.append(f" - {key} → {change['to']}") - else: - notification_parts.append(f" - {key}: {change}") - - notification_parts.append("正在重新执行...") - - return "\n".join(notification_parts) - - def record_error(self, error_info: Dict, diagnosis: Dict): - """记录错误到历史""" - self.error_history.append({ - "error": error_info, - "diagnosis": diagnosis, - "timestamp": datetime.now().isoformat() - }) - - if len(self.error_history) > 100: - self.error_history = self.error_history[-100:] - - -def create_diagnosis_agent(llm_manager: LLMManager): - """创建DiagnosisAgent实例""" - return DiagnosisAgent(llm_manager) diff --git a/bioflow_kernel/gfa_core/mas/tools/stage_status.py.orig b/bioflow_kernel/gfa_core/mas/tools/stage_status.py.orig deleted file mode 100755 index 993f6f5..0000000 --- a/bioflow_kernel/gfa_core/mas/tools/stage_status.py.orig +++ /dev/null @@ -1,1092 +0,0 @@ -""" -Unified stage status helpers. - -The final source of truth for a completed stage is the stage manifest under -workspace//final plus the files referenced by manifest.artifacts. -Runtime task states from Celery/LangGraph are only scheduling states. -""" -from __future__ import annotations - -import json -import re -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from gfa_core.ppi_string_policy import species_to_string_code - - -PROJECT_ROOT = Path(__file__).absolute().parents[3] -WORKSPACE_ROOT = PROJECT_ROOT / "workspace" -SUCCESS_STATUSES = {"success", "partial_success"} -RNASEQ_STATUS_STAGES = { - "rnaseq_qc", - "rnaseq_align", - "rnaseq_quant", - "rnaseq_de", - "rnaseq_enrich", - "rnaseq_go_enrich", - "rnaseq_kegg_enrich", - "rnaseq_sample_qc", - "rnaseq_fgsea", - "rnaseq_wgcna", - "rnaseq_candidate_family", - "annotation", - "annotation_go", - "annotation_kegg", -} -UNSCOPED_STATUS_STAGES = {"genome_prepare", "genome_prepare_v2", "sequence_extract_by_id", *RNASEQ_STATUS_STAGES} -STAGE_ALIASES = { - "genome_prepare": {"genome_prepare", "genome_prepare_v2"}, - "rnaseq_go_enrich": {"rnaseq_go_enrich", "rnaseq_enrich"}, - "rnaseq_kegg_enrich": {"rnaseq_kegg_enrich", "rnaseq_enrich"}, - "annotation": {"annotation", "annotation_go", "annotation_kegg"}, - "sequence_extract_by_id": {"sequence_extract_by_id", "sequence_extract"}, -} - - -@dataclass -class StageStatusReport: - stage: str - status: str - completed: bool - manifest_path: Path | None = None - manifest_status: str | None = None - manifest: dict[str, Any] | None = None - artifacts_checked: int = 0 - missing_artifacts: list[str] = field(default_factory=list) - errors: list[str] = field(default_factory=list) - diagnosis_report: dict[str, Any] = field(default_factory=dict) - binding_report: dict[str, Any] = field(default_factory=dict) - manifest_quality: dict[str, Any] = field(default_factory=dict) - - def to_dict(self) -> dict[str, Any]: - data = { - "stage": self.stage, - "status": self.status, - "completed": self.completed, - "manifest_path": str(self.manifest_path) if self.manifest_path else None, - "manifest_status": self.manifest_status, - "artifacts_checked": self.artifacts_checked, - "missing_artifacts": self.missing_artifacts, - "errors": self.errors, - "diagnosis_report": self.diagnosis_report, - "binding_report": self.binding_report, - "manifest_quality": self.manifest_quality, - } - manifest = self.manifest or {} - if self.stage == "ppi" and isinstance(manifest.get("ppi_provenance"), dict): - data["ppi_provenance"] = manifest["ppi_provenance"] - stage_context = manifest.get("stage_context") or manifest.get("stageContext") or {} - params = manifest.get("parameters") or {} - for key in ( - "prefix", - "output_prefix", - "scope_id", - "scope_mode", - "analysis_route", - "member_set_id", - "result_group", - "target_species", - "reference_species", - "comparative_species", - "selected_species", - "scope_file", - "id_contract_hash", - "scope_type", - "rnaseq_scope_id", - "rnaseq_species", - "rnaseq_sample_count", - "rnaseq_condition_count", - "rnaseq_condition_signature", - "foreground_mode", - "family_scope_id", - "family_members_file", - "triggered_expression", - "triggered_expression_manifest", - ): - value = manifest.get(key) - if value in (None, "", []): - value = params.get(key) - if value in (None, "", []): - value = stage_context.get(key) - if value not in (None, "", []): - data[key] = value - return data - - -def manifest_path_for( - job_id: str, - prefix: str, - stage: str, - workspace_root: Path = WORKSPACE_ROOT, - scope_id: str = "", -) -> Path: - final_dir = workspace_root / job_id / "final" - if scope_id: - from gfa_core.gene_family_scope import sanitize_token - - return final_dir / f"{prefix}.{sanitize_token(scope_id)}.{stage}.manifest.json" - return final_dir / f"{prefix}.{stage}.manifest.json" - - -def _scope_signature(scope_id: str) -> tuple[str, tuple[str, ...], tuple[str, ...], tuple[str, ...], str]: - """Compare gene-family scopes independent of species order.""" - parts = [part for part in str(scope_id or "").split(".") if part] - family = parts[0] if parts else "" - targets: list[str] = [] - refs: list[str] = [] - comps: list[str] = [] - mode = "" - for part in parts[1:]: - if part.startswith("target-"): - targets = [x for x in part[len("target-") :].split("-") if x] - elif part.startswith("ref-"): - refs = [x for x in part[len("ref-") :].split("-") if x] - elif part.startswith("comp-"): - comps = [x for x in part[len("comp-") :].split("-") if x] - else: - mode = part - return ( - family.lower(), - tuple(sorted(x.lower() for x in targets)), - tuple(sorted(x.lower() for x in refs)), - tuple(sorted(x.lower() for x in comps)), - mode.lower(), - ) - - -def _scope_matches(requested: tuple[str, tuple[str, ...], tuple[str, ...], tuple[str, ...], str], candidate: tuple[str, tuple[str, ...], tuple[str, ...], tuple[str, ...], str], stage: str) -> bool: - if requested == candidate: - return True - if stage == "identify": - req_family, req_targets, _req_refs, _req_comps, req_mode = requested - cand_family, cand_targets, _cand_refs, _cand_comps, cand_mode = candidate - # Identify materializes target family members. Reference/comparative roles - # are only upstream search context, so legacy scoped manifests that carried - # ref/comparative labels must still satisfy a target-only prerequisite. - return ( - req_family == cand_family - and set(req_targets).issubset(set(cand_targets)) - ) - return False - - -def _manifest_member_set_id(manifest: dict[str, Any]) -> str: - return str( - manifest.get("member_set_id") - or manifest.get("parameters", {}).get("member_set_id") - or manifest.get("stage_context", {}).get("member_set_id") - or manifest.get("stageContext", {}).get("member_set_id") - or "" - ) - - -def _member_set_matches(manifest: dict[str, Any], requested_member_set_id: str) -> bool: - if not requested_member_set_id: - return True - manifest_member_set = _manifest_member_set_id(manifest) - # Older stage manifests are scope-aware but do not persist member_set_id. - # Treat missing member_set_id as compatible; explicit mismatches remain invalid. - if not manifest_member_set: - return True - if manifest_member_set == requested_member_set_id: - return True - manifest_parts = manifest_member_set.split(":") - requested_parts = requested_member_set_id.split(":") - if len(manifest_parts) >= 3 and len(requested_parts) >= 3: - return ( - _scope_signature(manifest_parts[0]) == _scope_signature(requested_parts[0]) - and manifest_parts[1:] == requested_parts[1:] - ) - return manifest_member_set == requested_member_set_id - - -def _manifest_analysis_route(manifest: dict[str, Any]) -> str: - return str( - manifest.get("analysis_route") - or manifest.get("parameters", {}).get("analysis_route") - or manifest.get("stage_context", {}).get("analysis_route") - or manifest.get("stageContext", {}).get("analysis_route") - or "" - ) - - -def _identity_strict_for_route(stage: str, strict_identity: bool, analysis_route: str, member_set_id: str) -> bool: - if not strict_identity: - return False - route = str(analysis_route or "") - member = str(member_set_id or "") - return bool( - route in {"confirmed_subfamily", "subfamily_evidence", "subfamily_prediction", "multi_species"} - or member - or stage in {"phylogeny", "motif", "domain", "structure", "combo", "tfbs", "interspecies_synteny", "reference_classification"} - ) - - -def _find_exact_alias_manifest( - manifest_path: Path, - prefix: str, - stage: str, - scope_id: str = "", -) -> Path | None: - allowed_stages = STAGE_ALIASES.get(stage, {stage}) - for filename_stage in sorted((allowed_stages | {stage}) - {stage}): - if scope_id: - try: - from gfa_core.gene_family_scope import sanitize_token - - candidate = manifest_path.parent / f"{prefix}.{sanitize_token(scope_id)}.{filename_stage}.manifest.json" - except Exception: - continue - else: - candidate = manifest_path.parent / f"{prefix}.{filename_stage}.manifest.json" - if candidate.exists(): - return candidate - return None - - -def _find_latest_stage_manifest( - final_dir: Path, - prefix: str, - stage: str, - analysis_route: str = "", - member_set_id: str = "", -) -> Path | None: - """Find the newest manifest for a stage when the caller has no explicit scope. - - This is intentionally only a non-strict fallback. Strict identity callers should - pass scope_id/route/member_set so the exact scoped manifest is selected. - """ - allowed_stages = STAGE_ALIASES.get(stage, {stage}) - filename_stages = sorted(allowed_stages | {stage}) - patterns: list[str] = [] - if prefix: - for filename_stage in filename_stages: - patterns.extend([f"{prefix}.{filename_stage}.manifest.json", f"{prefix}.*.{filename_stage}.manifest.json"]) - for filename_stage in filename_stages: - patterns.append(f"*.{filename_stage}.manifest.json") - candidates: list[tuple[float, Path]] = [] - seen: set[Path] = set() - for pattern in patterns: - for candidate in final_dir.glob(pattern): - if candidate in seen: - continue - seen.add(candidate) - try: - candidate, manifest = _load_manifest_resolving_alias(candidate) - except Exception: - continue - manifest_stage = str(manifest.get("stage") or "") - if manifest_stage and manifest_stage not in allowed_stages: - continue - if prefix and str(manifest.get("prefix") or "") != prefix: - continue - if analysis_route: - manifest_route = str(manifest.get("analysis_route") or manifest.get("parameters", {}).get("analysis_route") or "") - if manifest_route and manifest_route != analysis_route: - continue - if member_set_id and not _member_set_matches(manifest, member_set_id): - continue - try: - mtime = candidate.stat().st_mtime - except OSError: - mtime = 0 - candidates.append((mtime, candidate)) - if not candidates: - return None - candidates.sort(reverse=True) - return candidates[0][1] - - -def _find_equivalent_scoped_manifest( - final_dir: Path, - prefix: str, - stage: str, - scope_id: str = "", - analysis_route: str = "", - member_set_id: str = "", -) -> Path | None: - if not scope_id: - return None - requested_scope = _scope_signature(scope_id) - candidates: list[tuple[float, Path]] = [] - allowed_stages = STAGE_ALIASES.get(stage, {stage}) - filename_stages = sorted(allowed_stages | {stage}) - patterns = [] - if prefix: - patterns.extend(f"{prefix}.*.{filename_stage}.manifest.json" for filename_stage in filename_stages) - patterns.extend(f"*.{filename_stage}.manifest.json" for filename_stage in filename_stages) - seen: set[Path] = set() - for pattern in patterns: - for candidate in final_dir.glob(pattern): - if candidate in seen: - continue - seen.add(candidate) - try: - candidate, manifest = _load_manifest_resolving_alias(candidate) - except Exception: - continue - if str(manifest.get("stage") or "") not in STAGE_ALIASES.get(stage, {stage}): - continue - manifest_scope = str(manifest.get("scope_id") or manifest.get("stage_context", {}).get("scope_id") or "") - if not manifest_scope or not _scope_matches(requested_scope, _scope_signature(manifest_scope), stage): - continue - if analysis_route: - manifest_route = str(manifest.get("analysis_route") or manifest.get("parameters", {}).get("analysis_route") or "") - if manifest_route and manifest_route != analysis_route: - continue - if member_set_id: - if not _member_set_matches(manifest, member_set_id): - continue - try: - mtime = candidate.stat().st_mtime - except OSError: - mtime = 0 - candidates.append((mtime, candidate)) - if not candidates: - return None - candidates.sort(reverse=True) - return candidates[0][1] - - -def _canonical_path_from_alias(manifest_path: Path, manifest: dict[str, Any]) -> Path | None: - for key in ("canonical_manifest", "canonical_manifest_path"): - value = str(manifest.get(key) or "").strip() - if not value: - continue - candidate = Path(value) - if not candidate.is_absolute(): - candidate = manifest_path.parent / candidate.name if value.startswith("final/") else manifest_path.parent / candidate - try: - resolved_candidate = candidate.resolve() - resolved_manifest = manifest_path.resolve() - if resolved_candidate.parent != resolved_manifest.parent: - continue - if candidate.exists() and resolved_candidate != resolved_manifest: - return candidate - except OSError: - continue - - # Older compatibility copies were byte-for-byte manifest copies and therefore - # have no alias marker. Keep exact scoped manifests authoritative; otherwise - # allow output_prefix to point old aliases back to their canonical manifest. - stage = str(manifest.get("stage") or "").strip() - prefix = str(manifest.get("prefix") or "").strip() - scope_id = str(manifest.get("scope_id") or manifest.get("stage_context", {}).get("scope_id") or "").strip() - if stage and prefix and scope_id: - try: - from gfa_core.gene_family_scope import sanitize_token - - scoped_name = f"{prefix}.{sanitize_token(scope_id)}.{stage}.manifest.json" - if manifest_path.name == scoped_name: - return None - except Exception: - pass - output_prefix = str( - manifest.get("output_prefix") - or manifest.get("parameters", {}).get("output_prefix") - or manifest.get("stage_context", {}).get("output_prefix") - or "" - ).strip() - if stage and output_prefix: - candidate = manifest_path.parent / f"{output_prefix}.{stage}.manifest.json" - try: - if candidate.exists() and candidate.resolve() != manifest_path.resolve(): - return candidate - except OSError: - pass - return None - - -def _load_manifest_resolving_alias(manifest_path: Path) -> tuple[Path, dict[str, Any]]: - with open(manifest_path, "r", encoding="utf-8") as handle: - manifest = json.load(handle) - canonical = _canonical_path_from_alias(manifest_path, manifest) - if canonical: - with open(canonical, "r", encoding="utf-8") as handle: - return canonical, json.load(handle) - return manifest_path, manifest - - -def _load_manifest_for_identity( - manifest_path: Path, - *, - stage: str, - strict_identity: bool, - analysis_route: str = "", - member_set_id: str = "", -) -> tuple[Path, dict[str, Any]]: - with open(manifest_path, "r", encoding="utf-8") as handle: - manifest = json.load(handle) - canonical = _canonical_path_from_alias(manifest_path, manifest) - if not canonical: - return manifest_path, manifest - if _identity_strict_for_route(stage, strict_identity, analysis_route, member_set_id): - alias_route = _manifest_analysis_route(manifest) - alias_member = _manifest_member_set_id(manifest) - if analysis_route and alias_route and alias_route != analysis_route: - return manifest_path, manifest - if member_set_id and alias_member and not _member_set_matches(manifest, member_set_id): - return manifest_path, manifest - with open(canonical, "r", encoding="utf-8") as handle: - return canonical, json.load(handle) - - -def _migrate_legacy_manifest_if_present(manifest_path: Path, prefix: str, stage: str) -> Path | None: - """Migrate legacy final/{stage}.manifest.json to final/{prefix}.{stage}.manifest.json.""" - legacy_path = manifest_path.parent / f"{stage}.manifest.json" - if manifest_path.exists() or not legacy_path.exists(): - return None - - try: - with open(legacy_path, "r", encoding="utf-8") as handle: - manifest = json.load(handle) - except Exception: - return None - - if manifest.get("stage") != stage: - return None - - manifest["prefix"] = manifest.get("prefix") or prefix - manifest["stage"] = stage - manifest["status"] = manifest.get("status") or "success" - - artifacts = manifest.get("artifacts") - if isinstance(artifacts, dict) and "manifest" in artifacts and isinstance(artifacts["manifest"], dict): - artifacts["manifest"]["path"] = f"final/{manifest_path.name}" - artifacts["manifest"]["full_path"] = str(manifest_path.absolute()) - - try: - with open(manifest_path, "w", encoding="utf-8") as handle: - json.dump(manifest, handle, ensure_ascii=False, indent=2) - except Exception: - return None - - return manifest_path - - -def _resolve_artifact_path(artifact: dict[str, Any], manifest: dict[str, Any], manifest_path: Path) -> Path | None: - rel_value = artifact.get("path") or artifact.get("relpath_or_hint") or artifact.get("file") - final_dir = manifest_path.parent - workspace_dir = final_dir.parent - project_root = workspace_dir.parent.parent - input_dir = project_root / "input" / workspace_dir.name - preferred_missing_path: Path | None = None - - def _inside_current_job(path: Path) -> bool: - try: - resolved = path.resolve() - return resolved.is_relative_to(workspace_dir.resolve()) or resolved.is_relative_to(input_dir.resolve()) - except OSError: - return False - - def _rebase_current_job_path(path: Path) -> Path | None: - parts = path.parts - for root_name, current_root in (("workspace", workspace_dir), ("input", input_dir)): - for idx, part in enumerate(parts): - if part != root_name: - continue - if idx + 1 >= len(parts) or parts[idx + 1] != workspace_dir.name: - continue - suffix = Path(*parts[idx + 2 :]) if idx + 2 < len(parts) else Path() - candidate = current_root / suffix - if candidate.exists(): - return candidate - return None - - if rel_value: - rel_path = Path(str(rel_value)) - if rel_path.is_absolute(): - if _inside_current_job(rel_path): - return rel_path - rebased = _rebase_current_job_path(rel_path) - if rebased: - return rebased - preferred_missing_path = rel_path - else: - workspace_root = manifest.get("workspace_root") - if workspace_root: - candidate = Path(workspace_root) / rel_path - if candidate.exists() and _inside_current_job(candidate): - return candidate - preferred_missing_path = preferred_missing_path or candidate - - workspace = manifest.get("workspace") - if workspace: - candidate = Path(workspace) / rel_path.name if str(rel_path).startswith("final/") else Path(workspace) / rel_path - if candidate.exists() and _inside_current_job(candidate): - return candidate - preferred_missing_path = preferred_missing_path or candidate - - # Most stage manifests live in workspace//final. Artifact paths are - # historically mixed: final/foo.png is final-relative, while pep/analysis/* - # and similar paths are workspace-relative. - if str(rel_path).startswith("final/"): - candidate = final_dir / rel_path.name - if candidate.exists(): - return candidate - preferred_missing_path = preferred_missing_path or candidate - else: - workspace_candidate = workspace_dir / rel_path - if workspace_candidate.exists(): - return workspace_candidate - preferred_missing_path = preferred_missing_path or workspace_candidate - final_candidate = final_dir / rel_path - if final_candidate.exists(): - return final_candidate - preferred_missing_path = preferred_missing_path or final_candidate - - path_value = artifact.get("full_path") - if path_value: - full_path = Path(str(path_value)) - if _inside_current_job(full_path): - return full_path - rebased = _rebase_current_job_path(full_path) - if rebased: - return rebased - if preferred_missing_path is not None: - return preferred_missing_path - return None - - -def _rebase_manifest_paths_for_current_job(manifest: dict[str, Any], manifest_path: Path) -> dict[str, Any]: - """Return a runtime view of a manifest with copied-job paths rebound locally. - - Historical manifests can contain absolute paths to the project that originally - created them. When a job is copied into PlantAI-dev, API/status payloads must - point at the current PlantAI-dev input/workspace trees instead of leaking or - validating against the source project. - """ - final_dir = manifest_path.parent - workspace_dir = final_dir.parent - project_root = workspace_dir.parent.parent - input_dir = project_root / "input" / workspace_dir.name - job_id = re.escape(workspace_dir.name) - provenance_keys = { - "external_resource_sources", - "resource_diagnostics", - "historical_annotation_roots", - "shared_resource_roots", - "checked_roots", - "missing_roots", - } - - def _rebase_text(value: str) -> str: - def repl(match: re.Match[str]) -> str: - root_kind = match.group("root") - replacement = workspace_dir if root_kind == "workspace" else input_dir - return str(replacement) + match.group("trail") - - pattern = re.compile(rf"/[^\s\"']*?/(?Pworkspace|input)/{job_id}(?P/?)") - rebased = pattern.sub(repl, value) - rebased = re.sub( - r"/[^\s\"']*/bbav5_0502_Yang_Tihuan\.7880(?:\.bak)?", - str(project_root), - rebased, - ) - return rebased - - def _walk(value: Any, key_path: tuple[str, ...] = ()) -> Any: - if key_path and key_path[-1] in provenance_keys: - return value - if isinstance(value, str): - return _rebase_text(value) - if isinstance(value, list): - return [_walk(item, key_path) for item in value] - if isinstance(value, tuple): - return [_walk(item, key_path) for item in value] - if isinstance(value, dict): - return {key: _walk(item, (*key_path, str(key))) for key, item in value.items()} - return value - - return _walk(manifest) - - -def _iter_artifacts(manifest: dict[str, Any]) -> list[tuple[str, dict[str, Any]]]: - artifacts = manifest.get("artifacts") or {} - if isinstance(artifacts, dict): - items = [] - for name, info in artifacts.items(): - if isinstance(info, dict): - items.append((str(name), info)) - elif isinstance(info, str) and info.strip(): - items.append((str(name), {"path": info.strip()})) - return items - if isinstance(artifacts, list): - items = [] - for index, info in enumerate(artifacts): - if isinstance(info, dict): - name = str(info.get("artifact") or info.get("name") or index) - items.append((name, info)) - elif isinstance(info, str) and info.strip(): - items.append((str(index), {"path": info.strip()})) - return items - return [] - - -def _is_optional_missing_artifact(stage: str, artifact_name: str, artifact: dict[str, Any]) -> bool: - """Artifacts that are useful for inspection but not required for stage completion. - - RNA-seq QC may keep MultiQC plus markdown summaries while pruning individual - per-sample FastQC HTML files. Those per-sample reports are redundant with - the MultiQC summary and should be surfaced as a quality warning, not as a - failed prerequisite for downstream RNA-seq stages. - """ - name = str(artifact_name or "") - path_text = " ".join(str(artifact.get(key) or "") for key in ("path", "full_path", "file", "relpath_or_hint")) - lowered = f"{name} {path_text}".lower() - if stage == "rnaseq_qc" and name.startswith("qc_report_") and ("fastqc" in lowered): - return True - if stage == "rnaseq_quant" and name in {"salmon_tximport_summary", "multiqc_report"}: - return True - if stage in {"rnaseq_de", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea", "rnaseq_wgcna"}: - return any(token in lowered for token in ("_plot", "barplot", "dotplot", "heatmap", ".png", ".pdf")) - return False - - -def _manifest_reference_species_code(manifest: dict[str, Any]) -> str: - for section in (manifest, manifest.get("parameters") or {}, manifest.get("stage_context") or {}, manifest.get("stageContext") or {}): - value = section.get("reference_species") or section.get("referenceSpecies") - if isinstance(value, list) and value: - return species_to_string_code(str(value[0] or ""), "").lower() - if isinstance(value, str) and value.strip(): - return species_to_string_code(value, "").lower() - return "" - - -def _ortholog_mapping_reference_species_code(mapping_path: Path) -> str: - try: - with mapping_path.open("r", encoding="utf-8", errors="ignore") as handle: - for raw in handle: - line = raw.strip() - if not line or line.startswith("#"): - continue - parts = line.split("\t") - if len(parts) >= 5: - return species_to_string_code(parts[4], "").lower() - return "" - except OSError: - return "" - return "" - - -def _legacy_rnaseq_status_from_outputs(final_dir: Path, prefix: str, stage: str) -> StageStatusReport | None: - """Infer status for older RNA-seq runs that produced outputs before manifests. - - This is deliberately narrow and evidence-based. It does not create or - rewrite manifests; it only lets the status API represent existing final - outputs when old jobs lack a stage manifest. - """ - evidence_specs: dict[str, list[tuple[str, str]]] = { - "rnaseq_align": [ - ("align_report", f"{prefix}.rnaseq_align_report.md"), - ("align_multiqc_data", f"{prefix}.rnaseq_align_multiqc_report_data"), - ], - "rnaseq_candidate_family": [ - ("recommendation", f"{prefix}.rnaseq_candidate_family.recommendation.tsv"), - ("evidence_matrix", f"{prefix}.rnaseq_candidate_family.evidence_matrix.tsv"), - ("family_enrichment", f"{prefix}.rnaseq_candidate_family.family_enrichment.tsv"), - ("report", f"{prefix}.rnaseq_candidate_family_report.md"), - ], - } - specs = evidence_specs.get(stage) - if not specs: - return None - artifacts: dict[str, dict[str, Any]] = {} - missing: list[str] = [] - for artifact_name, filename in specs: - path = final_dir / filename - if path.exists(): - artifacts[artifact_name] = { - "artifact": artifact_name, - "path": f"final/{filename}", - "full_path": str(path.absolute()), - } - else: - missing.append(f"{artifact_name}: {path}") - extra_metrics: dict[str, Any] = {} - if stage == "rnaseq_align": - bam_count = len(list(final_dir.glob("*.sorted.bam"))) - flagstat_count = len(list(final_dir.glob("*.flagstat.txt"))) - extra_metrics.update({"sorted_bam_count": bam_count, "flagstat_count": flagstat_count}) - if bam_count <= 0: - missing.append(f"sorted_bam: {final_dir}/*.sorted.bam") - if missing: - return None - manifest = { - "manifest_version": "legacy_inferred_v1", - "stage": stage, - "status": "legacy_inferred", - "prefix": prefix, - "artifacts": artifacts, - "metrics": extra_metrics, - } - representative = next(iter(artifacts.values())).get("full_path") if artifacts else "" - return StageStatusReport( - stage=stage, - status="legacy_inferred", - completed=True, - manifest_path=Path(representative) if representative else None, - manifest_status="legacy_inferred", - manifest=manifest, - artifacts_checked=len(artifacts), - manifest_quality={ - "legacy_inferred": True, - "reason": "stage manifest is missing but final outputs provide the stage evidence", - }, - ) - - -def _current_rnaseq_library_strandness_signature(job_id: str, manifest: dict[str, Any], workspace_root: Path) -> str: - if not job_id: - return "" - try: - from gfa_core.rnaseq_context import resolve_rnaseq_library_strandness - except Exception: - return "" - input_dir = workspace_root.parent / "input" / job_id - sample_config = input_dir / "samples.yaml" - if not sample_config.exists(): - sample_config = input_dir / "samples.yml" - if not sample_config.exists(): - return "" - sample_names: list[str] = [] - by_sample = manifest.get("rnaseq_library_strandness_by_sample") or manifest.get("parameters", {}).get("library_strandness_by_sample") - if isinstance(by_sample, dict) and by_sample: - sample_names = list(map(str, by_sample.keys())) - else: - metrics_samples = manifest.get("metrics", {}).get("samples") - if isinstance(metrics_samples, dict): - sample_names = list(map(str, metrics_samples.keys())) - try: - info = resolve_rnaseq_library_strandness(sample_config, samples=sample_names, require_all=False) - return str(info.get("library_strandness_signature") or "") - except Exception: - return "" - - -def get_stage_status( - job_id: str, - prefix: str, - stage: str, - workspace_root: Path = WORKSPACE_ROOT, - require_artifacts: bool = True, - scope_id: str = "", - analysis_route: str = "", - member_set_id: str = "", - strict_identity: bool = False, -) -> StageStatusReport: - stage = "genome_prepare" if stage == "genome_prepare_v2" else stage - if stage == "extension_stage": - try: - from gfa_core.extension_stage.status import latest_extension_status - - ext = latest_extension_status(job_id, workspace_root) - return StageStatusReport( - stage="extension_stage", - status=str(ext.get("status") or "unknown"), - completed=bool(ext.get("completed")), - manifest_path=Path(ext["manifest_path"]) if ext.get("manifest_path") else None, - manifest=ext.get("manifest") if isinstance(ext.get("manifest"), dict) else None, - artifacts_checked=int(ext.get("artifacts_checked") or 0), - missing_artifacts=list(ext.get("missing_artifacts") or []), - errors=list(ext.get("errors") or []), - diagnosis_report=ext.get("diagnosis_report") if isinstance(ext.get("diagnosis_report"), dict) else {}, - binding_report=ext.get("binding_report") if isinstance(ext.get("binding_report"), dict) else {}, - manifest_quality=ext.get("manifest_quality") if isinstance(ext.get("manifest_quality"), dict) else {}, - ) - except Exception as exc: - return StageStatusReport(stage="extension_stage", status="status_error", completed=False, errors=[str(exc)]) - if stage in UNSCOPED_STATUS_STAGES: - scope_id = "" - analysis_route = "" - member_set_id = "" - strict_identity = False - if stage == "genome_prepare": - # genome_prepare is job/species-level preprocessing. UI plans may still - # pass a family prefix such as Target.SPL, but the stage deliberately - # materializes as genome_prepare.genome_prepare.manifest.json. - prefix = "genome_prepare" - manifest_path = manifest_path_for(job_id, prefix, stage, workspace_root, scope_id=scope_id) - if stage == "subfamily_confirm" and not manifest_path.exists(): - final_dir_for_confirm = workspace_root / job_id / "final" - confirmed_candidates = [] - if final_dir_for_confirm.exists(): - family_token = prefix.split(".")[-1] if prefix else "" - species_token = prefix.split(".")[0] if "." in prefix else "" - for candidate in final_dir_for_confirm.glob("*confirmed_subfamily.tsv"): - if not candidate.is_file() or candidate.stat().st_size <= 0: - continue - name = candidate.name - if family_token and family_token not in name: - continue - if species_token and species_token not in name: - continue - confirmed_candidates.append(candidate) - if confirmed_candidates: - manifest = { - "stage": "subfamily_confirm", - "prefix": prefix, - "status": "success", - "artifacts": {"confirmed_subfamily": {"full_path": str(sorted(confirmed_candidates)[-1])}}, - "analysis_route": analysis_route or "confirmed_subfamily", - "member_set_id": member_set_id or "confirmed_subfamily", - } - return StageStatusReport( - stage=stage, - status="success", - completed=True, - manifest_path=sorted(confirmed_candidates)[-1], - manifest_status="success", - manifest=manifest, - artifacts_checked=1, - ) - if not manifest_path.exists(): - alias_path = _find_exact_alias_manifest(manifest_path, prefix, stage, scope_id=scope_id) - if alias_path: - manifest_path = alias_path - if not manifest_path.exists(): - migrated_path = _migrate_legacy_manifest_if_present(manifest_path, prefix, stage) - if migrated_path: - manifest_path = migrated_path - else: - equivalent_path = _find_equivalent_scoped_manifest( - manifest_path.parent, - prefix, - stage, - scope_id=scope_id, - analysis_route=analysis_route, - member_set_id=member_set_id, - ) - if equivalent_path: - manifest_path = equivalent_path - else: - latest_path = None - if not scope_id and not strict_identity: - latest_path = _find_latest_stage_manifest( - manifest_path.parent, - prefix, - stage, - analysis_route=analysis_route, - member_set_id=member_set_id, - ) - if latest_path: - manifest_path = latest_path - else: - legacy_report = _legacy_rnaseq_status_from_outputs(manifest_path.parent, prefix, stage) - if legacy_report: - return legacy_report - return StageStatusReport( - stage=stage, - status="not_found", - completed=False, - manifest_path=manifest_path, - errors=[f"Manifest not found: {manifest_path}"], - ) - - try: - manifest_path, manifest = _load_manifest_for_identity( - manifest_path, - stage=stage, - strict_identity=strict_identity, - analysis_route=analysis_route, - member_set_id=member_set_id, - ) - except Exception as exc: - return StageStatusReport( - stage=stage, - status="invalid_manifest", - completed=False, - manifest_path=manifest_path, - errors=[f"Manifest is not valid JSON: {exc}"], - ) - manifest = _rebase_manifest_paths_for_current_job(manifest, manifest_path) - - if analysis_route: - manifest_route = str(manifest.get("analysis_route") or manifest.get("parameters", {}).get("analysis_route") or "") - if manifest_route and manifest_route != analysis_route: - equivalent_path = _find_equivalent_scoped_manifest( - manifest_path.parent, - prefix, - stage, - scope_id=scope_id, - analysis_route=analysis_route, - member_set_id=member_set_id, - ) - if equivalent_path and equivalent_path != manifest_path: - try: - manifest_path, manifest = _load_manifest_for_identity( - equivalent_path, - stage=stage, - strict_identity=strict_identity, - analysis_route=analysis_route, - member_set_id=member_set_id, - ) - manifest = _rebase_manifest_paths_for_current_job(manifest, manifest_path) - except Exception: - pass - - manifest_status = str(manifest.get("status") or "unknown") - errors: list[str] = [] - missing_artifacts: list[str] = [] - optional_missing_artifacts: list[str] = [] - - manifest_stage = manifest.get("stage") - allowed_stages = STAGE_ALIASES.get(stage, {stage}) - if manifest_stage and manifest_stage not in allowed_stages: - errors.append(f"Manifest stage mismatch: expected {stage}, got {manifest_stage}") - if scope_id: - manifest_scope = str(manifest.get("scope_id") or manifest.get("stage_context", {}).get("scope_id") or "") - if not manifest_scope: - if strict_identity: - errors.append(f"Manifest scope_id missing: expected {scope_id}") - elif not _scope_matches(_scope_signature(scope_id), _scope_signature(manifest_scope), stage): - errors.append(f"Manifest scope_id mismatch: expected {scope_id}, got {manifest_scope}") - strict_stage_identity = bool(strict_identity and stage != "identify") - if analysis_route: - manifest_route = str(manifest.get("analysis_route") or manifest.get("parameters", {}).get("analysis_route") or "") - if not manifest_route and strict_stage_identity: - errors.append(f"Manifest analysis_route missing: expected {analysis_route}") - elif manifest_route and manifest_route != analysis_route: - errors.append(f"Manifest analysis_route mismatch: expected {analysis_route}, got {manifest_route}") - if member_set_id: - manifest_member_set = _manifest_member_set_id(manifest) - route_requires_member = _identity_strict_for_route(stage, strict_identity, analysis_route, member_set_id) - if not manifest_member_set and route_requires_member: - if stage != "identify": - errors.append(f"Manifest member_set_id missing: expected {member_set_id}") - elif manifest_member_set and not _member_set_matches(manifest, member_set_id): - errors.append(f"Manifest member_set_id mismatch: expected {member_set_id}, got {manifest_member_set}") - if stage in {"rnaseq_align", "rnaseq_quant"}: - manifest_strandness_signature = str( - manifest.get("rnaseq_library_strandness_signature") - or manifest.get("parameters", {}).get("library_strandness_signature") - or "" - ) - current_strandness_signature = _current_rnaseq_library_strandness_signature(job_id, manifest, workspace_root) - if current_strandness_signature and not manifest_strandness_signature: - errors.append("Manifest library_strandness_signature missing; rerun with current samples.yaml") - elif current_strandness_signature and manifest_strandness_signature and current_strandness_signature != manifest_strandness_signature: - errors.append( - "Manifest library_strandness_signature mismatch: " - f"expected {current_strandness_signature}, got {manifest_strandness_signature}" - ) - - if stage == "identify": - hmm_models = manifest.get("hmm_models") if isinstance(manifest.get("hmm_models"), dict) else {} - missing_hmms = hmm_models.get("missing_requested_pfams") or [] - hmm_status = str(hmm_models.get("status") or "").lower() - if missing_hmms or hmm_status == "failed": - missing_text = ",".join(str(x) for x in missing_hmms) if missing_hmms else "unknown" - errors.append(f"Identify HMM audit failed: missing requested Pfam model(s): {missing_text}") - - artifacts = _iter_artifacts(manifest) - artifacts_checked = 0 - required_artifacts_present = 0 - for artifact_name, artifact in artifacts: - artifact_path = _resolve_artifact_path(artifact, manifest, manifest_path) - optional_missing = _is_optional_missing_artifact(stage, artifact_name, artifact) - if artifact_path is None: - message = f"{artifact_name}: unresolved path" - if optional_missing: - optional_missing_artifacts.append(message) - else: - missing_artifacts.append(message) - continue - artifacts_checked += 1 - if not artifact_path.exists(): - message = f"{artifact_name}: {artifact_path}" - if optional_missing: - optional_missing_artifacts.append(message) - else: - missing_artifacts.append(message) - continue - if not optional_missing: - required_artifacts_present += 1 - if stage == "ortholog" and artifact_name == "ortholog_mapping": - expected_reference_code = _manifest_reference_species_code(manifest) - observed_reference_code = _ortholog_mapping_reference_species_code(artifact_path) - if expected_reference_code and observed_reference_code and expected_reference_code != observed_reference_code: - errors.append( - "Ortholog mapping reference species mismatch: " - f"expected {expected_reference_code}, got {observed_reference_code}" - ) - # Combo artifacts may use short user-facing names; scope identity is - # validated from manifest fields above rather than forced into filenames. - - if require_artifacts and not artifacts: - errors.append("Manifest has no artifacts") - - success_statuses = set(SUCCESS_STATUSES) - # genome_prepare partial_success means at least one species failed. Treat it as - # a visible result, but not as a completed prerequisite, so users can upload - # missing files and rerun preprocessing without manually deleting manifests. - if stage == "genome_prepare": - success_statuses.discard("partial_success") - - completed = ( - manifest_status in success_statuses - and not errors - and not missing_artifacts - and (required_artifacts_present > 0 or not require_artifacts) - ) - - status = manifest_status if completed else "invalid_outputs" - if manifest_status not in SUCCESS_STATUSES: - status = manifest_status - - return StageStatusReport( - stage=stage, - status=status, - completed=completed, - manifest_path=manifest_path, - manifest_status=manifest_status, - manifest=manifest, - artifacts_checked=artifacts_checked, - missing_artifacts=missing_artifacts, - manifest_quality={"optional_missing_artifacts": optional_missing_artifacts} if optional_missing_artifacts else {}, - errors=errors, - ) - - -def is_stage_completed( - job_id: str, - prefix: str, - stage: str, - workspace_root: Path = WORKSPACE_ROOT, - scope_id: str = "", -) -> bool: - return get_stage_status(job_id, prefix, stage, workspace_root, scope_id=scope_id).completed - - -def list_completed_stages(job_id: str, prefix: str = "", workspace_root: Path = WORKSPACE_ROOT) -> list[str]: - final_dir = workspace_root / job_id / "final" - if not final_dir.exists(): - return [] - - completed: list[str] = [] - pattern = f"{prefix}.*.manifest.json" if prefix else "*.manifest.json" - for manifest_path in sorted(final_dir.glob(pattern)): - try: - with open(manifest_path, "r", encoding="utf-8") as handle: - manifest = json.load(handle) - except Exception: - continue - - manifest_prefix = str(manifest.get("prefix") or prefix) - stage = str(manifest.get("stage") or "") - if not stage: - name = manifest_path.name - if manifest_prefix and name.startswith(f"{manifest_prefix}."): - stage = name[len(manifest_prefix) + 1 : -len(".manifest.json")] - else: - parts = name[: -len(".manifest.json")].split(".") - stage = parts[-1] if parts else "" - if not manifest_prefix or not stage: - continue - - status = get_stage_status(job_id, manifest_prefix, stage, workspace_root) - if status.completed and stage not in completed: - completed.append(stage) - return completed diff --git a/bioflow_kernel/gfa_core/mas/tools/stage_tools.py.orig b/bioflow_kernel/gfa_core/mas/tools/stage_tools.py.orig deleted file mode 100755 index 37a563d..0000000 --- a/bioflow_kernel/gfa_core/mas/tools/stage_tools.py.orig +++ /dev/null @@ -1,1044 +0,0 @@ -""" -Stage-level Tool Registry for Multi-Agent System -将bbav20311的分析阶段脚本封装为Agent可调用的Tool -""" -import os -import subprocess -import json -from pathlib import Path -from dataclasses import dataclass, field -from typing import Any, Optional, Dict, List -from datetime import datetime - -from gfa_core.project_paths import CONDA_BASE, PROJECT_ROOT - -CLI_DIR = PROJECT_ROOT / "apps" / "cli" -CONDA_ENV = "test4py311" - -# Conda环境路径配置 - 使用完整路径避免conda run的全局锁 -CONDA_ENVS = { - "test4py311": CONDA_BASE / "envs" / "test4py311" / "bin" / "python", - "rnaseq": CONDA_BASE / "envs" / "rnaseq" / "bin" / "python", - "rnaR": CONDA_BASE / "envs" / "rnaR" / "bin" / "Rscript", -} - - -@dataclass -class StageTool: - """分析阶段工具定义""" - id: str - name: str - script: str - category: str - description: str - required_inputs: List[str] = field(default_factory=list) - outputs: List[str] = field(default_factory=list) - parameters: Dict[str, Any] = field(default_factory=dict) - timeout: int = 3600 - conda_env: str = CONDA_ENV - - -class StageToolExecutor: - """分析阶段工具执行器 - 复用 pipeline_backend.run_stage()""" - - def __init__(self, project_root: Path = PROJECT_ROOT): - self.project_root = project_root - self.cli_dir = project_root / "apps" / "cli" - self.workspace_root = project_root / "workspace" - self.input_root = project_root / "input" - - @staticmethod - def _has_non_comment_data(path: Path) -> bool: - try: - if not path.exists() or path.stat().st_size == 0: - return False - with path.open("r", encoding="utf-8", errors="ignore") as handle: - for raw in handle: - line = raw.strip() - if line and not line.startswith("#"): - return True - except Exception: - return False - return False - - def _preflight_enrich_background( - self, - stage_tool: StageTool, - job_id: str, - enrich_type: str, - enrich_background_source: str, - enrich_species: str, - ) -> dict | None: - """Block implicit long annotation when enrichment background is missing.""" - if stage_tool.id not in {"rnaseq_enrich", "rnaseq_go_enrich", "rnaseq_kegg_enrich"}: - return None - - source = (enrich_background_source or "auto").strip().lower() - if source in {"auto", "annotation", "eggnog", "auto_annotation"}: - return None - - input_dir = self.input_root / job_id - final_dir = self.workspace_root / job_id / "final" - gene2go_files = ( - list(input_dir.glob("*gene2go*.txt")) + list(input_dir.glob("*gene2go*.tsv")) + - list(final_dir.glob("*gene2go*.txt")) + list(final_dir.glob("*gene2go*.tsv")) - ) - gene2ko_files = ( - list(input_dir.glob("*gene2ko*.txt")) + list(input_dir.glob("*gene2ko*.tsv")) + - list(final_dir.glob("*gene2ko*.txt")) + list(final_dir.glob("*gene2ko*.tsv")) - ) - gene2go_files = [p for p in gene2go_files if self._has_non_comment_data(p) and ".test." not in p.name] - gene2ko_files = [p for p in gene2ko_files if self._has_non_comment_data(p) and ".test." not in p.name] - - enrich_type_val = (enrich_type or "go").strip().lower() - if enrich_type_val not in {"go", "kegg", "both"}: - enrich_type_val = "go" - need_go = enrich_type_val in {"go", "both"} - need_kegg = enrich_type_val in {"kegg", "both"} - supported_orgdb_species = {"ath", "hsa", "mmu", "rno", "dme", "sce"} - species_val = (enrich_species or "").strip().lower() - - if source == "orgdb": - if need_kegg: - msg = ( - "❌ 当前 KEGG 富集不支持使用 OrgDb 作为背景。请上传 gene2ko.txt/gene2ko.tsv，" - "或选择“自动注释 + 继续富集”生成 gene2ko。" - ) - return {"status": "error", "stage": stage_tool.id, "message": msg, "error": msg} - if species_val not in supported_orgdb_species: - msg = ( - f"❌ RNA-seq GO 富集选择了 OrgDb，但物种代码 '{species_val or '未指定'}' 不受支持。" - "支持：ath, hsa, mmu, rno, dme, sce。请上传 gene2go 背景文件，" - "或选择“自动注释 + 继续富集”。" - ) - return {"status": "error", "stage": stage_tool.id, "message": msg, "error": msg} - return None - - missing_go = need_go and not gene2go_files and species_val not in supported_orgdb_species - missing_kegg = need_kegg and not gene2ko_files - if source == "gene2go": - missing_go = need_go and not gene2go_files - - if missing_go or missing_kegg: - missing = [] - if missing_go: - missing.append("gene2go.txt/gene2go.tsv") - if missing_kegg: - missing.append("gene2ko.txt/gene2ko.tsv") - msg = ( - "❌ 无法进行 RNA-seq 富集分析：缺少背景注释文件。\n\n" - f"当前 job 没有找到可用的 {' 和 '.join(missing)}。\n" - "请选择一种处理方式：上传已有背景文件后重试，或确认使用本地按需自动注释后继续富集。" - "GO 缺背景时运行 InterProScan，KEGG 缺背景时运行 KofamScan。" - ) - return {"status": "error", "stage": stage_tool.id, "message": msg, "error": msg} - - return None - - async def execute(self, stage_tool: StageTool, job_id: str, prefix: str, - extra_params: dict = None, input_files: dict = None) -> dict: - """执行分析阶段 - 使用 pipeline_backend.run_stage()""" - import asyncio - from datetime import datetime - - # 延迟导入避免循环依赖 - try: - from apps.web.pipeline_backend import run_stage, JobInfo - except ImportError as e: - return { - "status": "error", - "error": f"Failed to import pipeline_backend: {e}", - "stage": stage_tool.id - } - - # 确保目录存在 - workspace = self.workspace_root / job_id - workspace.mkdir(parents=True, exist_ok=True) - input_dir = self.input_root / job_id - input_dir.mkdir(parents=True, exist_ok=True) - logs_dir = workspace / "logs" - logs_dir.mkdir(parents=True, exist_ok=True) - - # 构建 JobInfo - job = JobInfo( - job_id=job_id, - input_dir=str(input_dir), - workspace_dir=str(workspace), - logs_dir=str(logs_dir), - created_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S") - ) - - requested_cpu = extra_params.get("cpu", None) if extra_params else None - try: - from gfa_core.mas.stage_catalog import clamp_stage_cpu - - cpu = clamp_stage_cpu(stage_tool.id, requested_cpu) - except Exception: - try: - cpu = max(1, min(int(requested_cpu), 4)) - except (TypeError, ValueError): - cpu = 4 - if extra_params is not None: - extra_params["requested_cpu"] = requested_cpu - extra_params["cpu"] = cpu - ref_species_value = (extra_params or {}).get("refSpecies") or None - if isinstance(ref_species_value, str): - ref_species_value = [item.strip() for item in ref_species_value.split(",") if item.strip()] - rnaseq_foreground_mode = (extra_params or {}).get("rnaseqForegroundMode", "auto") - enrich_background_source = (extra_params or {}).get("enrichBackgroundSource", "auto") - enrich_species = (extra_params or {}).get("enrichSpecies", "") - enrich_key_type = (extra_params or {}).get("enrichKeyType", "auto") - enrich_type = (extra_params or {}).get("enrichType", stage_tool.parameters.get("enrichType", "go")) - if stage_tool.id == "rnaseq_go_enrich": - enrich_type = "go" - elif stage_tool.id == "rnaseq_kegg_enrich": - enrich_type = "kegg" - kegg_organism = (extra_params or {}).get("keggOrganism", "") - annotation_applications = (extra_params or {}).get("annotationApplications", ["Pfam", "ProSiteProfiles"]) - target_species = (extra_params or {}).get("targetSpecies") or None - interspecies_ref_species = (extra_params or {}).get("refSpeciesList") or None - interspecies_blast_engine = (extra_params or {}).get("blastEngine", "diamond") or "diamond" - interspecies_plot_mode = (extra_params or {}).get("interspeciesPlotMode", "both") or "both" - - def param_value(*keys, default=None): - params = extra_params or {} - for key in keys: - if key in params and params.get(key) is not None and params.get(key) != "": - return params.get(key) - return default - - def bool_param(*keys, default=False): - value = param_value(*keys, default=default) - if isinstance(value, str): - text = value.strip().lower() - if text in {"1", "true", "yes", "y", "on"}: - return True - if text in {"0", "false", "no", "n", "off"}: - return False - return default - return bool(value) - - preflight_error = self._preflight_enrich_background( - stage_tool, - job_id, - enrich_type, - enrich_background_source, - enrich_species, - ) - if preflight_error: - return preflight_error - - # 在线程池中执行同步的 run_stage - loop = asyncio.get_event_loop() - - # 定义带工作目录切换的执行函数 - def run_stage_with_cwd(): - import os - try: - original_cwd = os.getcwd() - except FileNotFoundError: - # Celery workers can inherit a deleted project/workspace cwd after - # local project roots are moved or symlinked. Fall back to the - # stable project root before running stage code. - original_cwd = str(self.project_root) - try: - os.chdir(self.project_root) - return run_stage( - stage=stage_tool.id, - job=job, - prefix=prefix, - cpu=cpu, - required_pfams=(extra_params or {}).get("requiredPfams") or None, - ref_species=ref_species_value, - id_evalue=(extra_params or {}).get("idEvalue", 0.001), - id_coverage=(extra_params or {}).get("idCoverage", 0.3), - blast_evalue=(extra_params or {}).get("blastEvalue", (extra_params or {}).get("blast_evalue", 1e-5)), - blast_identity=(extra_params or {}).get("blastIdentity", (extra_params or {}).get("blast_identity", 30.0)), - phy_mafft=(extra_params or {}).get("mafftParams", f"--auto --thread {cpu}"), - phy_iqtree=(extra_params or {}).get("iqtreeParams", f"-m MFP -bb 1000 -nt {cpu}"), - reference_subfamily_file=(extra_params or {}).get("referenceSubfamilyFile", ""), - reference_target_prefix=(extra_params or {}).get("referenceTargetPrefix", ""), - reference_prefix=(extra_params or {}).get("referencePrefix", "At"), - reference_overwrite_gene_subfamily=bool((extra_params or {}).get("referenceOverwriteGeneSubfamily", False)), - motif_nmotifs=(extra_params or {}).get("nmotifs", 10), - motif_minw=(extra_params or {}).get("minw", 6), - motif_maxw=(extra_params or {}).get("maxw", 200), - dom_evalue=(extra_params or {}).get("domEvalue", 1e-5), - dom_coverage=(extra_params or {}).get("domCoverage", 0.3), - id_transform=(extra_params or {}).get("idTransform", "none"), - geo_id=(extra_params or {}).get("geoId", ""), - rnaseq_mode=(extra_params or {}).get("rnaseqMode", "upload"), - species_name=(extra_params or {}).get("speciesName", ""), - top_pathways=(extra_params or {}).get("topPathways", 20), - rnaseq_foreground_mode=rnaseq_foreground_mode, - enrich_background_source=enrich_background_source, - enrich_species=enrich_species, - enrich_key_type=enrich_key_type, - enrich_type=enrich_type, - kegg_organism=kegg_organism, - annotation_applications=annotation_applications, - rnaseq_sample_config=(extra_params or {}).get("rnaseqSampleConfig", ""), - rnaseq_sample_parallel_jobs=(extra_params or {}).get("sample_parallel_jobs") or (extra_params or {}).get("sampleParallelJobs") or 0, - rnaseq_threads_per_sample=(extra_params or {}).get("threads_per_sample") or (extra_params or {}).get("threadsPerSample") or 0, - rnaseq_project_max_cpu=(extra_params or {}).get("rnaseq_project_max_cpu") or (extra_params or {}).get("rnaseqProjectMaxCpu") or 0, - rnaseq_resource_decision=(extra_params or {}).get("resource_decision") or (extra_params or {}).get("resourceDecision") or None, - rnaseq_qc_tool=param_value("rnaseqQcTool", "qc_tool", default="fastp"), - rnaseq_fastp_bin=param_value("rnaseqFastpBin", "fastp_bin", default="tools/fastp"), - rnaseq_fastp_detect_adapter_for_pe=bool_param("rnaseqFastpDetectAdapterForPe", "detect_adapter_for_pe", default=True), - rnaseq_fastp_cut_front=bool_param("rnaseqFastpCutFront", "cut_front", default=True), - rnaseq_fastp_cut_tail=bool_param("rnaseqFastpCutTail", "cut_tail", default=True), - rnaseq_fastp_cut_window_size=param_value("rnaseqFastpCutWindowSize", "cut_window_size", default=4), - rnaseq_fastp_cut_mean_quality=param_value("rnaseqFastpCutMeanQuality", "cut_mean_quality", default=20), - rnaseq_fastp_qualified_quality_phred=param_value("rnaseqFastpQualifiedQualityPhred", "qualified_quality_phred", default=20), - rnaseq_fastp_unqualified_percent_limit=param_value("rnaseqFastpUnqualifiedPercentLimit", "unqualified_percent_limit", default=30), - rnaseq_fastp_n_base_limit=param_value("rnaseqFastpNBaseLimit", "n_base_limit", default=5), - rnaseq_fastp_length_required=param_value("rnaseqFastpLengthRequired", "length_required", default=30), - rnaseq_splice_site_source=param_value("rnaseqSpliceSiteSource", "splice_site_source", default="auto"), - rnaseq_known_splice_site=param_value("rnaseqKnownSpliceSite", "known_splice_site", default=""), - rnaseq_quant_mode=param_value("rnaseqQuantMode", "quant_mode", default="dual"), - rnaseq_abundance_engine=param_value("rnaseqAbundanceEngine", "abundance_engine", default="auto"), - rnaseq_transcriptome_fasta=param_value("rnaseqTranscriptomeFasta", "transcriptome_fasta", default=""), - rnaseq_tx2gene=param_value("rnaseqTx2gene", "tx2gene", default=""), - rnaseq_salmon_libtype=param_value("rnaseqSalmonLibtype", "salmon_libtype", default="A"), - rnaseq_tximport_counts_from_abundance=param_value("rnaseqTximportCountsFromAbundance", "tximport_counts_from_abundance", default="no"), - rnaseq_kallisto_bias=bool((extra_params or {}).get("rnaseqKallistoBias") or (extra_params or {}).get("kallisto_bias")), - rnaseq_de_padj=param_value("rnaseqDePadj", "padj", default=0.05), - rnaseq_de_log2fc=param_value("rnaseqDeLog2fc", "log2fc", default=1.0), - enrich_pvalue_cutoff=param_value("enrichPvalueCutoff", "pvalue_cutoff", default=0.05), - enrich_qvalue_cutoff=param_value("enrichQvalueCutoff", "qvalue_cutoff", default=0.05), - go_obo=param_value("goObo", "go_obo", default="data/go/go-basic.obo"), - kegg_level=param_value("keggLevel", "kegg_level", default="pathway"), - kegg_filter_scope=param_value("keggFilterScope", "kegg_filter_scope", default="plant"), - kegg_highlight_mode=param_value("keggHighlightMode", "kegg_highlight_mode", default="auto"), - kegg_no_network=bool((extra_params or {}).get("keggNoNetwork") or (extra_params or {}).get("no_kegg_network")), - kegg_skip_pathview=bool((extra_params or {}).get("keggSkipPathview") or (extra_params or {}).get("skip_pathview")), - enrich_plot_type=(extra_params or {}).get("enrichPlotType") or (extra_params or {}).get("plot_type") or "r_plot", - rnaseq_count_matrix=(extra_params or {}).get("rnaseqCountMatrix", ""), - rnaseq_expression_matrix=(extra_params or {}).get("rnaseqExpressionMatrix", ""), - rnaseq_sample_table=(extra_params or {}).get("rnaseqSampleTable", ""), - fgsea_comparison=(extra_params or {}).get("fgseaComparison", ""), - fgsea_min_size=(extra_params or {}).get("fgseaMinSize", 15), - fgsea_max_size=(extra_params or {}).get("fgseaMaxSize", 500), - fgsea_nperm=(extra_params or {}).get("fgseaNperm", 10000), - fgsea_seed=(extra_params or {}).get("fgseaSeed", 1), - wgcna_min_mean=(extra_params or {}).get("wgcnaMinMean", 1.0), - wgcna_max_genes=(extra_params or {}).get("wgcnaMaxGenes", 8000), - wgcna_min_module_size=(extra_params or {}).get("wgcnaMinModuleSize", 30), - wgcna_merge_cut_height=(extra_params or {}).get("wgcnaMergeCutHeight", 0.25), - wgcna_powers=(extra_params or {}).get("wgcnaPowers", "1,2,3,4,5,6,7,8,9,10,12,14,16,18,20"), - ppi_species=(extra_params or {}).get("ppiSpecies") or (extra_params or {}).get("ppi_species") or "auto", - normalization_species_label=(extra_params or {}).get("normalizationSpeciesLabel") or (extra_params or {}).get("normalization_species_label") or "", - ppi_reference_taxon_id=(extra_params or {}).get("ppiReferenceTaxonId") or (extra_params or {}).get("ppi_reference_taxon_id") or "", - ppi_reference_storage_key=(extra_params or {}).get("ppiReferenceStorageKey") or (extra_params or {}).get("ppi_reference_storage_key") or "", - ppi_reference_selection_mode=(extra_params or {}).get("ppiReferenceSelectionMode") or (extra_params or {}).get("ppi_reference_selection_mode") or "", - ppi_reference_user_confirmed=bool_param("ppiReferenceUserConfirmed", "ppi_reference_user_confirmed", default=False), - dependency_owner=(extra_params or {}).get("dependencyOwner") or (extra_params or {}).get("dependency_owner") or "", - ppi_score=(extra_params or {}).get("ppiScore", 200), - genome_prepare_type=(extra_params or {}).get("genomePrepareType", "auto"), - kaks_neutral_rate=(extra_params or {}).get("kaksNeutralRate", 7.08e-9), - tfbs_upstream_length=(extra_params or {}).get("tfbsUpstreamLength", 2000), - tfbs_motif_db_mode=(extra_params or {}).get("tfbsMotifDbMode", "auto"), - tfbs_fimo_pvalue=(extra_params or {}).get("tfbsFimoPvalue", 0.0001), - tfbs_fimo_qvalue=(extra_params or {}).get("tfbsFimoQvalue", 0.05), - tfbs_background_mode=(extra_params or {}).get("tfbsBackgroundMode", "none"), - tfbs_max_nearest_rank=(extra_params or {}).get("tfbsMaxNearestRank", "order"), - tfbs_top_motifs=(extra_params or {}).get("tfbsTopMotifs", 30), - tfbs_scan_both_strands=bool((extra_params or {}).get("tfbsScanBothStrands", True)), - scope_file=(extra_params or {}).get("scopeFile", "") or (extra_params or {}).get("scope_file", ""), - analysis_route=(extra_params or {}).get("analysisRoute", "") or (extra_params or {}).get("analysis_route", ""), - member_set_id=(extra_params or {}).get("memberSetId", "") or (extra_params or {}).get("member_set_id", ""), - analysis_mode=(extra_params or {}).get("analysisMode", "") or (extra_params or {}).get("analysis_mode", ""), - species_queue=(extra_params or {}).get("speciesQueue", []) or (extra_params or {}).get("species_queue", []), - target_species=target_species, - interspecies_ref_species=interspecies_ref_species, - interspecies_blast_engine=interspecies_blast_engine, - interspecies_plot_mode=interspecies_plot_mode, - ncbi_query=(extra_params or {}).get("ncbiQuery", ""), - ncbi_gene_ids=(extra_params or {}).get("ncbiGeneIds", ""), - ncbi_mode=(extra_params or {}).get("ncbiMode", "gene"), - ncbi_id_type=(extra_params or {}).get("ncbiIdType", "auto"), - ncbi_taxid=(extra_params or {}).get("ncbiTaxid", ""), - ncbi_species=(extra_params or {}).get("ncbiSpecies", ""), - ncbi_seq_types=(extra_params or {}).get("ncbiSeqTypes", "both"), - ncbi_transcript_policy=(extra_params or {}).get("ncbiTranscriptPolicy", "refseq_preferred"), - ncbi_allow_family_download=bool((extra_params or {}).get("ncbiAllowFamilyDownload", False)), - ncbi_email=(extra_params or {}).get("ncbiEmail", ""), - sequence_extract_fasta=(extra_params or {}).get("sequenceExtractFasta", ""), - sequence_extract_ids_file=(extra_params or {}).get("sequenceExtractIdsFile", ""), - sequence_extract_ids=(extra_params or {}).get("sequenceExtractIds", ""), - sequence_extract_seq_type=(extra_params or {}).get("sequenceExtractSeqType", "auto"), - sequence_extract_rename_mode=(extra_params or {}).get("sequenceExtractRenameMode", "keep"), - sequence_extract_clean_headers=bool((extra_params or {}).get("sequenceExtractCleanHeaders", False)), - ) - finally: - try: - os.chdir(original_cwd) - except FileNotFoundError: - os.chdir(self.project_root) - - try: - result_msg, manifest_path = await asyncio.wait_for( - loop.run_in_executor( - None, # 使用默认线程池 - run_stage_with_cwd - ), - timeout=stage_tool.timeout - ) - - # 解析 manifest - manifest = None - if manifest_path and manifest_path.exists(): - try: - with open(manifest_path) as f: - manifest = json.load(f) - except Exception as e: - manifest = {"error": str(e), "path": str(manifest_path)} - - # 判断成功/失败：run_stage 返回的业务消息优先于旧 manifest。 - # 例如 rnaseq_enrich 缺背景文件时可能返回上游 rnaseq_de manifest； - # 如果继续用旧 manifest.status=success 判断，会把业务失败误判为成功。 - manifest_stage = str((manifest or {}).get("stage") or "") - manifest_matches_stage = bool(manifest and manifest_stage == stage_tool.id) - failure_tokens = ( - "❌", - " requires ", - " required ", - " not found", - " missing ", - " failed", - "失败", - "缺少", - "找不到", - "不存在", - ) - lower_msg = result_msg.lower() if isinstance(result_msg, str) else "" - if isinstance(result_msg, str) and ( - result_msg.startswith("❌") or any(token in lower_msg for token in failure_tokens if token.isascii()) - or any(token in result_msg for token in ["失败", "缺少", "找不到", "不存在"]) - ): - success = False - elif isinstance(result_msg, str) and result_msg.startswith("✅"): - success = manifest_matches_stage - else: - success = bool(manifest_matches_stage and manifest.get("status") in ["success", "partial_success"]) - - payload = { - "status": "success" if success else "error", - "message": result_msg, - "stage": stage_tool.id, - "manifest": manifest, - "manifest_path": str(manifest_path) if manifest_path else None, - "requested_cpu": requested_cpu, - "effective_cpu": cpu, - "timestamp": datetime.now().isoformat() - } - if manifest and not manifest_matches_stage: - payload["manifest_stage_mismatch"] = { - "expected": stage_tool.id, - "actual": manifest_stage, - "manifest_path": str(manifest_path) if manifest_path else None, - } - if not success: - payload["error"] = result_msg - return payload - - except asyncio.TimeoutError: - return { - "status": "timeout", - "error": f"Stage {stage_tool.id} timed out after {stage_tool.timeout}s", - "stage": stage_tool.id - } - except Exception as e: - import traceback - return { - "status": "error", - "error": str(e), - "traceback": traceback.format_exc(), - "stage": stage_tool.id - } - - -class StageToolRegistry: - """分析阶段工具注册表""" - - STAGE_DEFINITIONS = [ - # 1. Gene Identification - { - "id": "identify", - "name": "Gene Family Identification", - "script": "run_identify.py", - "category": "identification", - "description": "Identify target-species gene family members using HMM/Pfam plus fixed BLAST evidence where reference family proteins query the target whole-proteome database", - "required_inputs": ["target whole-proteome protein FASTA for HMM search and BLAST database", "HMM/Pfam or reference family protein FASTA used as BLAST query evidence"], - "outputs": ["identified_members.pep.fa", "identified_members.id"], - "parameters": {"cpu": 8, "dom_i_evalue_max": 0.001, "hmm_cov_min": 0.3, "blast_evalue": "1e-05", "blast_identity_min": 30.0, "blast_database_role": "target_proteome", "blast_query_role": "reference_family_pep", "blast_candidate_id_column": "sseqid"}, - "timeout": 7200 - }, - { - "id": "identify_refine", - "name": "Refine Identify Members", - "script": "run_identify_refine.py", - "category": "identification", - "description": "Build curated_members.id and curated_members.pep.fa from a curated member ID list", - "required_inputs": ["id", "protein"], - "outputs": ["curated_members.pep.fa", "curated_members.id", "identify_refine.manifest.json"], - "parameters": {"cpu": 4}, - "timeout": 1800 - }, - # 2. Phylogeny - { - "id": "phylogeny", - "name": "Phylogenetic Analysis", - "script": "run_phylogeny.py", - "category": "phylogeny", - "description": "Build a scoped phylogenetic tree using target/reference/comparative family proteins", - "required_inputs": ["identify final member proteins or scoped family protein FASTA"], - "outputs": ["tree.nwk", "phylogeny.png"], - "parameters": {"cpu": 8, "bootstrap": 1000, "model": "MFP"}, - "timeout": 14400 - }, - { - "id": "reference_classification", - "name": "Reference-guided Subfamily Classification", - "script": "run_reference_classification.py", - "category": "phylogeny", - "description": "Transfer reference-species subfamily labels to target genes from a joint phylogenetic tree, with confidence evidence", - "required_inputs": ["tree", "reference_subfamily"], - "outputs": ["reference_classification.assignment.tsv", "reference_classification.subfamily.tsv", "reference_classification.manifest.json"], - "parameters": {"cpu": 4, "referencePrefix": "At"}, - "timeout": 1800 - }, - { - "id": "subfamily_confirm", - "name": "Confirmed Subfamily Labels", - "script": "run_subfamily_confirm.py", - "category": "phylogeny", - "description": "Freeze user-reviewed subfamily labels for downstream phylogeny, motif, domain, structure and promoter plots", - "required_inputs": ["reference_classification.subfamily.tsv or user-edited confirmed_subfamily table"], - "outputs": ["confirmed_subfamily.tsv", "confirmed_subfamily.review.tsv", "subfamily_confirm.manifest.json"], - "parameters": {"cpu": 4}, - "timeout": 600 - }, - # 3. Motif Analysis - { - "id": "motif", - "name": "Motif Analysis", - "script": "run_motif.py", - "category": "motif", - "description": "Analyze motifs on the scoped protein set used for the current tree", - "required_inputs": ["scoped protein FASTA from phylogeny or identify"], - "outputs": ["motif_results"], - "parameters": {"nmotifs": 10, "min_width": 6, "max_width": 200, "cpu": 8}, - "timeout": 7200 - }, - # 4. Domain Analysis - { - "id": "domain", - "name": "Domain Analysis", - "script": "run_domain.py", - "category": "domain", - "description": "Analyze Pfam domains on the scoped protein set used for the current tree", - "required_inputs": ["scoped protein FASTA", "Pfam-A.hmm"], - "outputs": ["domain_results"], - "parameters": {"cpu": 8, "e_value": 1e-5}, - "timeout": 7200 - }, - # 5. Structure Analysis - { - "id": "structure", - "name": "Structure Analysis", - "script": "run_structure.py", - "category": "structure", - "description": "Draw gene exon/intron/UTR structure for identified family members using target GFF3 files", - "required_inputs": ["final_members.id", "target GFF3"], - "outputs": ["structure_results"], - "parameters": {"cpu": 8}, - "timeout": 7200 - }, - # 6. PEP Analysis (Protein Features) - { - "id": "pep", - "name": "Protein Feature Analysis", - "script": "run_pep.py", - "category": "protein", - "description": "Analyze protein physicochemical properties on the scoped protein set used for the current tree", - "required_inputs": ["scoped protein FASTA"], - "outputs": ["pep_properties.tsv", "pep_report.md"], - "parameters": {"cpu": 8}, - "timeout": 3600 - }, - # 7. Promoter Analysis - { - "id": "promoter", - "name": "Promoter Analysis", - "script": "run_promoter.py", - "category": "promoter", - "description": "Analyze promoter regions for one target species using PlantCARE", - "required_inputs": ["final_members.id", "single target GFF3", "single target genome FASTA"], - "outputs": ["promoter_results"], - "parameters": {"cpu": 8}, - "timeout": 7200 - }, - { - "id": "tfbs", - "name": "TF Binding Site Prediction", - "script": "run_tfbs.py", - "category": "promoter", - "description": "Extract target promoters and scan plant TF motif databases with MEME Suite FIMO", - "required_inputs": ["final_members.id", "single target GFF3", "single target genome FASTA", "plant TF motif database in MEME format"], - "outputs": ["tfbs.sites.tsv", "tfbs.enrichment.tsv", "tfbs.gene_motif_matrix.tsv", "tfbs.manifest.json"], - "parameters": { - "cpu": 4, - "upstream_length": 2000, - "motif_db_mode": "auto", - "fimo_pvalue": 0.0001, - "fimo_qvalue": 0.05, - "background_mode": "none", - }, - "timeout": 14400 - }, - # 7. Synteny Analysis（种内） - { - "id": "synteny", - "name": "Intraspecies synteny (MCScanX)", - "script": "run_synteny.py", - "category": "synteny", - "description": "种内共线性：MCScanX/Circos，单个目标物种内的全蛋白组和染色体区块", - "required_inputs": ["single target pep", "single target gff", "single target genome"], - "outputs": ["synteny_results"], - "parameters": {"cpu": 8}, - "timeout": 7200 - }, - # 8. Karyotype - { - "id": "karyotype", - "name": "Karyotype Analysis", - "script": "run_karyotype.py", - "category": "karyotype", - "description": "Generate per-target-species karyotype visualizations for family members", - "required_inputs": ["final_members.id", "target GFF3", "target genome FASTA"], - "outputs": ["karyotype_results"], - "parameters": {"cpu": 8}, - "timeout": 3600 - }, - # 9. Interspecies Synteny（种间） - { - "id": "interspecies_synteny", - "name": "Interspecies synteny", - "script": "run_interspecies_synteny.py", - "category": "comparative", - "description": "物种间共线性：按 target/reference/comparative scope 进行多物种互比和 JCVI 绘图", - "required_inputs": ["target family pep", "reference family pep", "target/ref full pep", "target/ref gff"], - "outputs": ["interspecies_results"], - "parameters": {"cpu": 8}, - "timeout": 7200 - }, - # 10. Expression Analysis - { - "id": "expression", - "name": "Expression Analysis", - "script": "run_expression.py", - "category": "expression", - "description": "Draw FPKM/TPM expression plots for one selected target species, or record RNA-seq DE-triggered expression plots", - "required_inputs": ["single target member IDs", "FPKM/TPM matrix or GEO accession", "sample.txt"], - "outputs": ["expression_heatmaps", "expression_manifest"], - "parameters": {"cpu": 8}, - "timeout": 3600 - }, - # 11. Ka/Ks Analysis - { - "id": "kaks", - "name": "Ka/Ks Analysis", - "script": "run_kaks_v2.py", - "category": "evolution", - "description": "Calculate Ka/Ks ratios for scoped target-to-reference ortholog pairs using PAML", - "required_inputs": ["target CDS/PEP", "reference CDS/PEP", "ortholog mapping"], - "outputs": ["kaks_results"], - "parameters": {"cpu": 8, "model": "YN00"}, - "timeout": 7200 - }, - # 12. Ortholog Identification - { - "id": "ortholog", - "name": "Ortholog Identification", - "script": "run_ortholog.py", - "category": "evolution", - "description": "Identify orthologous gene-family mappings from selected target species to reference species", - "required_inputs": ["target family pep", "reference family pep"], - "outputs": ["ortholog_pairs", "orthologs", "ortholog_blastp", "ortholog_manifest"], - "parameters": {"cpu": 8, "evalue": 1e-5}, - "timeout": 3600 - }, - # 13. PPI Analysis - { - "id": "ppi", - "name": "PPI Analysis", - "script": "run_ppi.py", - "category": "interaction", - "description": "Protein-protein interaction analysis using STRING (requires ortholog mapping to reference species)", - "required_inputs": ["target member IDs", "ortholog mapping for multi-target/non-reference IDs"], - "outputs": ["ppi_network"], - "parameters": {"species": "ath", "score": 200}, - "timeout": 3600 - }, - # 14. Summary - { - "id": "summary", - "name": "Whole-job Summary", - "script": "run_summary.py", - "category": "report", - "description": "Aggregate all stage manifests, artifacts, metrics and consistency checks by scope", - "required_inputs": ["stage manifests"], - "outputs": ["summary_report.md", "summary.files.tsv", "summary.manifest.json"], - "parameters": {"scope_aware": True}, - "timeout": 1800 - }, - # 15. Genome Prepare - { - "id": "genome_prepare", - "name": "Genome Preparation", - "script": "run_genome_prepare.py", - "category": "preprocessing", - "description": "Prepare genome data for analysis", - "required_inputs": ["genome_fasta", "gff"], - "outputs": ["prepared_genome"], - "parameters": {"cpu": 8}, - "timeout": 3600 - }, - { - "id": "sequence_extract_by_id", - "name": "Utility: Extract/Clean FASTA Sequences", - "script": "run_sequence_extract_by_id.py", - "category": "utility", - "description": "Extract protein/CDS/FASTA records by ID, or clean FASTA headers without an ID list", - "required_inputs": [], - "outputs": ["sequence_extract.fa", "sequence_extract.mapping.tsv", "sequence_extract.manifest.json"], - "parameters": {"cpu": 4}, - "timeout": 1800 - }, - { - "id": "prepare_organize", - "name": "Organize Genome Input Files", - "script": "run_prepare_substep.py", - "category": "utility", - "description": "Organize files by config.xlsx, standardize names, merge reference family FASTA and write species_config.json", - "required_inputs": ["config.xlsx"], - "outputs": ["species_config.json", "standardized_inputs"], - "parameters": {"cpu": 4}, - "timeout": 1800 - }, - { - "id": "prepare_normalize_gff", - "name": "Normalize GFF IDs", - "script": "run_prepare_substep.py", - "category": "utility", - "description": "Normalize JGI/Ensembl/NCBI GFF ID prefixes and attributes", - "required_inputs": ["gff"], - "outputs": ["normalized.gff3"], - "parameters": {"cpu": 4}, - "timeout": 1800 - }, - { - "id": "prepare_longest_transcript", - "name": "Extract Longest Transcript", - "script": "run_prepare_substep.py", - "category": "utility", - "description": "Select longest transcript/CDS per gene and write filtered GFF plus ID lists", - "required_inputs": ["gff"], - "outputs": ["longest.gff3", "longest.ids"], - "parameters": {"cpu": 4}, - "timeout": 3600 - }, - { - "id": "prepare_extract_pep", - "name": "Extract PEP from Genome and GFF", - "script": "run_prepare_substep.py", - "category": "utility", - "description": "Use gffread to extract peptide sequences from genome FASTA and GFF", - "required_inputs": ["genome", "gff"], - "outputs": ["pep.fasta"], - "parameters": {"cpu": 4}, - "timeout": 3600 - }, - { - "id": "prepare_extract_cds", - "name": "Extract CDS from Genome and GFF", - "script": "run_prepare_substep.py", - "category": "utility", - "description": "Use gffread to extract CDS sequences from genome FASTA and GFF", - "required_inputs": ["genome", "gff"], - "outputs": ["cds.fasta"], - "parameters": {"cpu": 4}, - "timeout": 3600 - }, - { - "id": "prepare_filter_fasta", - "name": "Filter FASTA by ID List", - "script": "run_prepare_substep.py", - "category": "utility", - "description": "Use seqtk subseq to extract FASTA records using generated or uploaded ID lists", - "required_inputs": ["fasta", "id_list"], - "outputs": ["subset.fasta"], - "parameters": {"cpu": 4}, - "timeout": 1800 - }, - { - "id": "prepare_simplify_ncbi", - "name": "Simplify NCBI IDs", - "script": "run_prepare_substep.py", - "category": "utility", - "description": "Normalize NCBI CDS headers and simplify NCBI GFF attributes", - "required_inputs": ["gff", "cds"], - "outputs": ["ncbi_simplified.gff3", "ncbi_normalized.cds.fasta"], - "parameters": {"cpu": 4}, - "timeout": 1800 - }, - # 13b. NCBI sequence fetch - { - "id": "ncbi_fetch_sequences", - "name": "NCBI Sequence Fetch", - "script": "run_ncbi_fetch_sequences.py", - "category": "utility", - "description": "Fetch protein and CDS FASTA from NCBI Gene IDs or taxon-scoped gene/family queries", - "required_inputs": ["gene_ids_or_query"], - "outputs": ["protein_fasta", "cds_fasta", "mapping", "unresolved", "ambiguous"], - "parameters": {"cpu": 4}, - "timeout": 1800, - "conda_env": "test4py311" - }, - # 14. RNA-seq QC - { - "id": "rnaseq_qc", - "name": "RNA-seq QC", - "script": "run_rnaseq_qc.py", - "category": "rnaseq", - "description": "RNA-seq quality control", - "required_inputs": ["fastq"], - "outputs": ["qc_report"], - "parameters": {"cpu": 8}, - "timeout": 7200, - "conda_env": "rnaseq" - }, - # 15. RNA-seq Align - { - "id": "rnaseq_align", - "name": "RNA-seq Alignment", - "script": "run_rnaseq_align.py", - "category": "rnaseq", - "description": "Align RNA-seq reads to genome", - "required_inputs": ["fastq", "genome_index"], - "outputs": ["bam_files"], - "parameters": {"cpu": 8}, - "timeout": 14400, - "conda_env": "rnaseq" - }, - # 16. RNA-seq Quant - { - "id": "rnaseq_quant", - "name": "RNA-seq Quantification", - "script": "run_rnaseq_quant.py", - "category": "rnaseq", - "description": "Quantify gene expression from RNA-seq", - "required_inputs": ["bam", "gtf"], - "outputs": ["expression_matrix"], - "parameters": {"cpu": 8}, - "timeout": 7200, - "conda_env": "rnaseq" - }, - # 17. RNA-seq DE - { - "id": "rnaseq_de", - "name": "Differential Expression", - "script": "run_rnaseq_de.py", - "category": "rnaseq", - "description": "Differential expression analysis using DESeq2", - "required_inputs": ["counts", "sample_info"], - "outputs": ["de_results"], - "parameters": {"cpu": 8}, - "timeout": 3600, - "conda_env": "rnaseq" - }, - # 17b. RNA-seq Sample Quality Assessment - { - "id": "rnaseq_sample_qc", - "name": "RNA-seq Sample Quality Assessment", - "script": "run_rnaseq_sample_qc.py", - "category": "rnaseq", - "description": "DESeq2 VST, PCA, sample correlation, sample distance and hierarchical clustering", - "required_inputs": ["count_matrix"], - "outputs": ["pca", "sample_correlation", "sample_clustering"], - "parameters": {"cpu": 4}, - "timeout": 3600, - "conda_env": "rnaseq" - }, - # 17c. RNA-seq WGCNA - { - "id": "rnaseq_wgcna", - "name": "RNA-seq WGCNA", - "script": "run_rnaseq_wgcna.py", - "category": "rnaseq", - "description": "Weighted gene co-expression network analysis from VST/log expression or raw counts", - "required_inputs": ["expression_matrix_or_count_matrix"], - "outputs": ["gene_modules", "module_trait_correlation"], - "parameters": {"max_genes": 8000, "min_module_size": 30, "merge_cut_height": 0.25}, - "timeout": 7200, - "conda_env": "rnaseq" - }, - # 18. Background Annotation - { - "id": "annotation", - "name": "Background Annotation", - "script": "run_annotation.py", - "category": "annotation", - "description": "Generate GO/KO background annotations from protein sequences with local InterProScan and KofamScan", - "required_inputs": ["protein"], - "outputs": ["gene2go.txt", "gene2ko.txt", "interproscan.tsv", "kofamscan.detail.tsv", "annotation.manifest.json", "annotation_go.manifest.json", "annotation_kegg.manifest.json"], - "parameters": {"cpu": 8}, - "timeout": 43200, - "conda_env": "test4py311" - }, - # 19. RNA-seq GO Enrich - { - "id": "rnaseq_go_enrich", - "name": "RNA-seq GO Enrichment", - "script": "run_rnaseq_enrich.py", - "category": "rnaseq", - "description": "GO enrichment analysis for RNA-seq DE genes", - "required_inputs": ["de_results"], - "outputs": ["GO_enrich"], - "parameters": {"cpu": 8, "top_pathways": 20, "enrichType": "go"}, - "timeout": 43200, - "conda_env": "rnaseq" - }, - # 20. RNA-seq KEGG Enrich - { - "id": "rnaseq_kegg_enrich", - "name": "RNA-seq KEGG Enrichment", - "script": "run_rnaseq_enrich.py", - "category": "rnaseq", - "description": "KEGG enrichment analysis for RNA-seq DE genes", - "required_inputs": ["de_results"], - "outputs": ["KEGG_enrich"], - "parameters": {"cpu": 8, "top_pathways": 20, "enrichType": "kegg"}, - "timeout": 43200, - "conda_env": "rnaseq" - }, - # 21. RNA-seq Enrich - { - "id": "rnaseq_enrich", - "name": "Enrichment Analysis", - "script": "run_rnaseq_enrich.py", - "category": "rnaseq", - "description": "GO and KEGG enrichment analysis", - "required_inputs": ["de_results"], - "outputs": ["enrichment_results"], - "parameters": {"cpu": 8, "top_pathways": 20}, - "timeout": 43200, - "conda_env": "rnaseq" - }, - # 22. RNA-seq fgsea - { - "id": "rnaseq_fgsea", - "name": "RNA-seq fgsea", - "script": "run_rnaseq_fgsea.py", - "category": "rnaseq", - "description": "Plant-compatible KEGG fgsea from DESeq2 ranks and local gene2ko annotations", - "required_inputs": ["de_results", "gene2ko"], - "outputs": ["fgsea_kegg"], - "parameters": {"min_size": 15, "max_size": 500, "nperm": 10000}, - "timeout": 3600, - "conda_env": "rnaseq" - }, - # 23. RNA-seq Candidate Family - { - "id": "rnaseq_candidate_family", - "name": "RNA-seq Candidate Family", - "script": "run_rnaseq_candidate_family.py", - "category": "rnaseq", - "description": "Rank candidate gene families by integrating DE genes and functional annotations", - "required_inputs": ["de_results"], - "outputs": ["candidate_family_ranking", "gene_annotations", "enrichment_context", "family_enrichment", "evidence_matrix", "recommendation", "key_genes", "experiment_context", "rag_queries", "rag_references"], - "parameters": {"cpu": 4}, - "timeout": 7200, - "conda_env": "rnaseq" - }, - # Combo Plot - 组合图 - { - "id": "combo", - "name": "Combo Plot", - "script": "run_combo.py", - "category": "visualization", - "description": "Combine phylogeny, motif, structure and domain into one figure", - "required_inputs": ["tree", "motif", "structure", "domain"], - "outputs": ["combo_plot.pdf", "combo_plot.png"], - "parameters": {"cpu": 4}, - "timeout": 3600 - } - ] - - def __init__(self): - self._stages: Dict[str, StageTool] = {} - self._executor = StageToolExecutor() - self._register_all_stages() - - def _register_all_stages(self): - """注册所有分析阶段""" - for defn in self.STAGE_DEFINITIONS: - self.register(StageTool(**defn)) - - def register(self, stage: StageTool) -> None: - """注册分析阶段""" - self._stages[stage.id] = stage - - def get(self, stage_id: str) -> Optional[StageTool]: - """获取分析阶段定义""" - return self._stages.get(stage_id) - - def list_all(self) -> List[StageTool]: - """列出所有分析阶段""" - return list(self._stages.values()) - - def list_by_category(self, category: str) -> List[StageTool]: - """按类别列出分析阶段""" - return [s for s in self._stages.values() if s.category == category] - - async def execute(self, stage_id: str, job_id: str, prefix: str, - extra_params: dict = None, input_files: dict = None) -> dict: - """执行分析阶段""" - stage = self.get(stage_id) - if stage is None: - return { - "status": "error", - "error": f"Stage '{stage_id}' not found", - "available_stages": list(self._stages.keys()) - } - return await self._executor.execute(stage, job_id, prefix, extra_params, input_files) - - def get_execution_order(self, start_stage: str = "identify") -> List[str]: - """获取标准分析流程的执行顺序""" - standard_flow = [ - "identify", - "phylogeny", - "motif", - "domain", - "structure", - "promoter", - "tfbs", - "synteny", - "karyotype", - "interspecies_synteny", - "expression" - ] - if start_stage not in standard_flow: - return standard_flow - idx = standard_flow.index(start_stage) - return standard_flow[idx:] - - -# 全局注册表实例 -stage_registry = StageToolRegistry() - - -def create_stage_registry() -> StageToolRegistry: - """创建新的StageToolRegistry实例""" - return StageToolRegistry()