1323 lines
52 KiB
Python
Executable File
1323 lines
52 KiB
Python
Executable File
|
||
|
||
#!/usr/bin/env python3
|
||
"""
|
||
Domain stage: 蛋白结构域分析 (Pfam HMM 扫描 + 可视化)
|
||
符合 workspace_contract_v1.md 和 manifest_v1.schema.json 契约
|
||
|
||
执行路径:
|
||
1. 单独触发: 独立执行，自动从上游 manifest 查找输入
|
||
2. Run All: 顺序执行，依赖 Identify 和 Phylogeny 阶段产物
|
||
|
||
目录结构适配:
|
||
- 前端上传: input/{job_id}/ (如 input/20260212_143523_2480772_ec5c6d/)
|
||
- 工作目录: workspace/{job_id}/ (如 workspace/20260212_143523_2480772_ec5c6d/)
|
||
"""
|
||
|
||
import argparse
|
||
import json
|
||
import subprocess
|
||
import sys
|
||
import os
|
||
import re
|
||
import uuid
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Dict, List, Optional, Tuple
|
||
import hashlib
|
||
|
||
PROJECT_ROOT = Path(__file__).absolute().parents[2]
|
||
if str(PROJECT_ROOT) not in sys.path:
|
||
sys.path.insert(0, str(PROJECT_ROOT))
|
||
|
||
from gfa_core.gene_family_scope import read_scope, sanitize_token
|
||
from gfa_core.gene_family_stage_context import apply_stage_context, build_stage_context, user_output_prefix_for, write_legacy_manifest_copy
|
||
from gfa_core.mas.effective_members import MemberResolutionError, resolve_effective_members
|
||
|
||
|
||
# ===================== 配置常量 =====================
|
||
|
||
# Pfam 数据库项目内路径（可配置）
|
||
DEFAULT_PFAM_BASE = str(PROJECT_ROOT / "data" / "hmm")
|
||
DEFAULT_PFAM_HMM = "Pfam-A.hmm"
|
||
|
||
# E-value 默认阈值
|
||
DEFAULT_EVALUE = 1e-5
|
||
|
||
# R 脚本路径（相对于项目根目录）
|
||
DEFAULT_RSCRIPT = "scripts/tree_domain.R"
|
||
|
||
CONDA_PREFIX = Path(sys.executable).resolve().parent.parent
|
||
CONDA_BIN = str(CONDA_PREFIX / "bin")
|
||
if CONDA_BIN not in os.environ.get("PATH", "").split(os.pathsep):
|
||
os.environ["PATH"] = CONDA_BIN + os.pathsep + os.environ.get("PATH", "")
|
||
|
||
|
||
# ===================== 工具函数 =====================
|
||
|
||
def display_path(path: Path | str) -> str:
|
||
"""Return the user-facing symlink path instead of resolving data-root links."""
|
||
return str(Path(path).absolute())
|
||
|
||
|
||
def current_input_dir(workspace: Path) -> Path:
|
||
return workspace.parent.parent / "input" / workspace.name
|
||
|
||
|
||
def workspace_root_dir(workspace: Path) -> Path:
|
||
return workspace.parent
|
||
|
||
|
||
def inside_current_job(workspace: Path, path: Path) -> bool:
|
||
input_dir = current_input_dir(workspace)
|
||
try:
|
||
resolved = path.resolve()
|
||
return resolved.is_relative_to(workspace.resolve()) or resolved.is_relative_to(input_dir.resolve())
|
||
except OSError:
|
||
return False
|
||
|
||
|
||
def rebase_current_job_path(workspace: Path, path: Path) -> Optional[Path]:
|
||
"""Map copied legacy workspace/input paths back onto the current job roots."""
|
||
parts = path.parts
|
||
for root_name, current_root in (("workspace", workspace), ("input", current_input_dir(workspace))):
|
||
for idx, part in enumerate(parts):
|
||
if part != root_name:
|
||
continue
|
||
if idx + 1 >= len(parts) or parts[idx + 1] != workspace.name:
|
||
continue
|
||
suffix = Path(*parts[idx + 2:]) if idx + 2 < len(parts) else Path()
|
||
candidate = current_root / suffix
|
||
if candidate.exists():
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def resolve_job_path(workspace: Path, raw_path: str | Path, manifest_path: Optional[Path] = None) -> Optional[Path]:
|
||
"""Resolve a manifest path without trusting stale absolute full_path values."""
|
||
raw_text = str(raw_path or "").strip()
|
||
if not raw_text:
|
||
return None
|
||
path = Path(raw_text)
|
||
if path.is_absolute():
|
||
rebased = rebase_current_job_path(workspace, path)
|
||
if rebased:
|
||
return rebased.absolute()
|
||
if inside_current_job(workspace, path):
|
||
return path.absolute()
|
||
return None
|
||
|
||
candidates: List[Path] = [workspace / path]
|
||
if manifest_path:
|
||
candidates.extend([manifest_path.parent / path, manifest_path.parent / path.name])
|
||
if str(path).startswith("final/") and manifest_path:
|
||
candidates.append(manifest_path.parent / path.name)
|
||
for candidate in candidates:
|
||
if candidate.exists():
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def manifest_artifact_path(workspace: Path, manifest_path: Path, artifact: Dict) -> Optional[Path]:
|
||
if not isinstance(artifact, dict):
|
||
return None
|
||
for key in ("path", "relpath_or_hint", "file", "full_path"):
|
||
resolved = resolve_job_path(workspace, artifact.get(key) or "", manifest_path)
|
||
if resolved:
|
||
return resolved
|
||
return None
|
||
|
||
def get_tool_versions() -> Dict[str, str]:
|
||
"""获取工具版本信息"""
|
||
versions = {}
|
||
|
||
# HMMER 版本
|
||
try:
|
||
r = subprocess.run(["hmmscan", "-h"], capture_output=True, text=True, check=False)
|
||
if r.returncode == 0 and r.stdout:
|
||
first_line = r.stdout.strip().split('\\n')[0]
|
||
versions["hmmer"] = first_line
|
||
except:
|
||
versions["hmmer"] = "not_found"
|
||
|
||
# R 版本
|
||
try:
|
||
r = subprocess.run(["R", "--version"], capture_output=True, text=True, check=False)
|
||
if r.returncode == 0 and r.stdout:
|
||
versions["R"] = r.stdout.split("\\n")[0]
|
||
except:
|
||
versions["R"] = "not_found"
|
||
|
||
# Biopython 版本
|
||
try:
|
||
import Bio
|
||
versions["biopython"] = Bio.__version__
|
||
except:
|
||
versions["biopython"] = "not_found"
|
||
|
||
return versions
|
||
|
||
|
||
def prepare_directories(workspace: Path) -> Dict[str, Path]:
|
||
"""创建Domain阶段的标准目录结构"""
|
||
paths = {
|
||
"analysis_dir": workspace / "domain" / "analysis",
|
||
"final_dir": workspace / "final",
|
||
"logs_dir": workspace / "logs",
|
||
}
|
||
|
||
print("创建目录结构:")
|
||
for name, path in paths.items():
|
||
path.mkdir(parents=True, exist_ok=True)
|
||
print(f" {name}: {path}")
|
||
|
||
return paths
|
||
|
||
|
||
def compute_file_hash(filepath: Path) -> str:
|
||
"""计算文件 SHA256 哈希"""
|
||
sha256_hash = hashlib.sha256()
|
||
try:
|
||
with open(filepath, "rb") as f:
|
||
for byte_block in iter(lambda: f.read(4096), b""):
|
||
sha256_hash.update(byte_block)
|
||
return sha256_hash.hexdigest()
|
||
except:
|
||
return ""
|
||
|
||
|
||
def find_latest_manifest(final_dir: Path, stage: str) -> Optional[Path]:
|
||
"""查找指定阶段的最新 manifest 文件"""
|
||
pattern = f"*.{stage}.manifest.json"
|
||
manifests = list(final_dir.glob(pattern))
|
||
if not manifests:
|
||
return None
|
||
return max(manifests, key=lambda p: p.stat().st_mtime)
|
||
|
||
|
||
def read_manifest(manifest_file: Path) -> Dict:
|
||
"""读取 manifest 文件"""
|
||
with open(manifest_file, 'r') as f:
|
||
return json.load(f)
|
||
|
||
|
||
def resolve_scope_tree_file(ws: Path, prefix: str, scope_data: Optional[Dict], cli_tree: Optional[str] = None) -> Path:
|
||
final_dir = ws / "final"
|
||
if cli_tree:
|
||
tree_path = resolve_job_path(ws, cli_tree) or Path(cli_tree)
|
||
if tree_path.exists() and tree_path.stat().st_size > 0:
|
||
return tree_path.absolute()
|
||
raise FileNotFoundError(f"Specified tree file not found: {cli_tree}")
|
||
|
||
manifest_candidates: List[Path] = []
|
||
scope_id = str((scope_data or {}).get("scope_id") or "").strip()
|
||
if scope_id:
|
||
manifest_candidates.append(final_dir / f"{prefix}.{sanitize_token(scope_id)}.phylogeny.manifest.json")
|
||
else:
|
||
manifest_candidates.append(final_dir / f"{prefix}.phylogeny.manifest.json")
|
||
|
||
for manifest_file in manifest_candidates:
|
||
if not manifest_file.exists() or manifest_file.stat().st_size == 0:
|
||
continue
|
||
try:
|
||
manifest_data = read_manifest(manifest_file)
|
||
except Exception:
|
||
continue
|
||
artifact = (manifest_data.get("artifacts", {}) or {}).get("tree")
|
||
tree_path = None
|
||
if isinstance(artifact, dict):
|
||
tree_path = manifest_artifact_path(ws, manifest_file, artifact)
|
||
elif isinstance(artifact, str):
|
||
tree_path = resolve_job_path(ws, artifact, manifest_file)
|
||
if tree_path and tree_path.exists() and tree_path.stat().st_size > 0:
|
||
return tree_path.absolute()
|
||
|
||
fallback = final_dir / f"{prefix}.tree.nwk"
|
||
if not scope_id and fallback.exists() and fallback.stat().st_size > 0:
|
||
return fallback.absolute()
|
||
raise FileNotFoundError(f"Tree file not found for prefix {prefix}. Please run phylogeny first or specify --tree.")
|
||
|
||
|
||
def count_fasta_records(fasta_file: Path) -> int:
|
||
count = 0
|
||
with open(fasta_file, encoding="utf-8", errors="ignore") as handle:
|
||
for line in handle:
|
||
if line.startswith(">"):
|
||
count += 1
|
||
return count
|
||
|
||
|
||
def ensure_protein_fasta_without_stop_markers(protein_fasta: Path) -> Path:
|
||
"""Use a job-local derived FASTA without '*' stop markers when needed."""
|
||
has_stop = False
|
||
with open(protein_fasta, encoding="utf-8", errors="ignore") as handle:
|
||
for line in handle:
|
||
if not line.startswith(">") and "*" in line:
|
||
has_stop = True
|
||
break
|
||
if not has_stop:
|
||
return protein_fasta
|
||
clean_name = re.sub(r"(\.fa(?:sta)?)$", r".clean\1", protein_fasta.name, flags=re.IGNORECASE)
|
||
if clean_name == protein_fasta.name:
|
||
clean_name = protein_fasta.name + ".clean.fa"
|
||
clean_fasta = protein_fasta.with_name(clean_name)
|
||
if clean_fasta.exists() and clean_fasta.stat().st_mtime >= protein_fasta.stat().st_mtime:
|
||
return clean_fasta
|
||
with open(protein_fasta, "r", encoding="utf-8", errors="ignore") as src, open(clean_fasta, "w", encoding="utf-8") as dst:
|
||
for line in src:
|
||
dst.write(line if line.startswith(">") else line.replace("*", ""))
|
||
print(f"[INFO] Created stop-marker-clean FASTA for hmmscan: {clean_fasta}")
|
||
return clean_fasta
|
||
|
||
|
||
def resolve_scope_protein_fasta(ws: Path, prefix: str, scope_data: Dict, cli_proteins: Optional[str] = None) -> Path:
|
||
if cli_proteins:
|
||
protein_path = resolve_job_path(ws, cli_proteins) or Path(cli_proteins)
|
||
if protein_path.exists() and protein_path.stat().st_size > 0:
|
||
return ensure_protein_fasta_without_stop_markers(protein_path)
|
||
raise FileNotFoundError(f"Specified protein file not found: {protein_path}")
|
||
|
||
final_dir = ws / "final"
|
||
scope_id = str(scope_data.get("scope_id") or "")
|
||
manifest_candidates = []
|
||
if scope_id:
|
||
manifest_candidates.append(final_dir / f"{prefix}.{sanitize_token(scope_id)}.phylogeny.manifest.json")
|
||
else:
|
||
manifest_candidates.append(final_dir / f"{prefix}.phylogeny.manifest.json")
|
||
|
||
for manifest_file in manifest_candidates:
|
||
if not manifest_file.exists():
|
||
continue
|
||
try:
|
||
manifest_data = read_manifest(manifest_file)
|
||
except Exception:
|
||
continue
|
||
fasta_text = manifest_data.get("parameters", {}).get("scope_input_fasta")
|
||
if fasta_text:
|
||
fasta_path = resolve_job_path(ws, fasta_text, manifest_file)
|
||
if fasta_path and fasta_path.exists() and fasta_path.stat().st_size > 0:
|
||
print(f"[INFO] Found scoped protein FASTA from phylogeny manifest: {fasta_path}")
|
||
return ensure_protein_fasta_without_stop_markers(fasta_path)
|
||
|
||
raise FileNotFoundError(
|
||
"Cannot find scoped protein FASTA. Run phylogeny with the same scope first, "
|
||
"or provide --proteins explicitly."
|
||
)
|
||
|
||
|
||
def resolve_project_root(ws: Path) -> Path:
|
||
"""
|
||
解析项目根目录
|
||
从 workspace/{job_id}/ 向上回溯到项目根目录
|
||
假设结构: .../bba/workspace/{job_id}/ -> .../bba/
|
||
"""
|
||
# ws 是 workspace/{job_id}/，parent 是 workspace/，parent.parent 才是项目根目录
|
||
return ws.parent.parent
|
||
|
||
|
||
def resolve_subfamily_file(ws: Path, prefix: str, job_id: str, cli_subfamily: Optional[str] = None, output_prefix: Optional[str] = None) -> Optional[Path]:
|
||
"""
|
||
解析亚家族文件路径
|
||
命名规则: {prefix}.confirmed_subfamily.tsv (如 Ft.SPL.confirmed_subfamily.tsv)
|
||
优先级: final/ > input/{job_id}/ (项目根目录)
|
||
|
||
实际目录结构：
|
||
- workspace/{job_id}/final/ <- 阶段产物
|
||
- input/{job_id}/ <- 前端上传（项目根目录下）
|
||
"""
|
||
# 方式一：命令行指定
|
||
if cli_subfamily:
|
||
if str(cli_subfamily).strip().lower() in {"none", "skip", "disabled"}:
|
||
print("[INFO] Subfamily coloring explicitly disabled")
|
||
return None
|
||
subfamily_path = resolve_job_path(ws, cli_subfamily) or Path(cli_subfamily)
|
||
if subfamily_path.exists():
|
||
print(f"[INFO] Using specified subfamily file: {subfamily_path}")
|
||
return subfamily_path
|
||
else:
|
||
print(f"[WARNING] Specified subfamily file not found: {cli_subfamily}")
|
||
return None
|
||
|
||
# 自动查找只接受 confirmed_subfamily；prediction/geneID 必须由 --subfamily 显式传入。
|
||
names = []
|
||
for base in [output_prefix, prefix]:
|
||
if not base or base in names:
|
||
continue
|
||
names.extend([
|
||
f"{base}.confirmed_subfamily.tsv",
|
||
])
|
||
subfamily_name = f"{prefix}.confirmed_subfamily.tsv"
|
||
|
||
# 项目根目录（workspace 的父目录的父目录）
|
||
project_root = resolve_project_root(ws)
|
||
|
||
# 搜索路径优先级：
|
||
search_paths = []
|
||
|
||
# 1. final/ 目录（阶段产物，优先级最高）
|
||
final_dir = ws / "final"
|
||
if final_dir.exists():
|
||
search_paths.extend(final_dir / name for name in names)
|
||
|
||
# 2. 项目根目录的 input/{job_id}/ 目录（前端上传的实际位置）
|
||
# 修正：input/ 直接在项目根目录下，不是 workspace/ 下
|
||
search_paths.extend(
|
||
project_root / "input" / job_id / name
|
||
for name in names
|
||
|
||
)
|
||
|
||
# 3. workspace 下的 input/ 目录（兼容性检查，某些旧版本可能用这里）
|
||
workspace_input = ws / "input"
|
||
if workspace_input.exists():
|
||
search_paths.extend(workspace_input / name for name in names)
|
||
|
||
for candidate in search_paths:
|
||
if candidate.exists():
|
||
print(f"[INFO] Found subfamily file: {candidate}")
|
||
return candidate
|
||
|
||
print(f"[INFO] No subfamily file found ({subfamily_name})")
|
||
print(f"[DEBUG] Searched paths:")
|
||
for p in search_paths:
|
||
print(f" - {p} (exists: {p.exists()})")
|
||
return None
|
||
|
||
def prepare_pfam_database(domain_dir: Path, pfam_base: str = DEFAULT_PFAM_BASE) -> Tuple[Path, bool, str]:
|
||
"""
|
||
准备 Pfam 数据库（软连接方式）
|
||
|
||
Returns:
|
||
(pfam_hmm_path, success, error_message)
|
||
"""
|
||
pfam_source = Path(pfam_base) / DEFAULT_PFAM_HMM
|
||
|
||
# 检查源文件是否存在
|
||
if not pfam_source.exists():
|
||
return None, False, f"Pfam source not found: {pfam_source}"
|
||
|
||
# 检查索引文件
|
||
index_extensions = ['.h3f', '.h3i', '.h3m', '.h3p']
|
||
missing_indices = []
|
||
|
||
for ext in index_extensions:
|
||
idx_file = Path(pfam_base) / f"{DEFAULT_PFAM_HMM}{ext}"
|
||
if not idx_file.exists():
|
||
missing_indices.append(str(idx_file))
|
||
|
||
if missing_indices:
|
||
return None, False, f"Missing Pfam index files: {', '.join(missing_indices)}. Run 'hmmpress {pfam_source}' first."
|
||
|
||
# 创建软连接
|
||
pfam_link = domain_dir / DEFAULT_PFAM_HMM
|
||
|
||
try:
|
||
if pfam_link.exists() or pfam_link.is_symlink():
|
||
pfam_link.unlink()
|
||
os.symlink(pfam_source, pfam_link)
|
||
print(f"[INFO] Linked Pfam database: {pfam_source} -> {pfam_link}")
|
||
|
||
# 链接索引文件
|
||
for ext in index_extensions:
|
||
src_idx = Path(pfam_base) / f"{DEFAULT_PFAM_HMM}{ext}"
|
||
link_idx = domain_dir / f"{DEFAULT_PFAM_HMM}{ext}"
|
||
if link_idx.exists() or link_idx.is_symlink():
|
||
link_idx.unlink()
|
||
os.symlink(src_idx, link_idx)
|
||
|
||
print(f"[INFO] Linked {len(index_extensions)} index files")
|
||
return pfam_link, True, ""
|
||
|
||
except Exception as e:
|
||
return None, False, f"Failed to create Pfam symlinks: {str(e)}"
|
||
|
||
|
||
def run_hmmscan(pfam_hmm: Path, protein_fasta: Path, domain_dir: Path,
|
||
output_prefix: str = "", cpu: int = 8, evalue: float = DEFAULT_EVALUE) -> Tuple[bool, Dict]:
|
||
"""
|
||
运行 hmmscan 进行结构域搜索
|
||
|
||
Returns:
|
||
(success, result_dict)
|
||
"""
|
||
basename = sanitize_token(output_prefix) if output_prefix else "results"
|
||
domtblout = domain_dir / f"{basename}.results.domtblout"
|
||
tblout = domain_dir / f"{basename}.results.tblout"
|
||
output_log = domain_dir / f"{basename}.results.out"
|
||
|
||
cmd = [
|
||
"hmmscan",
|
||
"--cpu", str(cpu),
|
||
"--domtblout", str(domtblout),
|
||
"--tblout", str(tblout),
|
||
"-o", str(output_log),
|
||
"-E", str(evalue), # 报告阈值
|
||
"--domE", str(evalue), # 结构域报告阈值
|
||
str(pfam_hmm),
|
||
str(protein_fasta)
|
||
]
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
|
||
start_time = datetime.now()
|
||
|
||
try:
|
||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||
end_time = datetime.now()
|
||
|
||
success = result.returncode == 0
|
||
elapsed = (end_time - start_time).total_seconds()
|
||
|
||
# 检查输出文件
|
||
domtblout_exists = domtblout.exists() and domtblout.stat().st_size > 0
|
||
|
||
if success and not domtblout_exists:
|
||
success = False
|
||
error_msg = "hmmscan returned 0 but domtblout is empty"
|
||
elif not success:
|
||
error_msg = result.stderr[-2000:] if result.stderr else "hmmscan failed"
|
||
else:
|
||
error_msg = ""
|
||
|
||
return success, {
|
||
"domtblout": domtblout if domtblout.exists() else None,
|
||
"tblout": tblout if tblout.exists() else None,
|
||
"output_log": output_log if output_log.exists() else None,
|
||
"returncode": result.returncode,
|
||
"stderr": result.stderr[-2000:] if result.stderr else "",
|
||
"elapsed_sec": elapsed,
|
||
"error": error_msg
|
||
}
|
||
|
||
except Exception as e:
|
||
return False, {
|
||
"domtblout": None,
|
||
"tblout": None,
|
||
"output_log": None,
|
||
"returncode": -1,
|
||
"stderr": str(e),
|
||
"elapsed_sec": 0,
|
||
"error": str(e)
|
||
}
|
||
|
||
|
||
def parse_domtblout(domtblout: Path, evalue_cutoff: float = DEFAULT_EVALUE) -> Tuple[List[Dict], Dict]:
|
||
"""
|
||
解析 hmmscan domtblout 文件
|
||
|
||
Returns:
|
||
(domains_list, stats_dict)
|
||
"""
|
||
domains = []
|
||
stats = {
|
||
"total_hits": 0,
|
||
"significant_hits": 0,
|
||
"unique_proteins": set(),
|
||
"unique_domains": set(),
|
||
"protein_domain_counts": {}
|
||
}
|
||
|
||
if not domtblout.exists():
|
||
return [], stats
|
||
|
||
with open(domtblout, 'r') as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line or line.startswith('#'):
|
||
continue
|
||
|
||
stats["total_hits"] += 1
|
||
|
||
# domtblout 格式（空格分隔，22列）
|
||
parts = line.split()
|
||
if len(parts) < 22:
|
||
continue
|
||
|
||
try:
|
||
# 解析关键字段
|
||
target_name = parts[0] # Domain name
|
||
t_accession = parts[1] # Domain accession
|
||
tlen = int(parts[2]) # Domain length
|
||
query_name = parts[3] # Protein ID
|
||
q_accession = parts[4] # Protein accession
|
||
qlen = int(parts[5]) # Protein length
|
||
|
||
# E-values and scores
|
||
E_full = float(parts[6]) # Full sequence E-value
|
||
score_full = float(parts[7]) # Full sequence score
|
||
|
||
# Domain coordinates
|
||
dom_index = int(parts[9]) # Domain index
|
||
dom_of = int(parts[10]) # Total domains
|
||
|
||
c_evalue = float(parts[11]) # Conditional E-value
|
||
i_evalue = float(parts[12]) # Independent E-value
|
||
dom_score = float(parts[13]) # Domain score
|
||
|
||
hmm_from = int(parts[15]) # HMM start
|
||
hmm_to = int(parts[16]) # HMM end
|
||
ali_from = int(parts[17]) # Alignment start (protein)
|
||
ali_to = int(parts[18]) # Alignment end (protein)
|
||
env_from = int(parts[19]) # Envelope start
|
||
env_to = int(parts[20]) # Envelope end
|
||
acc = float(parts[21]) # Average posterior probability
|
||
|
||
# 应用 E-value 过滤（使用 i-Evalue，更严格）
|
||
if i_evalue > evalue_cutoff:
|
||
continue
|
||
|
||
stats["significant_hits"] += 1
|
||
stats["unique_proteins"].add(query_name)
|
||
stats["unique_domains"].add(target_name)
|
||
|
||
if query_name not in stats["protein_domain_counts"]:
|
||
stats["protein_domain_counts"][query_name] = 0
|
||
stats["protein_domain_counts"][query_name] += 1
|
||
|
||
domains.append({
|
||
"target_name": target_name,
|
||
"target_accession": t_accession,
|
||
"query_name": query_name,
|
||
"query_length": qlen,
|
||
"i_evalue": i_evalue,
|
||
"domain_score": dom_score,
|
||
"hmm_from": hmm_from,
|
||
"hmm_to": hmm_to,
|
||
"ali_from": ali_from,
|
||
"ali_to": ali_to,
|
||
"env_from": env_from,
|
||
"env_to": env_to,
|
||
"accuracy": acc
|
||
})
|
||
|
||
except (ValueError, IndexError) as e:
|
||
continue
|
||
|
||
# 转换 set 为 count
|
||
stats["unique_proteins"] = len(stats["unique_proteins"])
|
||
stats["unique_domains"] = len(stats["unique_domains"])
|
||
|
||
return domains, stats
|
||
|
||
|
||
def run_r_plotting(tree_file: Path, subfamily_file: Optional[Path],
|
||
domtblout_file: Path, output_prefix: Path,
|
||
rscript_path: Optional[Path] = None,
|
||
evalue_cutoff: float = DEFAULT_EVALUE,
|
||
coverage_cutoff: Optional[float] = None) -> Dict:
|
||
"""
|
||
运行 R 绘图脚本 tree_domain.R
|
||
|
||
Args:
|
||
tree_file: 系统发育树文件 (Newick)
|
||
subfamily_file: 亚家族分类文件 (可选)
|
||
domtblout_file: hmmscan domtblout 文件
|
||
output_prefix: 输出文件前缀（不含扩展名）
|
||
rscript_path: R 脚本路径
|
||
evalue_cutoff: E-value 阈值
|
||
coverage_cutoff: 覆盖度阈值 (百分比，如 50 表示 50%)
|
||
|
||
Returns:
|
||
Dict with success, output_files, etc.
|
||
"""
|
||
if rscript_path is None:
|
||
# 从项目根目录推断
|
||
rscript_path = Path(__file__).parent.parent.parent / "scripts" / "tree_domain.R"
|
||
|
||
if not rscript_path.exists():
|
||
return {
|
||
"success": False,
|
||
"error": f"R script not found: {rscript_path}"
|
||
}
|
||
|
||
# 构建命令 - 适配 tree_domain.R 的参数格式
|
||
# 用法: Rscript tree_domain.R <treefile> <subfamily_file> <domtblout_file> [evalue] [coverage] [output_prefix]
|
||
cmd = [
|
||
"Rscript", str(rscript_path),
|
||
str(tree_file),
|
||
str(subfamily_file) if subfamily_file else "NONE", # R 脚本需要处理 NONE
|
||
str(domtblout_file),
|
||
str(evalue_cutoff),
|
||
str(coverage_cutoff) if coverage_cutoff else "", # 空字符串表示不启用
|
||
str(output_prefix)
|
||
]
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
|
||
start_time = datetime.now()
|
||
|
||
try:
|
||
result = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=600)
|
||
end_time = datetime.now()
|
||
|
||
success = result.returncode == 0
|
||
|
||
# 检查输出文件（R 脚本输出: {prefix}.pdf 和 {prefix}.png）
|
||
output_files = []
|
||
for ext in ['.pdf', '.png']:
|
||
f = Path(f"{output_prefix}{ext}")
|
||
if f.exists():
|
||
output_files.append(str(f))
|
||
|
||
return {
|
||
"success": success and len(output_files) > 0,
|
||
"returncode": result.returncode,
|
||
"stdout": result.stdout[-2000:],
|
||
"stderr": result.stderr[-2000:],
|
||
"output_files": output_files,
|
||
"elapsed_sec": (end_time - start_time).total_seconds()
|
||
}
|
||
|
||
except subprocess.TimeoutExpired:
|
||
return {
|
||
"success": False,
|
||
"error": "R plotting timeout (>600s)",
|
||
"elapsed_sec": 600
|
||
}
|
||
except Exception as e:
|
||
return {
|
||
"success": False,
|
||
"error": str(e),
|
||
"elapsed_sec": 0
|
||
}
|
||
|
||
|
||
def generate_report(prefix: str, metrics: Dict, parameters: Dict,
|
||
output_file: Path, artifacts: Dict, plot_success: bool) -> None:
|
||
"""生成 domain_report.md 报告"""
|
||
|
||
report_lines = [
|
||
f"# Domain Analysis Report: {prefix}",
|
||
"",
|
||
f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
|
||
f"**Status:** {'Success' if plot_success else 'Partial Success (Plotting Failed)'}",
|
||
"",
|
||
"## Summary",
|
||
"",
|
||
f"- **Total Proteins:** {metrics['proteins_total']}",
|
||
f"- **Proteins with Domains:** {metrics['proteins_with_domains']} ({metrics['domain_coverage_rate']*100:.1f}%)",
|
||
f"- **Total Domain Hits:** {metrics['total_domain_hits']}",
|
||
f"- **Significant Hits (E < {parameters.get('evalue_cutoff', DEFAULT_EVALUE)}):** {metrics['significant_domain_hits']}",
|
||
f"- **Unique Domain Types:** {metrics['unique_domain_types']}",
|
||
f"- **Average Domains per Protein:** {metrics['avg_domains_per_protein']:.2f}",
|
||
"",
|
||
"## Domain Statistics",
|
||
"",
|
||
]
|
||
|
||
# 添加 top domains 信息
|
||
if "top_domains" in metrics and metrics["top_domains"]:
|
||
report_lines.append("### Top Domains by Frequency")
|
||
report_lines.append("")
|
||
for domain, count in metrics["top_domains"]:
|
||
report_lines.append(f"- **{domain}:** {count} hits")
|
||
report_lines.append("")
|
||
|
||
report_lines.extend([
|
||
"## Parameters",
|
||
"",
|
||
f"- **E-value Cutoff:** {parameters.get('evalue_cutoff', DEFAULT_EVALUE)}",
|
||
f"- **Coverage Cutoff:** {parameters.get('coverage_cutoff', 'Not applied')}",
|
||
f"- **CPU Cores:** {parameters.get('cpu', 'N/A')}",
|
||
f"- **Pfam Database:** {parameters.get('pfam_hmm', 'N/A')}",
|
||
f"- **Protein Source:** {parameters.get('protein_source', 'N/A')}",
|
||
f"- **Tree File:** {parameters.get('tree_file', 'N/A')}",
|
||
f"- **Subfamily File:** {parameters.get('subfamily_file', 'Not provided')}",
|
||
"",
|
||
"## Output Files",
|
||
""
|
||
])
|
||
|
||
for key, artifact in artifacts.items():
|
||
if isinstance(artifact, dict):
|
||
report_lines.append(f"- **{key}:** `{artifact.get('path', 'N/A')}` ({artifact.get('type', 'unknown')})")
|
||
|
||
report_lines.extend([
|
||
"",
|
||
"## Visualization",
|
||
""
|
||
])
|
||
|
||
if plot_success:
|
||
report_lines.append("Protein domain architecture visualization was successfully generated with three panels:")
|
||
report_lines.append("1. **Phylogenetic Tree** - Gene relationships with subfamily coloring")
|
||
report_lines.append("2. **Subfamily Bars** - Visual connection between tree and domains")
|
||
report_lines.append("3. **Domain Architecture** - Pfam domain positions along protein sequences")
|
||
report_lines.append("")
|
||
report_lines.append("Output formats:")
|
||
report_lines.append("- **PDF**: Vector graphics for publication")
|
||
report_lines.append("- **PNG**: 300 DPI raster image for presentation")
|
||
else:
|
||
report_lines.append("**Warning:** Domain visualization failed. See logs for details.")
|
||
report_lines.append("")
|
||
report_lines.append("Raw domain data is available in the domtblout file for manual analysis.")
|
||
|
||
report_lines.extend([
|
||
"",
|
||
"---",
|
||
"*Generated by Gene Family Analysis Pipeline - Domain Stage*"
|
||
])
|
||
|
||
with open(output_file, 'w') as f:
|
||
f.write('\\n'.join(report_lines))
|
||
|
||
print(f"[INFO] Report generated: {output_file}")
|
||
|
||
|
||
def write_manifest(manifest: Dict, manifest_file: Path) -> None:
|
||
"""写入 manifest 文件"""
|
||
with open(manifest_file, 'w') as f:
|
||
json.dump(manifest, f, indent=2)
|
||
print(f"[INFO] Manifest: {manifest_file}")
|
||
|
||
|
||
def write_failure_manifest(workspace: Path, final_dir: Path, prefix: str,
|
||
job_id: str, run_id: str, start_time: datetime,
|
||
error_msg: str, partial_artifacts: Dict = None) -> None:
|
||
"""写入失败状态的 manifest"""
|
||
end_time = datetime.now()
|
||
|
||
manifest = {
|
||
"manifest_version": "1.0",
|
||
"job_id": job_id,
|
||
"stage": "domain",
|
||
"run_id": run_id,
|
||
"prefix": prefix,
|
||
"timestamp": end_time.isoformat(),
|
||
"status": "failed",
|
||
"start_time": start_time.isoformat(),
|
||
"end_time": end_time.isoformat(),
|
||
"elapsed_sec": (end_time - start_time).total_seconds(),
|
||
"cmd": [str(Path(__file__).name)],
|
||
"workspace_root": display_path(workspace_root_dir(workspace)),
|
||
"workspace": display_path(workspace),
|
||
"input_dir": display_path(current_input_dir(workspace)),
|
||
"artifacts": partial_artifacts or {},
|
||
"parameters": {
|
||
"stage": "domain",
|
||
"error": error_msg
|
||
},
|
||
"metrics": {
|
||
"error_type": "execution_failed"
|
||
}
|
||
}
|
||
|
||
manifest_file = final_dir / f"{prefix}.domain.manifest.json"
|
||
write_manifest(manifest, manifest_file)
|
||
print(f"[ERROR] Failure manifest written: {manifest_file}")
|
||
|
||
|
||
def quarantine_stale_unscoped_failure(final_dir: Path, prefix: str, stage: str, canonical_manifest: Path) -> None:
|
||
"""Move stale unscoped failure manifests aside after a scoped success is available."""
|
||
stale_manifest = final_dir / f"{prefix}.{stage}.manifest.json"
|
||
if stale_manifest == canonical_manifest or not stale_manifest.exists():
|
||
return
|
||
try:
|
||
with open(stale_manifest, "r", encoding="utf-8") as handle:
|
||
stale_data = json.load(handle)
|
||
except Exception:
|
||
return
|
||
if stale_data.get("status") != "failed":
|
||
return
|
||
if stale_data.get("scope_id") or stale_data.get("stage_context"):
|
||
return
|
||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
archived = final_dir / f"{prefix}.{stage}.stale_failed.{timestamp}.json"
|
||
stale_manifest.rename(archived)
|
||
print(f"[INFO] Quarantined stale unscoped failure manifest: {archived}")
|
||
|
||
|
||
def calculate_domain_metrics(domains: List[Dict], total_proteins: int,
|
||
hmmscan_stats: Dict) -> Dict:
|
||
"""计算结构域相关的 metrics"""
|
||
|
||
# 统计每个 domain 的频率
|
||
domain_freq = {}
|
||
for d in domains:
|
||
name = d["target_name"]
|
||
domain_freq[name] = domain_freq.get(name, 0) + 1
|
||
|
||
# Top 10 domains
|
||
top_domains = sorted(domain_freq.items(), key=lambda x: x[1], reverse=True)[:10]
|
||
|
||
# 计算有 domain 的蛋白数量
|
||
proteins_with_domains = len(set(d["query_name"] for d in domains))
|
||
|
||
metrics = {
|
||
"proteins_total": total_proteins,
|
||
"proteins_with_domains": proteins_with_domains,
|
||
"domain_coverage_rate": round(proteins_with_domains / max(total_proteins, 1), 4),
|
||
"total_domain_hits": hmmscan_stats.get("total_hits", 0),
|
||
"significant_domain_hits": len(domains),
|
||
"unique_domain_types": len(domain_freq),
|
||
"avg_domains_per_protein": round(len(domains) / max(proteins_with_domains, 1), 2),
|
||
"top_domains": top_domains
|
||
}
|
||
|
||
return metrics
|
||
|
||
|
||
# ===================== 主函数 =====================
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(
|
||
description='Domain stage - Protein domain analysis (Pfam HMM scan + visualization)'
|
||
)
|
||
parser.add_argument('--prefix', required=True,
|
||
help='Analysis prefix (e.g., Ft.SPL)')
|
||
parser.add_argument('--workspace', required=True,
|
||
help='Workspace directory (e.g., workspace/20260212_143523_2480772_ec5c6d)')
|
||
parser.add_argument('--proteins',
|
||
help='Protein FASTA file (auto-detect from Identify manifest if not specified)')
|
||
parser.add_argument('--scope-file',
|
||
help='Gene-family scope JSON from resolve_scope.py')
|
||
parser.add_argument('--analysis-route', default='', help='Explicit analysis route identity for manifest/status')
|
||
parser.add_argument('--member-set-id', default='', help='Explicit member-set identity for manifest/status')
|
||
parser.add_argument('--tree',
|
||
help='Tree file (auto-detect from Phylogeny manifest if not specified)')
|
||
parser.add_argument('--subfamily',
|
||
help='Subfamily classification file (explicit path; auto-detect only uses confirmed_subfamily.tsv)')
|
||
parser.add_argument('--pfam-base',
|
||
default=DEFAULT_PFAM_BASE,
|
||
help=f'Pfam database base directory (default: {DEFAULT_PFAM_BASE})')
|
||
parser.add_argument('--evalue', type=float,
|
||
default=DEFAULT_EVALUE,
|
||
help=f'E-value cutoff (default: {DEFAULT_EVALUE})')
|
||
parser.add_argument('--coverage', type=float,
|
||
help='Coverage cutoff percentage (optional, e.g., 50 for 50%%)')
|
||
parser.add_argument('--cpu', type=int, default=8,
|
||
help='CPU cores for hmmscan (default: 8)')
|
||
parser.add_argument('--rscript',
|
||
help='Path to tree_domain.R script (auto-detect if not specified)')
|
||
parser.add_argument('--log-file',
|
||
help='Log file path')
|
||
parser.add_argument('--run-id',
|
||
help='Run ID (auto-generated if not provided)')
|
||
|
||
args = parser.parse_args()
|
||
|
||
run_id = args.run_id or str(uuid.uuid4())[:8]
|
||
job_id = Path(args.workspace).name
|
||
|
||
start_time = datetime.now()
|
||
ws = Path(args.workspace)
|
||
|
||
# 验证 workspace 路径
|
||
if not ws.exists():
|
||
print(f"[ERROR] Workspace directory not found: {ws}")
|
||
sys.exit(1)
|
||
|
||
# 1. 创建标准目录结构（契约要求）
|
||
domain_dir = ws / "domain" / "analysis"
|
||
domain_dir.mkdir(parents=True, exist_ok=True)
|
||
final_dir = ws / "final"
|
||
final_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
scope_data: Optional[Dict] = None
|
||
scope_file: Optional[Path] = None
|
||
if args.scope_file:
|
||
scope_file = Path(args.scope_file)
|
||
if not scope_file.exists():
|
||
error_msg = f"Scope file not found: {args.scope_file}"
|
||
print(f"[ERROR] {error_msg}")
|
||
write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, error_msg)
|
||
sys.exit(1)
|
||
scope_data = read_scope(scope_file)
|
||
scope_id = str((scope_data or {}).get("scope_id") or "").strip()
|
||
output_prefix = user_output_prefix_for(
|
||
"domain",
|
||
args.prefix,
|
||
scope_data or {},
|
||
analysis_route=args.analysis_route,
|
||
member_set_id=args.member_set_id,
|
||
) if scope_id else args.prefix
|
||
|
||
print(f"[INFO] Domain stage started for {args.prefix}")
|
||
print(f"[INFO] Job ID: {job_id}, Run ID: {run_id}")
|
||
print(f"[INFO] Workspace: {display_path(ws)}")
|
||
|
||
# 2. 解析输入文件（支持命令行指定或自动查找）
|
||
|
||
# 2.1 蛋白序列文件
|
||
if scope_data:
|
||
try:
|
||
protein_file = resolve_scope_protein_fasta(ws, args.prefix, scope_data, args.proteins)
|
||
except FileNotFoundError as e:
|
||
error_msg = str(e)
|
||
print(f"[ERROR] {error_msg}")
|
||
write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, error_msg)
|
||
sys.exit(1)
|
||
elif args.proteins:
|
||
protein_file = resolve_job_path(ws, args.proteins) or Path(args.proteins)
|
||
if not protein_file.exists():
|
||
error_msg = f"Specified protein file not found: {protein_file}"
|
||
print(f"[ERROR] {error_msg}")
|
||
write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, error_msg)
|
||
sys.exit(1)
|
||
else:
|
||
phase = "subfamily_evidence_required" if args.analysis_route == "subfamily_evidence" else "post_confirm_required"
|
||
try:
|
||
resolved = resolve_effective_members(
|
||
workspace=ws,
|
||
final_dir=final_dir,
|
||
input_dir=current_input_dir(ws),
|
||
prefix=args.prefix,
|
||
phase=phase, # type: ignore[arg-type]
|
||
require="pep",
|
||
member_set_id=args.member_set_id,
|
||
)
|
||
if resolved.path_pep:
|
||
protein_file = resolved.path_pep
|
||
print(f"[INFO] Protein file from {resolved.source_kind}: {protein_file}")
|
||
else:
|
||
protein_file = Path("")
|
||
except MemberResolutionError as e:
|
||
if args.analysis_route == "subfamily_evidence":
|
||
error_msg = f"Complete identify member review before Domain subfamily evidence: {e}"
|
||
else:
|
||
error_msg = f"Complete subfamily confirmation before regular Domain analysis: {e}"
|
||
print(f"[ERROR] {error_msg}")
|
||
write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, error_msg)
|
||
sys.exit(1)
|
||
if protein_file.exists() and protein_file.stat().st_size > 0:
|
||
protein_file = ensure_protein_fasta_without_stop_markers(protein_file)
|
||
print(f"[INFO] Protein file: {protein_file}")
|
||
protein_record_count = count_fasta_records(protein_file)
|
||
else:
|
||
protein_file = Path("")
|
||
|
||
if scope_data or args.proteins:
|
||
protein_file = ensure_protein_fasta_without_stop_markers(protein_file)
|
||
print(f"[INFO] Protein file: {protein_file}")
|
||
protein_record_count = count_fasta_records(protein_file)
|
||
|
||
if not scope_data and not args.proteins and not protein_file:
|
||
candidates = [final_dir / f"{args.prefix}.final_members.pep.fa"]
|
||
if "." in args.prefix:
|
||
species, family = args.prefix.split(".", 1)
|
||
if species not in {"Target", "Multi", "Reference", "Comparative"} and family:
|
||
candidates.append(final_dir / f"Target.{species}.{family}.final_members.pep.fa")
|
||
# 从 Identify manifest 查找
|
||
identify_manifest = find_latest_manifest(final_dir, "identify")
|
||
if identify_manifest:
|
||
try:
|
||
manifest_data = read_manifest(identify_manifest)
|
||
artifacts = manifest_data.get("artifacts", {}) if isinstance(manifest_data.get("artifacts", {}), dict) else {}
|
||
for key in ("final_members", "final_proteins", "proteins"):
|
||
item = artifacts.get(key)
|
||
protein_path = manifest_artifact_path(ws, identify_manifest, item) if isinstance(item, dict) else None
|
||
if protein_path:
|
||
candidates.append(protein_path)
|
||
except Exception:
|
||
pass
|
||
protein_file = next((path for path in candidates if path.exists() and path.stat().st_size > 0), candidates[0])
|
||
|
||
if not protein_file.exists():
|
||
error_msg = "Protein file not found. Checked: " + ", ".join(str(path) for path in candidates) + ". Please run Identify stage first or specify --proteins."
|
||
print(f"[ERROR] {error_msg}")
|
||
write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, error_msg)
|
||
sys.exit(1)
|
||
protein_file = ensure_protein_fasta_without_stop_markers(protein_file)
|
||
print(f"[INFO] Protein file: {protein_file}")
|
||
protein_record_count = count_fasta_records(protein_file)
|
||
|
||
# 2.2 树文件
|
||
try:
|
||
tree_file = resolve_scope_tree_file(ws, args.prefix, scope_data, args.tree)
|
||
except FileNotFoundError as e:
|
||
error_msg = str(e)
|
||
print(f"[ERROR] {error_msg}")
|
||
write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, error_msg)
|
||
sys.exit(1)
|
||
|
||
print(f"[INFO] Tree file: {tree_file}")
|
||
|
||
# 2.3 亚家族文件（可选，但 R 脚本需要）
|
||
# 自动查找只接受 confirmed_subfamily；prediction/geneID 必须显式传入
|
||
subfamily_file = resolve_subfamily_file(ws, args.prefix, job_id, args.subfamily, output_prefix=output_prefix)
|
||
|
||
# 3. 准备 Pfam 数据库
|
||
print("[INFO] Preparing Pfam database...")
|
||
pfam_hmm, pfam_success, pfam_error = prepare_pfam_database(domain_dir, args.pfam_base)
|
||
|
||
if not pfam_success:
|
||
print(f"[ERROR] {pfam_error}")
|
||
write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, pfam_error)
|
||
sys.exit(1)
|
||
|
||
print(f"[INFO] Pfam ready: {pfam_hmm}")
|
||
|
||
# 4. 运行 hmmscan
|
||
print(f"[INFO] Running hmmscan (this may take a while)...")
|
||
hmm_success, hmm_result = run_hmmscan(
|
||
pfam_hmm, protein_file, domain_dir,
|
||
output_prefix=output_prefix,
|
||
cpu=args.cpu, evalue=args.evalue
|
||
)
|
||
|
||
if not hmm_success:
|
||
error_msg = f"hmmscan failed: {hmm_result.get('error', 'unknown error')}"
|
||
print(f"[ERROR] {error_msg}")
|
||
print(f"[ERROR] stderr: {hmm_result.get('stderr', '')}")
|
||
write_failure_manifest(ws, final_dir, args.prefix, job_id, run_id, start_time, error_msg)
|
||
sys.exit(1)
|
||
|
||
print(f"[INFO] hmmscan completed in {hmm_result['elapsed_sec']:.1f}s")
|
||
print(f"[INFO] domtblout: {hmm_result['domtblout']}")
|
||
|
||
# 5. 解析 domtblout
|
||
print(f"[INFO] Parsing domain hits (E < {args.evalue})...")
|
||
domains, hmm_stats = parse_domtblout(hmm_result['domtblout'], args.evalue)
|
||
|
||
print(f"[INFO] - Total hits: {hmm_stats['total_hits']}")
|
||
print(f"[INFO] - Significant hits: {hmm_stats['significant_hits']}")
|
||
print(f"[INFO] - Unique proteins: {hmm_stats['unique_proteins']}")
|
||
print(f"[INFO] - Unique domains: {hmm_stats['unique_domains']}")
|
||
|
||
if not domains:
|
||
print(f"[WARNING] No significant domain hits found with E-value < {args.evalue}")
|
||
# 不退出，继续生成空报告
|
||
|
||
# 6. 计算 metrics
|
||
# 获取总蛋白数量（从蛋白文件）
|
||
try:
|
||
from Bio import SeqIO
|
||
total_proteins = len(list(SeqIO.parse(protein_file, "fasta")))
|
||
except:
|
||
# 回退：从 domains 估算
|
||
total_proteins = hmm_stats.get("unique_proteins", 0)
|
||
|
||
domain_metrics = calculate_domain_metrics(domains, total_proteins, hmm_stats)
|
||
|
||
print(f"[INFO] Metrics calculated:")
|
||
for k, v in domain_metrics.items():
|
||
if k != "top_domains":
|
||
print(f" - {k}: {v}")
|
||
|
||
# 7. R 绘图（输出到 final/）
|
||
# 使用标准命名：{prefix}.domain（不含扩展名，R脚本会自动添加 .pdf 和 .png）
|
||
output_path_prefix = final_dir / f"{output_prefix}.domain"
|
||
|
||
# 确定 R 脚本路径
|
||
if args.rscript:
|
||
rscript_path = Path(args.rscript)
|
||
else:
|
||
rscript_path = Path(__file__).parent.parent.parent / "scripts" / "tree_domain.R"
|
||
|
||
plot_result = run_r_plotting(
|
||
tree_file,
|
||
subfamily_file,
|
||
hmm_result['domtblout'],
|
||
output_path_prefix,
|
||
rscript_path=rscript_path,
|
||
evalue_cutoff=args.evalue,
|
||
coverage_cutoff=args.coverage
|
||
)
|
||
|
||
plot_success = plot_result.get("success", False)
|
||
if not plot_success:
|
||
print(f"[WARNING] R plotting failed: {plot_result.get('error', plot_result.get('stderr', 'unknown'))}")
|
||
else:
|
||
print(f"[INFO] Plot files generated: {plot_result.get('output_files', [])}")
|
||
|
||
# 8. 准备 artifacts
|
||
end_time = datetime.now()
|
||
artifacts = {}
|
||
|
||
# 中间文件（domtblout, tblout）
|
||
if hmm_result['domtblout'] and hmm_result['domtblout'].exists():
|
||
artifacts["domtblout"] = {
|
||
"artifact": "domtblout",
|
||
"path": str(hmm_result['domtblout'].relative_to(ws)),
|
||
"full_path": display_path(hmm_result['domtblout']),
|
||
"type": "txt",
|
||
"size": hmm_result['domtblout'].stat().st_size,
|
||
"sha256": compute_file_hash(hmm_result['domtblout']),
|
||
"description": "hmmscan domain table output (raw)"
|
||
}
|
||
|
||
if hmm_result['tblout'] and hmm_result['tblout'].exists():
|
||
artifacts["tblout"] = {
|
||
"artifact": "tblout",
|
||
"path": str(hmm_result['tblout'].relative_to(ws)),
|
||
"full_path": display_path(hmm_result['tblout']),
|
||
"type": "txt",
|
||
"size": hmm_result['tblout'].stat().st_size,
|
||
"description": "hmmscan per-sequence hits (raw)"
|
||
}
|
||
|
||
# 最终产物（图片）- 同时支持 PNG 和 PDF
|
||
for img_file in plot_result.get("output_files", []):
|
||
img_path = Path(img_file)
|
||
if img_path.exists():
|
||
key = f"domain_plot_{img_path.suffix[1:]}" # domain_plot_png, domain_plot_pdf
|
||
artifacts[key] = {
|
||
"artifact": "domain_plot",
|
||
"path": str(img_path.relative_to(ws)),
|
||
"full_path": display_path(img_path),
|
||
"type": "image",
|
||
"format": img_path.suffix[1:],
|
||
"size": img_path.stat().st_size,
|
||
"sha256": compute_file_hash(img_path),
|
||
"description": f"Domain architecture visualization ({img_path.suffix[1:].upper()})"
|
||
}
|
||
|
||
# 9. 生成报告
|
||
report_file = final_dir / f"{output_prefix}.domain_report.md"
|
||
|
||
parameters = {
|
||
"cpu": args.cpu,
|
||
"prefix": args.prefix,
|
||
"output_prefix": output_prefix,
|
||
"stage": "domain",
|
||
"job_id": job_id,
|
||
"run_id": run_id,
|
||
"evalue_cutoff": args.evalue,
|
||
"coverage_cutoff": args.coverage,
|
||
"pfam_hmm": str(pfam_hmm),
|
||
"pfam_base": args.pfam_base,
|
||
"protein_source": str(protein_file),
|
||
"protein_sha256": compute_file_hash(protein_file),
|
||
"protein_record_count": protein_record_count,
|
||
"tree_file": str(tree_file),
|
||
"subfamily_file": str(subfamily_file) if subfamily_file else None,
|
||
"hmmscan_elapsed_sec": hmm_result.get("elapsed_sec", 0),
|
||
"Rscript_path": str(rscript_path),
|
||
"tool_versions": get_tool_versions(),
|
||
"timestamp_start": start_time.isoformat(),
|
||
"timestamp_end": end_time.isoformat(),
|
||
"workspace_root": display_path(workspace_root_dir(ws)),
|
||
"workspace": display_path(ws),
|
||
"input_dir": display_path(current_input_dir(ws)),
|
||
"domain_analysis_dir": display_path(domain_dir),
|
||
"final_dir": display_path(final_dir)
|
||
}
|
||
if scope_data:
|
||
parameters["scope_file"] = str(scope_file)
|
||
parameters["scope_id"] = scope_data.get("scope_id", "")
|
||
parameters["scope_mode"] = scope_data.get("scope_mode", "")
|
||
parameters["target_species"] = scope_data.get("target_species", [])
|
||
parameters["reference_species"] = scope_data.get("reference_species", [])
|
||
parameters["comparative_species"] = scope_data.get("comparative_species", [])
|
||
parameters["selected_species"] = scope_data.get("selected_species", [])
|
||
|
||
generate_report(output_prefix, domain_metrics, parameters, report_file, artifacts, plot_success)
|
||
|
||
if report_file.exists():
|
||
artifacts["domain_report"] = {
|
||
"artifact": "domain_report",
|
||
"path": str(report_file.relative_to(ws)),
|
||
"full_path": display_path(report_file),
|
||
"type": "report",
|
||
"format": "markdown",
|
||
"size": report_file.stat().st_size,
|
||
"sha256": compute_file_hash(report_file),
|
||
"description": "Human-readable analysis report"
|
||
}
|
||
|
||
# 10. 确定状态
|
||
# if not domains:
|
||
# status = "partial_success" # hmmscan 成功但无显著 hits
|
||
# elif plot_success:
|
||
# status = "success"
|
||
# else:
|
||
# status = "partial_success
|
||
# 检查输出文件是否存在
|
||
expected_png = final_dir / f"{output_prefix}.domain.png"
|
||
expected_pdf = final_dir / f"{output_prefix}.domain.pdf"
|
||
|
||
if plot_success and (expected_png.exists() or expected_pdf.exists()):
|
||
status = "success" # 只要绘图成功，就视为成功
|
||
else:
|
||
status = "partial_success"
|
||
|
||
# 11. 构建完整 manifest
|
||
manifest = {
|
||
"manifest_version": "1.0",
|
||
"job_id": job_id,
|
||
"stage": "domain",
|
||
"run_id": run_id,
|
||
"prefix": args.prefix,
|
||
"output_prefix": output_prefix,
|
||
"timestamp": end_time.isoformat(),
|
||
"status": status,
|
||
"start_time": start_time.isoformat(),
|
||
"end_time": end_time.isoformat(),
|
||
"elapsed_sec": (end_time - start_time).total_seconds(),
|
||
"cmd": [
|
||
str(Path(__file__).name),
|
||
"--prefix", args.prefix,
|
||
"--workspace", str(ws),
|
||
"--proteins", str(protein_file),
|
||
"--tree", str(tree_file),
|
||
"--pfam-base", args.pfam_base,
|
||
"--evalue", str(args.evalue),
|
||
"--cpu", str(args.cpu),
|
||
"--run-id", run_id
|
||
] + (["--scope-file", str(scope_file)] if scope_file else [])
|
||
+ (["--subfamily", str(subfamily_file)] if subfamily_file else [])
|
||
+ (["--coverage", str(args.coverage)] if args.coverage else [])
|
||
+ (["--rscript", str(rscript_path)] if args.rscript else []),
|
||
"workspace_root": display_path(workspace_root_dir(ws)),
|
||
"workspace": display_path(ws),
|
||
"input_dir": display_path(current_input_dir(ws)),
|
||
"artifacts": artifacts,
|
||
"parameters": parameters,
|
||
"metrics": domain_metrics
|
||
}
|
||
manifest["metrics"]["protein_record_count"] = protein_record_count
|
||
if scope_data:
|
||
manifest["scope_id"] = scope_data.get("scope_id", "")
|
||
manifest["scope_mode"] = scope_data.get("scope_mode", "")
|
||
manifest["scope_file"] = str(scope_file)
|
||
manifest["target_species"] = scope_data.get("target_species", [])
|
||
manifest["reference_species"] = scope_data.get("reference_species", [])
|
||
manifest["comparative_species"] = scope_data.get("comparative_species", [])
|
||
manifest["selected_species"] = scope_data.get("selected_species", [])
|
||
manifest["scope"] = {
|
||
"family": scope_data.get("family", ""),
|
||
"prefix": scope_data.get("prefix", ""),
|
||
"scope_id": scope_data.get("scope_id", ""),
|
||
"scope_mode": scope_data.get("scope_mode", ""),
|
||
"target_species": scope_data.get("target_species", []),
|
||
"reference_species": scope_data.get("reference_species", []),
|
||
"comparative_species": scope_data.get("comparative_species", []),
|
||
"selected_species": scope_data.get("selected_species", []),
|
||
}
|
||
context = build_stage_context(
|
||
stage="domain",
|
||
prefix=args.prefix,
|
||
scope_data=scope_data or {},
|
||
scope_file=scope_file or "",
|
||
output_prefix=output_prefix,
|
||
subfamily_file=subfamily_file,
|
||
analysis_route=args.analysis_route,
|
||
member_set_id=args.member_set_id,
|
||
)
|
||
manifest["tree_source"] = str(tree_file)
|
||
manifest["protein_fasta"] = str(protein_file)
|
||
apply_stage_context(manifest, context)
|
||
|
||
manifest_file = final_dir / f"{output_prefix}.domain.manifest.json"
|
||
write_manifest(manifest, manifest_file)
|
||
if scope_data and scope_data.get("scope_id"):
|
||
write_legacy_manifest_copy(manifest_file, args.prefix, str(scope_data.get("scope_id") or ""), "domain")
|
||
quarantine_stale_unscoped_failure(final_dir, args.prefix, "domain", manifest_file)
|
||
|
||
# 12. 完成
|
||
if status == "success":
|
||
print(f"[INFO] Domain stage completed successfully")
|
||
else:
|
||
print(f"[WARNING] Domain stage completed with status: {status}")
|
||
|
||
print("\\n=== OUTPUT_FILES ===")
|
||
for k, v in artifacts.items():
|
||
if isinstance(v, dict) and v.get("type") == "image":
|
||
print(f"IMAGE: {v['full_path']}")
|
||
|
||
sys.exit(0 if status == "success" else 1)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|