10836 lines
512 KiB
Plaintext
10836 lines
512 KiB
Plaintext
# web/pipeline_backend.py
|
||
import csv
|
||
import json
|
||
import os
|
||
import re
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import traceback
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
import zipfile
|
||
from dataclasses import dataclass
|
||
from datetime import datetime, timedelta
|
||
from functools import lru_cache
|
||
from pathlib import Path
|
||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||
|
||
# 可复现性支持
|
||
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
||
from gfa_core.project_paths import INPUT_ROOT, PROJECT_ROOT, WORKSPACE_ROOT
|
||
from gfa_core.reproducibility_v2 import ReproducibilityManager
|
||
from gfa_core.gene_family_scope import build_gene_family_scope, build_scope_id, read_scope, sanitize_token, write_scope
|
||
from gfa_core.gene_family_stage_context import apply_stage_context, build_stage_context, member_set_id_for, user_output_prefix_for
|
||
from gfa_core.mas.effective_members import MemberResolutionError, resolve_effective_members
|
||
from gfa_core.ppi_string_policy import (
|
||
build_ppi_species_diagnostic,
|
||
recommended_string_reference_species,
|
||
reference_protein_candidate_paths,
|
||
species_to_string_code as _shared_species_to_string_code,
|
||
string_taxon_id,
|
||
)
|
||
from gfa_core.ppi_provenance import (
|
||
apply_ppi_provenance,
|
||
build_ppi_provenance,
|
||
record_ppi_preflight_decision,
|
||
)
|
||
from gfa_core.string_species_registry import (
|
||
StringSnapshotError,
|
||
load_string_snapshot,
|
||
resolve_ppi_string_species,
|
||
)
|
||
from gfa_core.string_rate_gate import (
|
||
StringRateGate,
|
||
is_authoritative_string_https_url,
|
||
)
|
||
from gfa_core.rnaseq_context import (
|
||
find_rnaseq_hisat2_index,
|
||
find_rnaseq_reference_file,
|
||
resolve_rnaseq_project_context,
|
||
)
|
||
|
||
# API 客户端支持
|
||
from gfa_core.api_clients import (
|
||
MultiAPIQuery,
|
||
PlantCAREClient,
|
||
PfamClient,
|
||
NCBIClient,
|
||
EnsemblPlantsClient,
|
||
)
|
||
|
||
# 你要求：上传落到项目 input/；输出在 workspace/
|
||
ATH_FAMILY_ROOT = Path("/home/xzq/wutx/data/raw_genome/ath_family")
|
||
INPUT_ROOT.mkdir(parents=True, exist_ok=True)
|
||
WORKSPACE_ROOT.mkdir(parents=True, exist_ok=True)
|
||
|
||
RETENTION_DAYS = 7
|
||
|
||
IMAGE_SUFFIX = {".png", ".jpg", ".jpeg", ".svg", ".webp"}
|
||
PDF_SUFFIX = {".pdf"}
|
||
HTML_SUFFIX = {".html", ".htm"}
|
||
|
||
PREPARE_SUBSTAGES = {
|
||
"prepare_organize": "organize",
|
||
"prepare_normalize_gff": "normalize_gff",
|
||
"prepare_longest_transcript": "longest_transcript",
|
||
"prepare_extract_pep": "extract_pep",
|
||
"prepare_extract_cds": "extract_cds",
|
||
"prepare_filter_fasta": "filter_fasta",
|
||
"prepare_simplify_ncbi": "simplify_ncbi",
|
||
}
|
||
|
||
CUSTOM_PHYLOGENY_ROUTES = {"custom_phylogeny", "comparative_phylogeny"}
|
||
|
||
NCBI_SPECIES_DEFAULTS = {
|
||
"at": {"species": "Arabidopsis thaliana", "taxid": "3702"},
|
||
"ath": {"species": "Arabidopsis thaliana", "taxid": "3702"},
|
||
"os": {"species": "Oryza sativa", "taxid": "4530"},
|
||
"zm": {"species": "Zea mays", "taxid": "4577"},
|
||
"sl": {"species": "Solanum lycopersicum", "taxid": "4081"},
|
||
"ft": {"species": "Fagopyrum tataricum", "taxid": "62330"},
|
||
}
|
||
|
||
|
||
def _ncbi_species_defaults_from_prefix(prefix: str) -> Dict[str, str]:
|
||
token = str(prefix or "").strip().split(".", 1)[0].lower()
|
||
return dict(NCBI_SPECIES_DEFAULTS.get(token) or {})
|
||
|
||
|
||
def phylogeny_route_output_suffix(analysis_route: str) -> str:
|
||
route = str(analysis_route or "").strip()
|
||
if route in CUSTOM_PHYLOGENY_ROUTES:
|
||
return sanitize_token(route)
|
||
return ""
|
||
|
||
|
||
def stage_output_prefix(stage: str, prefix: str, scope_data: Optional[Dict[str, Any]] = None, analysis_route: str = "", member_set_id: str = "") -> str:
|
||
scope_data = scope_data or {}
|
||
if not str(scope_data.get("scope_id") or "").strip():
|
||
return prefix
|
||
return user_output_prefix_for(
|
||
stage,
|
||
prefix,
|
||
scope_data,
|
||
analysis_route=analysis_route,
|
||
member_set_id=member_set_id,
|
||
)
|
||
|
||
|
||
def stage_manifest_candidates(final_dir: Path, stage: str, prefix: str, scope_data: Optional[Dict[str, Any]] = None, analysis_route: str = "", member_set_id: str = "") -> List[Path]:
|
||
scope_data = scope_data or {}
|
||
candidates: List[Path] = []
|
||
output_prefix = stage_output_prefix(stage, prefix, scope_data, analysis_route, member_set_id)
|
||
candidates.append(final_dir / f"{output_prefix}.{stage}.manifest.json")
|
||
scope_id = str(scope_data.get("scope_id") or "").strip()
|
||
if scope_id:
|
||
candidates.append(final_dir / f"{prefix}.{sanitize_token(scope_id)}.{stage}.manifest.json")
|
||
candidates.append(final_dir / f"{prefix}.{stage}.manifest.json")
|
||
deduped: List[Path] = []
|
||
seen = set()
|
||
for item in candidates:
|
||
key = str(item)
|
||
if key not in seen:
|
||
seen.add(key)
|
||
deduped.append(item)
|
||
return deduped
|
||
|
||
|
||
def first_existing_stage_manifest(final_dir: Path, stage: str, prefix: str, scope_data: Optional[Dict[str, Any]] = None, analysis_route: str = "", member_set_id: str = "") -> Path | None:
|
||
for candidate in stage_manifest_candidates(final_dir, stage, prefix, scope_data, analysis_route, member_set_id):
|
||
if candidate.exists() and candidate.stat().st_size > 0:
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def _float_or_none(value: Any) -> Optional[float]:
|
||
try:
|
||
if value is None or value == "":
|
||
return None
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def _int_or_zero(value: Any) -> int:
|
||
try:
|
||
if value is None or value == "":
|
||
return 0
|
||
number = int(float(value))
|
||
return number if number > 0 else 0
|
||
except (TypeError, ValueError):
|
||
return 0
|
||
|
||
|
||
def _summarize_kaks_results_csv(path: Path) -> Dict[str, Any]:
|
||
rows: List[Dict[str, Any]] = []
|
||
try:
|
||
with path.open("r", encoding="utf-8", errors="ignore", newline="") as handle:
|
||
reader = csv.DictReader(handle)
|
||
for row in reader:
|
||
rows.append(row)
|
||
except Exception:
|
||
return {}
|
||
if not rows:
|
||
return {}
|
||
ratios = [
|
||
value
|
||
for value in (_float_or_none(row.get("Ka/Ks") or row.get("Ka.Ks")) for row in rows)
|
||
if value is not None
|
||
]
|
||
ks_values = [
|
||
value
|
||
for value in (_float_or_none(row.get("Ks")) for row in rows)
|
||
if value is not None
|
||
]
|
||
if not ratios:
|
||
return {"successful_pairs": len(rows), "total_pairs": len(rows)}
|
||
return {
|
||
"total_pairs": len(rows),
|
||
"successful_pairs": len(rows),
|
||
"mean_ka_ks": sum(ratios) / len(ratios),
|
||
"min_ka_ks": min(ratios),
|
||
"max_ka_ks": max(ratios),
|
||
"mean_ks": (sum(ks_values) / len(ks_values)) if ks_values else None,
|
||
"positive_selection_count": sum(1 for value in ratios if value > 1),
|
||
"purifying_selection_count": sum(1 for value in ratios if value < 1),
|
||
}
|
||
|
||
|
||
def _aggregate_kaks_run_metrics(run_records: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||
run_summaries: List[Dict[str, Any]] = []
|
||
total_pairs = 0
|
||
successful_pairs = 0
|
||
positive_selection_count = 0
|
||
purifying_selection_count = 0
|
||
neutral_selection_count = 0
|
||
weighted_mean_ka_ks = 0.0
|
||
weighted_mean_ks = 0.0
|
||
mean_ks_weight = 0
|
||
min_ka_ks: Optional[float] = None
|
||
max_ka_ks: Optional[float] = None
|
||
|
||
for record in run_records:
|
||
manifest_path = Path(str(record.get("manifest") or ""))
|
||
result_path = Path(str(record.get("results") or ""))
|
||
child_manifest: Dict[str, Any] = {}
|
||
if manifest_path.exists():
|
||
try:
|
||
child_manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
child_manifest = {}
|
||
metrics = child_manifest.get("metrics") if isinstance(child_manifest.get("metrics"), dict) else {}
|
||
if not metrics and result_path.exists():
|
||
metrics = _summarize_kaks_results_csv(result_path)
|
||
|
||
run_total = _int_or_zero(metrics.get("total_pairs")) or _int_or_zero(record.get("pair_count"))
|
||
run_success = _int_or_zero(metrics.get("successful_pairs"))
|
||
mean_ka_ks = _float_or_none(metrics.get("mean_ka_ks"))
|
||
mean_ks = _float_or_none(metrics.get("mean_ks"))
|
||
run_min = _float_or_none(metrics.get("min_ka_ks"))
|
||
run_max = _float_or_none(metrics.get("max_ka_ks"))
|
||
|
||
total_pairs += run_total
|
||
successful_pairs += run_success
|
||
positive_selection_count += _int_or_zero(metrics.get("positive_selection_count"))
|
||
purifying_selection_count += _int_or_zero(metrics.get("purifying_selection_count"))
|
||
neutral_selection_count += _int_or_zero(metrics.get("neutral_selection_count"))
|
||
if mean_ka_ks is not None and run_success:
|
||
weighted_mean_ka_ks += mean_ka_ks * run_success
|
||
if mean_ks is not None and run_success:
|
||
weighted_mean_ks += mean_ks * run_success
|
||
mean_ks_weight += run_success
|
||
if run_min is not None:
|
||
min_ka_ks = run_min if min_ka_ks is None else min(min_ka_ks, run_min)
|
||
if run_max is not None:
|
||
max_ka_ks = run_max if max_ka_ks is None else max(max_ka_ks, run_max)
|
||
|
||
run_summaries.append({
|
||
"target_species": record.get("target_species"),
|
||
"reference_species": record.get("reference_species"),
|
||
"prefix": record.get("prefix"),
|
||
"total_pairs": run_total,
|
||
"successful_pairs": run_success,
|
||
"mean_ka_ks": mean_ka_ks,
|
||
"mean_ks": mean_ks,
|
||
"positive_selection_count": _int_or_zero(metrics.get("positive_selection_count")),
|
||
"purifying_selection_count": _int_or_zero(metrics.get("purifying_selection_count")),
|
||
"manifest": str(manifest_path) if manifest_path else "",
|
||
"results": str(result_path) if result_path else "",
|
||
})
|
||
|
||
aggregate: Dict[str, Any] = {
|
||
"species_runs": len(run_records),
|
||
"input_pairs": total_pairs,
|
||
"total_pairs": total_pairs,
|
||
"successful_pairs": successful_pairs,
|
||
"positive_selection_count": positive_selection_count,
|
||
"purifying_selection_count": purifying_selection_count,
|
||
"neutral_selection_count": neutral_selection_count,
|
||
"run_summaries": run_summaries,
|
||
}
|
||
if successful_pairs:
|
||
aggregate["mean_ka_ks"] = round(weighted_mean_ka_ks / successful_pairs, 6)
|
||
if mean_ks_weight:
|
||
aggregate["mean_ks"] = round(weighted_mean_ks / mean_ks_weight, 6)
|
||
if min_ka_ks is not None:
|
||
aggregate["min_ka_ks"] = min_ka_ks
|
||
if max_ka_ks is not None:
|
||
aggregate["max_ka_ks"] = max_ka_ks
|
||
return aggregate
|
||
|
||
|
||
def _clean_prefix(value) -> str:
|
||
if value is None:
|
||
return ""
|
||
value = str(value).strip()
|
||
return value if value and value.lower() not in {"none", "null", "undefined", "nan", "-"} else ""
|
||
|
||
|
||
def _truthy(value) -> bool:
|
||
if isinstance(value, str):
|
||
return value.strip().lower() in {"1", "true", "yes", "y", "on"}
|
||
return bool(value)
|
||
|
||
|
||
def _bool_with_default(value, default: bool = False) -> bool:
|
||
if value is None or value == "":
|
||
return default
|
||
if isinstance(value, str):
|
||
text = value.strip().lower()
|
||
if text in {"1", "true", "yes", "y", "on"}:
|
||
return True
|
||
if text in {"0", "false", "no", "n", "off"}:
|
||
return False
|
||
return default
|
||
return bool(value)
|
||
|
||
|
||
def should_disable_cli_subfamily_auto(analysis_route: str = "", label_source: str = "") -> bool:
|
||
source = str(label_source or "").strip().lower()
|
||
route = str(analysis_route or "").strip()
|
||
if source in {"none", "skip", "disabled"}:
|
||
return True
|
||
return route == "subfamily_evidence" and source in {"", "auto"}
|
||
|
||
|
||
def resolve_subfamily_for_plotting(
|
||
final_dir: Path,
|
||
input_dir: Path,
|
||
prefix: str,
|
||
scope_id: str = "",
|
||
label_source: str = "",
|
||
) -> Path | None:
|
||
"""Resolve subfamily labels for downstream plots.
|
||
|
||
Empty/auto/default modes are deliberately conservative: formal downstream
|
||
stages may only consume confirmed labels. Prediction tables are accepted
|
||
only when a caller explicitly asks for a prediction source.
|
||
"""
|
||
source = str(label_source or "").strip()
|
||
source_lower = source.lower()
|
||
known_sources = {
|
||
"auto", "confirmed", "confirmed_subfamily", "confirmed_subfamily.tsv",
|
||
"confirmed_or_uploaded_subfamily", "uploaded", "uploaded_subfamily", "subfamily",
|
||
"prediction", "predicted", "reference_classification",
|
||
"reference_classification.subfamily.tsv", "geneid.subfamily", "geneid_subfamily",
|
||
}
|
||
if source_lower in {"none", "skip", "disabled"}:
|
||
return None
|
||
explicit_path = Path(source).expanduser() if source and source_lower not in known_sources else None
|
||
if explicit_path:
|
||
if not explicit_path.is_absolute():
|
||
for base in (final_dir, input_dir, final_dir.parent, Path(".")):
|
||
candidate = base / explicit_path
|
||
if candidate.exists() and candidate.stat().st_size > 0:
|
||
return candidate
|
||
if explicit_path.exists() and explicit_path.stat().st_size > 0:
|
||
return explicit_path
|
||
return None
|
||
|
||
scoped_prefix = f"{prefix}.{sanitize_token(scope_id)}" if scope_id else ""
|
||
prediction_sources = {
|
||
"prediction", "predicted", "reference_classification",
|
||
"reference_classification.subfamily.tsv", "geneid.subfamily", "geneid_subfamily",
|
||
}
|
||
prediction_candidates = [
|
||
final_dir / f"{scoped_prefix}.reference_classification.subfamily.tsv" if scoped_prefix else None,
|
||
final_dir / f"{scoped_prefix}.geneID.subfamily" if scoped_prefix else None,
|
||
input_dir / f"{scoped_prefix}.geneID.subfamily" if scoped_prefix else None,
|
||
final_dir / f"{prefix}.target_plus_reference.reference_classification.subfamily.tsv",
|
||
final_dir / f"{prefix}.target_plus_reference.geneID.subfamily",
|
||
input_dir / f"{prefix}.target_plus_reference.geneID.subfamily",
|
||
final_dir / f"{prefix}.reference_classification.subfamily.tsv",
|
||
final_dir / f"{prefix}.geneID.subfamily",
|
||
input_dir / f"{prefix}.geneID.subfamily",
|
||
]
|
||
if source_lower in prediction_sources:
|
||
return next((path for path in prediction_candidates if path is not None and path.exists() and path.stat().st_size > 0), None)
|
||
|
||
confirmed_candidates = [
|
||
final_dir / f"{scoped_prefix}.confirmed_subfamily.tsv" if scoped_prefix else None,
|
||
input_dir / f"{scoped_prefix}.confirmed_subfamily.tsv" if scoped_prefix else None,
|
||
final_dir / f"{prefix}.target_plus_reference.confirmed_subfamily.tsv",
|
||
input_dir / f"{prefix}.target_plus_reference.confirmed_subfamily.tsv",
|
||
final_dir / f"{prefix}.confirmed_subfamily.tsv",
|
||
input_dir / f"{prefix}.confirmed_subfamily.tsv",
|
||
final_dir / f"{prefix}.subfamily",
|
||
input_dir / f"{prefix}.subfamily",
|
||
final_dir / f"{prefix}.subfamily.tsv",
|
||
input_dir / f"{prefix}.subfamily.tsv",
|
||
]
|
||
return next((path for path in confirmed_candidates if path is not None and path.exists() and path.stat().st_size > 0), None)
|
||
|
||
|
||
def materialize_compact_combo_outputs(final_dir: Path, species_prefix: str, combo_manifest: Path | None) -> Dict[str, Path]:
|
||
"""Copy scoped combo outputs to the compact per-species names used by review UI."""
|
||
if not combo_manifest or not combo_manifest.exists():
|
||
return {}
|
||
try:
|
||
manifest = json.loads(combo_manifest.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return {}
|
||
|
||
outputs: Dict[str, Path] = {}
|
||
for artifact in (manifest.get("artifacts") or {}).values():
|
||
if not isinstance(artifact, dict) or artifact.get("artifact") != "combo_plot":
|
||
continue
|
||
raw_path = str(artifact.get("full_path") or artifact.get("path") or "").strip()
|
||
if not raw_path:
|
||
continue
|
||
source = Path(raw_path)
|
||
if not source.is_absolute():
|
||
source = final_dir.parent / source
|
||
if not source.exists() or source.stat().st_size <= 0:
|
||
continue
|
||
suffix = source.suffix.lower()
|
||
if suffix not in {".png", ".pdf"}:
|
||
continue
|
||
target = final_dir / f"{species_prefix}.target_plus_reference.combo{suffix}"
|
||
if source.resolve() != target.resolve():
|
||
shutil.copy2(source, target)
|
||
outputs[suffix] = target
|
||
return outputs
|
||
|
||
|
||
def _read_subfamily_rows(path: Path) -> list[tuple[str, str]]:
|
||
try:
|
||
lines = path.read_text(encoding="utf-8", errors="ignore").splitlines()
|
||
except Exception:
|
||
return []
|
||
rows = [line for line in lines if line.strip() and not line.lstrip().startswith("#")]
|
||
if not rows:
|
||
return []
|
||
delimiter = "\t" if "\t" in rows[0] else ","
|
||
first = [part.strip() for part in rows[0].split(delimiter)]
|
||
lowered = [part.lower() for part in first]
|
||
has_header = any(key in lowered for key in ["gene_id", "gene", "id", "subfamily", "confirmed_subfamily", "predicted_subfamily"])
|
||
parsed: list[tuple[str, str]] = []
|
||
if has_header:
|
||
gene_idx = next((i for i, name in enumerate(lowered) if name in {"gene_id", "gene", "id", "target_gene", "label"}), 0)
|
||
sub_idx = next((i for i, name in enumerate(lowered) if name in {"confirmed_subfamily", "subfamily", "assigned_subfamily", "predicted_subfamily", "prediction"}), 1 if len(first) > 1 else 0)
|
||
data_lines = rows[1:]
|
||
else:
|
||
gene_idx = 0
|
||
sub_idx = 1
|
||
data_lines = rows
|
||
for line in data_lines:
|
||
parts = [part.strip() for part in line.split(delimiter)]
|
||
if len(parts) <= max(gene_idx, sub_idx):
|
||
continue
|
||
gene_id = parts[gene_idx]
|
||
subfamily = parts[sub_idx]
|
||
if gene_id and subfamily:
|
||
parsed.append((gene_id, subfamily))
|
||
return parsed
|
||
|
||
|
||
|
||
def _read_subfamily_metadata_for_source(path: Path) -> dict[str, dict[str, str]]:
|
||
"""Return per-gene confidence metadata associated with a subfamily table."""
|
||
candidates: list[Path] = []
|
||
name = path.name
|
||
for old, new in [
|
||
(".confirmed_subfamily.tsv", ".confirmed_subfamily.review.tsv"),
|
||
(".reference_classification.subfamily.tsv", ".reference_classification.confidence.tsv"),
|
||
(".reference_classification.subfamily.tsv", ".reference_classification.assignment.tsv"),
|
||
(".subfamily.tsv", ".assignment.tsv"),
|
||
(".subfamily.tsv", ".confidence.tsv"),
|
||
]:
|
||
if name.endswith(old):
|
||
candidates.append(path.with_name(name[: -len(old)] + new))
|
||
|
||
out: dict[str, dict[str, str]] = {}
|
||
for candidate in candidates:
|
||
if not candidate.exists() or candidate.stat().st_size == 0:
|
||
continue
|
||
try:
|
||
with candidate.open(encoding="utf-8", errors="ignore", newline="") as handle:
|
||
sample = handle.read(2048)
|
||
handle.seek(0)
|
||
first_line = sample.splitlines()[0] if sample.splitlines() else ""
|
||
delimiter = "\t" if "\t" in first_line else ","
|
||
reader = csv.DictReader(handle, delimiter=delimiter)
|
||
for row in reader:
|
||
lowered = {str(k or "").strip().lower(): str(v or "").strip() for k, v in row.items()}
|
||
gene_id = lowered.get("gene_id") or lowered.get("target_gene") or lowered.get("gene") or lowered.get("id") or lowered.get("label")
|
||
if not gene_id:
|
||
continue
|
||
out[gene_id] = {
|
||
"confidence": lowered.get("confidence") or lowered.get("score") or "",
|
||
"note": lowered.get("note") or lowered.get("classification_note") or lowered.get("status") or "",
|
||
"predicted_subfamily": lowered.get("predicted_subfamily") or lowered.get("assigned_subfamily") or lowered.get("subfamily") or "",
|
||
}
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to read subfamily confidence metadata {candidate}: {exc}")
|
||
return out
|
||
|
||
|
||
def merge_subfamily_for_scope(final_dir: Path, input_dir: Path, prefix: str, scope_data: dict, label_source: str = "") -> Path | None:
|
||
"""Create one de-duplicated confirmed subfamily table for multi-species plotting.
|
||
|
||
Target prediction tables are only consumed when the caller explicitly asks
|
||
for a prediction source. This keeps subfamily evidence plots from silently
|
||
reusing unconfirmed reference_classification output.
|
||
"""
|
||
family = str(scope_data.get("family") or (prefix.split(".")[-1] if "." in prefix else "")).strip()
|
||
scope_id = str(scope_data.get("scope_id") or "").strip()
|
||
species = []
|
||
for group in ("target_species", "reference_species", "comparative_species"):
|
||
for item in scope_data.get(group) or []:
|
||
item = str(item or "").strip()
|
||
if item and item not in species:
|
||
species.append(item)
|
||
if not family or not species:
|
||
return None
|
||
|
||
source_lower = str(label_source or "").strip().lower()
|
||
allow_prediction = source_lower in {
|
||
"prediction", "predicted", "reference_classification",
|
||
"reference_classification.subfamily.tsv", "geneid.subfamily", "geneid_subfamily",
|
||
}
|
||
|
||
labels: dict[str, tuple[str, int, str]] = {}
|
||
labels_meta: dict[str, dict[str, str]] = {}
|
||
used_sources: list[str] = []
|
||
for sp in species:
|
||
candidates: list[tuple[Path, int]] = [
|
||
(final_dir / f"{sp}.{family}.confirmed_subfamily.tsv", 0),
|
||
(final_dir / f"{sp}.{family}.target_plus_reference.confirmed_subfamily.tsv", 0),
|
||
(input_dir / f"{sp}.{family}.confirmed_subfamily.tsv", 0),
|
||
(input_dir / f"{sp}.{family}.target_plus_reference.confirmed_subfamily.tsv", 0),
|
||
(final_dir / f"{sp}.{family}.subfamily", 0),
|
||
(input_dir / f"{sp}.{family}.subfamily", 0),
|
||
(final_dir / f"{sp}.{family}.subfamily.tsv", 0),
|
||
(input_dir / f"{sp}.{family}.subfamily.tsv", 0),
|
||
]
|
||
for pattern, rank in [
|
||
(f"{sp}.{family}*target_plus_reference*confirmed_subfamily.tsv", 0),
|
||
(f"*target-{sp}*target_plus_reference*confirmed_subfamily.tsv", 0),
|
||
]:
|
||
candidates.extend((path, rank) for path in final_dir.glob(pattern))
|
||
candidates.extend((path, rank) for path in input_dir.glob(pattern))
|
||
if allow_prediction:
|
||
candidates.extend([
|
||
(final_dir / f"{sp}.{family}.reference_classification.subfamily.tsv", 2),
|
||
(final_dir / f"{sp}.{family}.target_plus_reference.reference_classification.subfamily.tsv", 2),
|
||
(final_dir / f"{sp}.{family}.geneID.subfamily", 3),
|
||
(input_dir / f"{sp}.{family}.geneID.subfamily", 3),
|
||
])
|
||
for pattern, rank in [
|
||
(f"{sp}.{family}*target_plus_reference*reference_classification.subfamily.tsv", 2),
|
||
(f"*target-{sp}*target_plus_reference*reference_classification.subfamily.tsv", 2),
|
||
]:
|
||
candidates.extend((path, rank) for path in final_dir.glob(pattern))
|
||
candidates.extend((path, rank) for path in input_dir.glob(pattern))
|
||
seen_paths: set[Path] = set()
|
||
for candidate, rank in sorted(candidates, key=lambda item: (item[1], str(item[0]))):
|
||
if candidate in seen_paths or not candidate.exists() or candidate.stat().st_size == 0:
|
||
continue
|
||
seen_paths.add(candidate)
|
||
rows = _read_subfamily_rows(candidate)
|
||
if not rows:
|
||
continue
|
||
source_meta = _read_subfamily_metadata_for_source(candidate)
|
||
used_sources.append(str(candidate))
|
||
for gene_id, subfamily in rows:
|
||
current = labels.get(gene_id)
|
||
if current is None or rank < current[1]:
|
||
labels[gene_id] = (subfamily, rank, str(candidate))
|
||
labels_meta[gene_id] = source_meta.get(gene_id, {})
|
||
if not labels:
|
||
return None
|
||
out = final_dir / f"{prefix}.{sanitize_token(scope_id)}.merged_subfamily.tsv" if scope_id else final_dir / f"{prefix}.merged_subfamily.tsv"
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
with out.open("w", encoding="utf-8") as handle:
|
||
handle.write("gene_id\tsubfamily\n")
|
||
for gene_id in sorted(labels):
|
||
handle.write(f"{gene_id}\t{labels[gene_id][0]}\n")
|
||
|
||
review_out = out.with_name(out.name.replace(".merged_subfamily.tsv", ".merged_subfamily.review.tsv"))
|
||
with review_out.open("w", encoding="utf-8", newline="") as handle:
|
||
writer = csv.DictWriter(
|
||
handle,
|
||
fieldnames=["gene_id", "subfamily", "confidence", "note", "predicted_subfamily", "source"],
|
||
delimiter="\t",
|
||
)
|
||
writer.writeheader()
|
||
for gene_id in sorted(labels):
|
||
subfamily, _rank, source = labels[gene_id]
|
||
meta = labels_meta.get(gene_id, {})
|
||
writer.writerow({
|
||
"gene_id": gene_id,
|
||
"subfamily": subfamily,
|
||
"confidence": meta.get("confidence", ""),
|
||
"note": meta.get("note", ""),
|
||
"predicted_subfamily": meta.get("predicted_subfamily", ""),
|
||
"source": source,
|
||
})
|
||
print(f"[INFO] Merged subfamily labels for phylogeny: {out} ({len(labels)} genes; {len(set(used_sources))} sources)")
|
||
return out
|
||
|
||
|
||
def ensure_protein_fasta_without_stop_markers(protein_fasta: Path, final_dir: Path, prefix: str) -> Path:
|
||
"""Return a job-local FASTA without '*' stop markers when needed."""
|
||
if not protein_fasta.exists():
|
||
return protein_fasta
|
||
try:
|
||
has_stop = False
|
||
with open(protein_fasta, "r", encoding="utf-8", errors="ignore") as handle:
|
||
for line in handle:
|
||
if line.startswith(">"):
|
||
continue
|
||
if "*" in line:
|
||
has_stop = True
|
||
break
|
||
if not has_stop:
|
||
return protein_fasta
|
||
|
||
default_fasta = final_dir / f"{prefix}.final_members.pep.fa"
|
||
try:
|
||
is_default_fasta = protein_fasta.resolve() == default_fasta.resolve()
|
||
except Exception:
|
||
is_default_fasta = protein_fasta == default_fasta
|
||
if is_default_fasta:
|
||
clean_fasta = final_dir / f"{prefix}.final_members.pep.clean.fa"
|
||
else:
|
||
clean_name = re.sub(r"(\.fa(?:sta)?)$", r".clean\1", protein_fasta.name, flags=re.IGNORECASE)
|
||
if clean_name == protein_fasta.name:
|
||
clean_name = protein_fasta.name + ".clean.fa"
|
||
clean_fasta = protein_fasta.with_name(clean_name)
|
||
if clean_fasta.exists() and clean_fasta.stat().st_mtime >= protein_fasta.stat().st_mtime:
|
||
print(f"[INFO] Using existing stop-marker-clean FASTA: {clean_fasta}")
|
||
return clean_fasta
|
||
|
||
clean_fasta.parent.mkdir(parents=True, exist_ok=True)
|
||
seq_count = 0
|
||
modified = 0
|
||
removed = 0
|
||
current_changed = False
|
||
with open(protein_fasta, "r", encoding="utf-8", errors="ignore") as src, open(clean_fasta, "w", encoding="utf-8") as dst:
|
||
for line in src:
|
||
if line.startswith(">"):
|
||
if seq_count and current_changed:
|
||
modified += 1
|
||
seq_count += 1
|
||
current_changed = False
|
||
dst.write(line)
|
||
continue
|
||
count = line.count("*")
|
||
if count:
|
||
removed += count
|
||
current_changed = True
|
||
line = line.replace("*", "")
|
||
dst.write(line)
|
||
if seq_count and current_changed:
|
||
modified += 1
|
||
meta = {
|
||
"source": str(protein_fasta),
|
||
"clean_fasta": str(clean_fasta),
|
||
"sequence_count": seq_count,
|
||
"modified_sequences": modified,
|
||
"removed_asterisks": removed,
|
||
"created_at": datetime.now().isoformat(),
|
||
"policy": "job-local derived FASTA; original file is unchanged",
|
||
}
|
||
clean_fasta.with_suffix(clean_fasta.suffix + ".repair.json").write_text(
|
||
json.dumps(meta, indent=2, ensure_ascii=False),
|
||
encoding="utf-8",
|
||
)
|
||
print(f"[INFO] Removed {removed} '*' stop markers from {modified} sequences: {clean_fasta}")
|
||
return clean_fasta
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to prepare stop-marker-clean FASTA for {protein_fasta}: {exc}")
|
||
return protein_fasta
|
||
|
||
|
||
def resolve_scope_path(scope_file: str, in_dir: Path, ws: Path) -> Path:
|
||
scope_path = Path(scope_file).expanduser()
|
||
if not scope_path.is_absolute():
|
||
for candidate in (in_dir / scope_path, ws / scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
return candidate
|
||
return scope_path
|
||
|
||
|
||
def resolve_scoped_protein_fasta_from_phylogeny(
|
||
ws: Path,
|
||
final_dir: Path,
|
||
prefix: str,
|
||
scope_path: Path,
|
||
in_dir: Optional[Path] = None,
|
||
) -> tuple[Path, dict]:
|
||
scope_data = read_scope(scope_path)
|
||
manifest_candidates = stage_manifest_candidates(final_dir, "phylogeny", prefix, scope_data)
|
||
|
||
for manifest_file in manifest_candidates:
|
||
if not manifest_file.exists():
|
||
continue
|
||
try:
|
||
manifest_data = json.loads(manifest_file.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
fasta_text = manifest_data.get("parameters", {}).get("scope_input_fasta")
|
||
if fasta_text:
|
||
protein_file = Path(fasta_text)
|
||
if protein_file.exists() and protein_file.stat().st_size > 0:
|
||
return ensure_protein_fasta_without_stop_markers(protein_file, final_dir, prefix), scope_data
|
||
|
||
fallback = resolve_identify_protein_fasta_for_scope(ws, final_dir, in_dir or INPUT_ROOT / ws.name, prefix, scope_data)
|
||
if fallback:
|
||
return fallback, scope_data
|
||
|
||
raise FileNotFoundError(
|
||
"Cannot find scoped protein FASTA from phylogeny manifest. "
|
||
"Run phylogeny with the same scope first, or use target_only scope after Identify."
|
||
)
|
||
|
||
|
||
def resolve_scoped_tree_from_phylogeny(
|
||
ws: Path,
|
||
final_dir: Path,
|
||
prefix: str,
|
||
scope_path: Path,
|
||
) -> tuple[Path, dict]:
|
||
scope_data = read_scope(scope_path)
|
||
manifest_candidates = stage_manifest_candidates(final_dir, "phylogeny", prefix, scope_data)
|
||
|
||
for manifest_file in manifest_candidates:
|
||
if not manifest_file.exists():
|
||
continue
|
||
try:
|
||
manifest_data = json.loads(manifest_file.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
artifacts = manifest_data.get("artifacts", {}) or {}
|
||
tree_item = artifacts.get("tree") if isinstance(artifacts, dict) else None
|
||
tree_text = ""
|
||
if isinstance(tree_item, dict):
|
||
tree_text = str(tree_item.get("full_path") or tree_item.get("path") or "")
|
||
elif isinstance(tree_item, str):
|
||
tree_text = tree_item
|
||
if tree_text:
|
||
tree_file = Path(tree_text)
|
||
if not tree_file.is_absolute():
|
||
tree_file = ws / tree_file
|
||
if tree_file.exists() and tree_file.stat().st_size > 0:
|
||
return tree_file.resolve(), scope_data
|
||
|
||
scope_id = str(scope_data.get("scope_id") or "")
|
||
fallback = final_dir / f"{prefix}.tree.nwk"
|
||
if not scope_id and fallback.exists() and fallback.stat().st_size > 0:
|
||
return fallback.resolve(), scope_data
|
||
raise FileNotFoundError(
|
||
"Cannot find scoped tree from phylogeny manifest. "
|
||
"Run phylogeny with the same scope before reference classification."
|
||
)
|
||
|
||
|
||
def annotate_pep_manifest_with_scope(manifest_path: Path, prefix: str, protein_file: Path, scope_path: Path, scope_data: dict, extra_parameters: Optional[dict] = None) -> None:
|
||
if not manifest_path.exists():
|
||
return
|
||
try:
|
||
manifest_data = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return
|
||
scope_fields = {
|
||
"scope_file": str(scope_path),
|
||
"scope_id": scope_data.get("scope_id", ""),
|
||
"scope_mode": scope_data.get("scope_mode", ""),
|
||
"target_species": scope_data.get("target_species", []),
|
||
"reference_species": scope_data.get("reference_species", []),
|
||
"comparative_species": scope_data.get("comparative_species", []),
|
||
"selected_species": scope_data.get("selected_species", []),
|
||
"family": scope_data.get("family", ""),
|
||
}
|
||
manifest_data.update(scope_fields)
|
||
params = manifest_data.setdefault("parameters", {})
|
||
if isinstance(params, dict):
|
||
params.update(scope_fields)
|
||
params["protein_fasta"] = str(protein_file)
|
||
if extra_parameters:
|
||
params.update(extra_parameters)
|
||
if extra_parameters:
|
||
manifest_data.update({k: v for k, v in extra_parameters.items() if k not in manifest_data})
|
||
input_data = manifest_data.setdefault("input", {})
|
||
if isinstance(input_data, dict):
|
||
input_data["protein_fasta"] = str(protein_file)
|
||
manifest_path.write_text(json.dumps(manifest_data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
if scope_fields["scope_id"]:
|
||
scoped_manifest = manifest_path.parent / f"{prefix}.{sanitize_token(str(scope_fields['scope_id']))}.pep.manifest.json"
|
||
manifest_data["scope_manifest"] = str(scoped_manifest.absolute())
|
||
scoped_manifest.write_text(json.dumps(manifest_data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
|
||
|
||
def resolve_single_target_gene_id_file(ws: Path, in_dir: Path, final_dir: Path, species: str, prefix: str) -> Optional[Path]:
|
||
family = prefix.split(".")[-1]
|
||
try:
|
||
resolved = resolve_effective_members(
|
||
workspace=ws,
|
||
final_dir=final_dir,
|
||
input_dir=in_dir,
|
||
prefix=f"Target.{species}.{family}" if species and family else prefix,
|
||
phase="auto",
|
||
require="id",
|
||
species=species,
|
||
family=family,
|
||
)
|
||
if resolved.path_id:
|
||
if resolved.warnings:
|
||
print(f"[WARN] {'; '.join(resolved.warnings)}")
|
||
return resolved.path_id
|
||
except MemberResolutionError as exc:
|
||
print(f"[WARN] Effective member ID resolution failed for {prefix}: {exc}")
|
||
if "Missing confirmed final members" in str(exc):
|
||
return None
|
||
candidates = [
|
||
final_dir / f"Target.{species}.{family}.curated_members.id",
|
||
in_dir / f"Target.{species}.{family}.curated_members.id",
|
||
final_dir / f"{species}.{family}.curated_members.id",
|
||
in_dir / f"{species}.{family}.curated_members.id",
|
||
final_dir / f"{prefix}.curated_members.id",
|
||
in_dir / f"{prefix}.curated_members.id",
|
||
final_dir / f"Target.{species}.{family}.final_members.id",
|
||
in_dir / f"Target.{species}.{family}.final_members.id",
|
||
final_dir / f"{species}.{family}.final_members.id",
|
||
in_dir / f"{species}.{family}.final_members.id",
|
||
final_dir / f"{prefix}.final_members.id",
|
||
in_dir / f"{prefix}.final_members.id",
|
||
ws / "identify" / species / "final" / f"{species}.final_gfmembers.id",
|
||
final_dir / f"{species}.final_gfmembers.id",
|
||
in_dir / f"{species}.final_gfmembers.id",
|
||
]
|
||
for candidate in candidates:
|
||
if candidate.exists() and candidate.stat().st_size > 0:
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def find_identify_manifest_for_target(final_dir: Path, prefix: str, species: str, family: str = "") -> Optional[Path]:
|
||
species = str(species or "").strip()
|
||
family = str(family or (prefix.split(".")[-1] if "." in prefix else prefix)).strip()
|
||
if not species:
|
||
return None
|
||
candidates = []
|
||
exact = final_dir / f"{prefix}.identify.manifest.json"
|
||
if exact.exists():
|
||
candidates.append(exact)
|
||
candidates.extend(sorted(final_dir.glob("*.identify.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True))
|
||
seen = set()
|
||
for candidate in candidates:
|
||
if candidate in seen:
|
||
continue
|
||
seen.add(candidate)
|
||
try:
|
||
data = json.loads(candidate.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
if data.get("stage") != "identify" or data.get("status") not in {"success", "partial_success"}:
|
||
continue
|
||
manifest_family = str(data.get("gene_family") or data.get("family") or "").strip()
|
||
if family and manifest_family and manifest_family != family:
|
||
continue
|
||
target_species = {str(x).strip() for x in data.get("target_species") or [] if str(x).strip()}
|
||
result_species = {
|
||
str(item.get("species") or "").strip()
|
||
for item in data.get("species_results") or []
|
||
if isinstance(item, dict) and str(item.get("species") or "").strip()
|
||
}
|
||
if species in target_species or species in result_species:
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def first_existing_scope_file(scope_data: dict, species: str, file_types: list[str]) -> Optional[Path]:
|
||
files_by_species = scope_data.get("files", {})
|
||
species_files = files_by_species.get(species, {}) if isinstance(files_by_species, dict) else {}
|
||
for file_type in file_types:
|
||
entries = species_files.get(file_type, [])
|
||
if not isinstance(entries, list):
|
||
continue
|
||
for entry in entries:
|
||
if not isinstance(entry, dict):
|
||
continue
|
||
path_text = str(entry.get("path") or "").strip()
|
||
if not path_text:
|
||
continue
|
||
path = Path(path_text)
|
||
if path.exists() and path.stat().st_size > 0:
|
||
return path
|
||
return None
|
||
|
||
|
||
def resolve_species_family_pep(ws: Path, final_dir: Path, in_dir: Path, species: str, family: str) -> Optional[Path]:
|
||
prefix = f"Target.{species}.{family}" if species and family else f"{species}.{family}".strip(".")
|
||
try:
|
||
resolved = resolve_effective_members(
|
||
workspace=ws,
|
||
final_dir=final_dir,
|
||
input_dir=in_dir,
|
||
prefix=prefix,
|
||
phase="auto",
|
||
require="pep",
|
||
species=species,
|
||
family=family,
|
||
)
|
||
if resolved.path_pep:
|
||
if resolved.warnings:
|
||
print(f"[WARN] {'; '.join(resolved.warnings)}")
|
||
return resolved.path_pep
|
||
except MemberResolutionError as exc:
|
||
print(f"[WARN] Effective member FASTA resolution failed for {prefix}: {exc}")
|
||
if "Missing confirmed final members" in str(exc):
|
||
return None
|
||
candidates = [
|
||
final_dir / f"Target.{species}.{family}.final_members.pep.fa",
|
||
in_dir / f"Target.{species}.{family}.final_members.pep.fa",
|
||
final_dir / f"{species}.final_gfmembers.pep.fa",
|
||
ws / "identify" / species / "final" / f"{species}.final_gfmembers.pep.fa",
|
||
final_dir / f"{species}.{family}.final_members.pep.fa",
|
||
in_dir / f"{species}.{family}.final_members.pep.fa",
|
||
final_dir / f"{species}.{family}.pep.fa",
|
||
final_dir / f"{species}.{family}.pep.fasta",
|
||
in_dir / f"{species}.{family}.pep.fa",
|
||
in_dir / f"{species}.{family}.pep.fasta",
|
||
in_dir / f"{species}.pep.fa",
|
||
in_dir / f"{species}.pep.fasta",
|
||
]
|
||
for candidate in candidates:
|
||
if candidate.exists() and candidate.stat().st_size > 0:
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def resolve_member_protein_for_stage_route(
|
||
*,
|
||
ws: Path,
|
||
final_dir: Path,
|
||
in_dir: Path,
|
||
prefix: str,
|
||
analysis_route: str = "",
|
||
species: str = "",
|
||
family: str = "",
|
||
member_set_id: str = "",
|
||
) -> tuple[Optional[Path], Optional[str]]:
|
||
route = str(analysis_route or "").strip()
|
||
phase = "subfamily_evidence_required" if route == "subfamily_evidence" else "post_confirm_required"
|
||
try:
|
||
resolved = resolve_effective_members(
|
||
workspace=ws,
|
||
final_dir=final_dir,
|
||
input_dir=in_dir,
|
||
prefix=prefix,
|
||
phase=phase,
|
||
require="pep",
|
||
species=species,
|
||
family=family,
|
||
member_set_id=member_set_id,
|
||
)
|
||
except MemberResolutionError as exc:
|
||
if phase == "subfamily_evidence_required":
|
||
return None, f"❌ 亚家族证据阶段需要先完成 identify 成员初筛确认，生成 curated_members 后才能继续：{exc}"
|
||
return None, f"❌ 常规基因家族分析需要先完成亚家族确认，生成 final_members 后才能继续：{exc}"
|
||
if resolved.path_pep:
|
||
if resolved.warnings:
|
||
print(f"[WARN] {'; '.join(resolved.warnings)}")
|
||
print(
|
||
f"[INFO] {route or 'regular'} member protein source: "
|
||
f"{resolved.source_kind} ({resolved.member_count} sequences)"
|
||
)
|
||
return ensure_protein_fasta_without_stop_markers(resolved.path_pep, final_dir, prefix), None
|
||
return None, "❌ 未找到可用的成员蛋白 FASTA。"
|
||
|
||
|
||
def resolve_identify_protein_fasta_for_scope(ws: Path, final_dir: Path, in_dir: Path, prefix: str, scope_data: dict) -> Optional[Path]:
|
||
scope_mode = str((scope_data or {}).get("scope_mode") or "").strip()
|
||
if scope_mode not in {"", "target_only"}:
|
||
return None
|
||
try:
|
||
resolved = resolve_effective_members(
|
||
workspace=ws,
|
||
final_dir=final_dir,
|
||
input_dir=in_dir,
|
||
prefix=prefix,
|
||
phase="auto",
|
||
require="pep",
|
||
family=str((scope_data or {}).get("family") or (prefix.split(".")[-1] if "." in prefix else prefix)).strip(),
|
||
)
|
||
if resolved.path_pep:
|
||
if resolved.warnings:
|
||
print(f"[WARN] {'; '.join(resolved.warnings)}")
|
||
return ensure_protein_fasta_without_stop_markers(resolved.path_pep, final_dir, prefix)
|
||
except MemberResolutionError as exc:
|
||
print(f"[WARN] Effective identify protein FASTA resolution failed for {prefix}: {exc}")
|
||
if "Missing confirmed final members" in str(exc):
|
||
return None
|
||
candidates = [
|
||
final_dir / f"{prefix}.final_members.pep.fa",
|
||
in_dir / f"{prefix}.final_members.pep.fa",
|
||
]
|
||
family = str((scope_data or {}).get("family") or (prefix.split(".")[-1] if "." in prefix else prefix)).strip()
|
||
targets = [str(x).strip() for x in (scope_data or {}).get("target_species") or [] if str(x).strip()]
|
||
if len(targets) == 1:
|
||
single = resolve_species_family_pep(ws, final_dir, in_dir, targets[0], family)
|
||
if single:
|
||
candidates.insert(0, single)
|
||
for candidate in candidates:
|
||
if candidate.exists() and candidate.stat().st_size > 0:
|
||
return ensure_protein_fasta_without_stop_markers(candidate, final_dir, prefix)
|
||
return None
|
||
|
||
|
||
def species_to_string_code(species: str, fallback: str = "ath") -> str:
|
||
return _shared_species_to_string_code(species, fallback)
|
||
|
||
|
||
def resolve_ppi_execution_species(species: str) -> str:
|
||
"""Return a legacy STRING code or a snapshot-verified numeric TaxID."""
|
||
raw = str(species or "").strip()
|
||
if raw.isdigit():
|
||
snapshot = load_string_snapshot(STRING_SPECIES_SNAPSHOT_ROOT)
|
||
if raw not in snapshot.registry:
|
||
raise ValueError(f"STRING TaxID {raw} is not in the fixed snapshot registry")
|
||
return raw
|
||
code = species_to_string_code(raw, "")
|
||
if not code:
|
||
raise ValueError(f"Unsupported or empty STRING species: {species!r}")
|
||
return code
|
||
|
||
|
||
def _normalize_species_alias_token(value: str) -> str:
|
||
return re.sub(r"[^a-z0-9]+", "", str(value or "").strip().lower())
|
||
|
||
|
||
def _preferred_species_code(code: str) -> str:
|
||
raw = str(code or "").strip()
|
||
preferred = {
|
||
"at": "At",
|
||
"ath": "At",
|
||
"ft": "Ft",
|
||
"os": "Os",
|
||
"osa": "Os",
|
||
"zm": "Zm",
|
||
"zma": "Zm",
|
||
"gm": "Gm",
|
||
"gma": "Gm",
|
||
"sl": "Sl",
|
||
"sly": "Sl",
|
||
}
|
||
return preferred.get(raw.lower(), raw)
|
||
|
||
|
||
@lru_cache(maxsize=32)
|
||
def _species_alias_map_for_input_dir(input_dir_text: str) -> Dict[str, str]:
|
||
aliases: Dict[str, str] = {}
|
||
|
||
def add(alias: Any, code: str) -> None:
|
||
code = _preferred_species_code(str(code or "").strip())
|
||
alias_text = str(alias or "").strip()
|
||
if not code or not alias_text:
|
||
return
|
||
aliases[alias_text.lower()] = code
|
||
aliases[_normalize_species_alias_token(alias_text)] = code
|
||
|
||
alias_file = PROJECT_ROOT / "gfa_core" / "config" / "species_aliases.json"
|
||
if alias_file.exists():
|
||
try:
|
||
data = json.loads(alias_file.read_text(encoding="utf-8"))
|
||
if isinstance(data, dict):
|
||
for code, names in data.items():
|
||
add(code, code)
|
||
if isinstance(names, list):
|
||
for name in names:
|
||
add(name, code)
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to read species aliases: {exc}")
|
||
|
||
input_dir = Path(input_dir_text)
|
||
config_xlsx = input_dir / "config.xlsx"
|
||
if config_xlsx.exists():
|
||
try:
|
||
from openpyxl import load_workbook
|
||
|
||
wb = load_workbook(config_xlsx, read_only=True, data_only=True)
|
||
for sheet_name in ("config", "Analysis", "_autofill_review"):
|
||
if sheet_name not in wb.sheetnames:
|
||
continue
|
||
ws = wb[sheet_name]
|
||
rows = ws.iter_rows(values_only=True)
|
||
headers = [str(x or "").strip() for x in (next(rows, []) or [])]
|
||
if "Species_Code" not in headers:
|
||
continue
|
||
code_idx = headers.index("Species_Code")
|
||
name_indexes = [
|
||
idx for idx, header in enumerate(headers)
|
||
if header in {"Species_Name", "Scientific_Name", "Latin_Name", "Species"}
|
||
]
|
||
for row in rows:
|
||
code = str(row[code_idx] or "").strip() if code_idx < len(row) else ""
|
||
if not code:
|
||
continue
|
||
add(code, code)
|
||
for idx in name_indexes:
|
||
if idx < len(row):
|
||
add(row[idx], code)
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to read config species aliases from {config_xlsx}: {exc}")
|
||
return aliases
|
||
|
||
|
||
def canonicalize_species_code_for_job(input_dir: Path, species: str) -> str:
|
||
raw = str(species or "").strip()
|
||
if not raw:
|
||
return ""
|
||
aliases = _species_alias_map_for_input_dir(str(input_dir))
|
||
return aliases.get(raw.lower()) or aliases.get(_normalize_species_alias_token(raw)) or raw
|
||
|
||
|
||
def canonicalize_species_codes_for_job(input_dir: Path, species_values: list[str]) -> list[str]:
|
||
seen = set()
|
||
out = []
|
||
for value in species_values or []:
|
||
mapped = canonicalize_species_code_for_job(input_dir, str(value))
|
||
if mapped and mapped not in seen:
|
||
out.append(mapped)
|
||
seen.add(mapped)
|
||
return out
|
||
|
||
|
||
_PPI_IDENTITY_NAME_COLUMNS = (
|
||
"Scientific_Name",
|
||
"Latin_Name",
|
||
"Species_Name",
|
||
)
|
||
STRING_SPECIES_SNAPSHOT_ROOT = PROJECT_ROOT / "data" / "string" / "v12.0"
|
||
PPI_REFERENCE_POLICY_ID = "string_species_tree_v12_topology_v1"
|
||
_PPI_IDENTITY_TAXON_COLUMNS = (
|
||
"Taxon_ID",
|
||
"TaxID",
|
||
"NCBI_TaxID",
|
||
)
|
||
|
||
|
||
def _normalize_ppi_scientific_name(value: Any) -> str:
|
||
return re.sub(r"\s+", " ", str(value or "").strip().replace("_", " "))
|
||
|
||
|
||
def _normalize_ppi_taxon_id(value: Any) -> str:
|
||
raw = str(value or "").strip()
|
||
if raw.endswith(".0") and raw[:-2].isdigit():
|
||
raw = raw[:-2]
|
||
return raw if raw.isdigit() else ""
|
||
|
||
|
||
def _read_job_species_identity(input_dir: Path, species_code: str) -> Dict[str, Any]:
|
||
config_path = Path(input_dir) / "config.xlsx"
|
||
scientific_names: Dict[str, str] = {}
|
||
taxon_ids: set[str] = set()
|
||
if not config_path.exists():
|
||
return {"scientific_names": [], "taxon_ids": []}
|
||
|
||
try:
|
||
from openpyxl import load_workbook
|
||
|
||
workbook = load_workbook(config_path, read_only=True, data_only=True)
|
||
try:
|
||
for sheet_name in ("config", "Analysis", "_autofill_review"):
|
||
if sheet_name not in workbook.sheetnames:
|
||
continue
|
||
rows = workbook[sheet_name].iter_rows(values_only=True)
|
||
headers = [str(item or "").strip() for item in (next(rows, []) or [])]
|
||
if "Species_Code" not in headers:
|
||
continue
|
||
code_index = headers.index("Species_Code")
|
||
name_indexes = [headers.index(name) for name in _PPI_IDENTITY_NAME_COLUMNS if name in headers]
|
||
taxon_indexes = [headers.index(name) for name in _PPI_IDENTITY_TAXON_COLUMNS if name in headers]
|
||
for row in rows:
|
||
row_code = str(row[code_index] or "").strip() if code_index < len(row) else ""
|
||
if row_code.casefold() != str(species_code or "").strip().casefold():
|
||
continue
|
||
for index in name_indexes:
|
||
name = _normalize_ppi_scientific_name(row[index] if index < len(row) else "")
|
||
if name:
|
||
scientific_names.setdefault(name.casefold(), name)
|
||
for index in taxon_indexes:
|
||
taxon_id = _normalize_ppi_taxon_id(row[index] if index < len(row) else "")
|
||
if taxon_id:
|
||
taxon_ids.add(taxon_id)
|
||
break
|
||
finally:
|
||
workbook.close()
|
||
except Exception as exc:
|
||
return {
|
||
"scientific_names": [],
|
||
"taxon_ids": [],
|
||
"read_error": str(exc),
|
||
}
|
||
response = {
|
||
"scientific_names": list(scientific_names.values()),
|
||
"taxon_ids": sorted(taxon_ids, key=lambda value: int(value)),
|
||
}
|
||
return response
|
||
|
||
|
||
def resolve_ppi_target_identity(
|
||
input_dir: Path,
|
||
species_code: str,
|
||
*,
|
||
explicit_scientific_name: str = "",
|
||
explicit_tree_taxon_id: str = "",
|
||
) -> Dict[str, Any]:
|
||
"""Resolve job-local target identity without making a STRING support decision."""
|
||
code = str(species_code or "").strip()
|
||
configured = _read_job_species_identity(Path(input_dir), code)
|
||
configured_names = configured.get("scientific_names") or []
|
||
configured_taxa = configured.get("taxon_ids") or []
|
||
config_name = configured_names[0] if len(configured_names) == 1 else ""
|
||
config_taxon_id = configured_taxa[0] if len(configured_taxa) == 1 else ""
|
||
explicit_name = _normalize_ppi_scientific_name(explicit_scientific_name)
|
||
explicit_taxon_raw = str(explicit_tree_taxon_id or "").strip()
|
||
explicit_taxon_id = _normalize_ppi_taxon_id(explicit_taxon_raw)
|
||
|
||
base = {
|
||
"species_code": code,
|
||
"scientific_name": "",
|
||
"tree_taxon_id": "",
|
||
"identity_source": "",
|
||
"explicit_scientific_name": explicit_name,
|
||
"explicit_tree_taxon_id": explicit_taxon_raw,
|
||
"config_scientific_name": config_name,
|
||
"config_tree_taxon_id": config_taxon_id,
|
||
}
|
||
if configured.get("read_error"):
|
||
return {
|
||
**base,
|
||
"status": "unresolved",
|
||
"issue_code": "target_identity_config_unreadable",
|
||
"message": f"Cannot read target species identity from config.xlsx: {configured['read_error']}",
|
||
}
|
||
if len(configured_names) > 1 or len(configured_taxa) > 1:
|
||
return {
|
||
**base,
|
||
"status": "conflict",
|
||
"issue_code": "target_species_identity_ambiguous",
|
||
"message": "Target species has multiple scientific names or TaxIDs in config.xlsx.",
|
||
}
|
||
if explicit_taxon_raw and not explicit_taxon_id:
|
||
return {
|
||
**base,
|
||
"status": "unresolved",
|
||
"issue_code": "target_taxon_id_invalid",
|
||
"message": "The explicit target tree TaxID must contain digits only.",
|
||
}
|
||
if explicit_taxon_id and config_taxon_id and explicit_taxon_id != config_taxon_id:
|
||
return {
|
||
**base,
|
||
"status": "conflict",
|
||
"issue_code": "target_taxon_id_conflict",
|
||
"message": "Explicit target tree TaxID conflicts with the unique TaxID in config.xlsx.",
|
||
}
|
||
if explicit_name and config_name and explicit_name.casefold() != config_name.casefold():
|
||
return {
|
||
**base,
|
||
"status": "conflict",
|
||
"issue_code": "target_scientific_name_conflict",
|
||
"message": "Explicit target scientific name conflicts with the unique scientific name in config.xlsx.",
|
||
}
|
||
if explicit_taxon_id:
|
||
return {
|
||
**base,
|
||
"status": "resolved",
|
||
"issue_code": "",
|
||
"message": "Target identity uses the explicit tree TaxID.",
|
||
"scientific_name": explicit_name or config_name,
|
||
"tree_taxon_id": explicit_taxon_id,
|
||
"identity_source": "explicit_taxon_id",
|
||
}
|
||
if config_taxon_id:
|
||
return {
|
||
**base,
|
||
"status": "resolved",
|
||
"issue_code": "",
|
||
"message": "Target identity uses the unique TaxID in config.xlsx.",
|
||
"scientific_name": explicit_name or config_name,
|
||
"tree_taxon_id": config_taxon_id,
|
||
"identity_source": "job_taxon_id",
|
||
}
|
||
if explicit_name or config_name:
|
||
return {
|
||
**base,
|
||
"status": "resolved",
|
||
"issue_code": "",
|
||
"message": "Target identity uses a unique scientific name.",
|
||
"scientific_name": explicit_name or config_name,
|
||
"tree_taxon_id": "",
|
||
"identity_source": "explicit_scientific_name" if explicit_name else "job_scientific_name",
|
||
}
|
||
|
||
from gfa_core.string_species_registry import PROJECT_SPECIES_ALIASES
|
||
|
||
compatibility_taxon_id = PROJECT_SPECIES_ALIASES.get(code.casefold(), "")
|
||
if compatibility_taxon_id:
|
||
return {
|
||
**base,
|
||
"status": "resolved",
|
||
"issue_code": "",
|
||
"message": "Target identity uses an existing curated project alias.",
|
||
"tree_taxon_id": compatibility_taxon_id,
|
||
"identity_source": "project_alias",
|
||
}
|
||
return {
|
||
**base,
|
||
"status": "unresolved",
|
||
"issue_code": "target_string_tree_unresolved",
|
||
"message": "Provide a unique target scientific name or NCBI TaxID; the species code is not guessed.",
|
||
}
|
||
|
||
|
||
def build_ppi_gene_ids_from_ortholog_mapping(mapping_file: Path, output_file: Path, selected_species: list[str] | None = None) -> int:
|
||
selected = {str(x) for x in (selected_species or []) if str(x)}
|
||
seen = set()
|
||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||
with mapping_file.open("r", encoding="utf-8", errors="ignore") as src, output_file.open("w", encoding="utf-8") as dst:
|
||
for raw in src:
|
||
line = raw.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
parts = line.split("\t")
|
||
if not parts:
|
||
continue
|
||
source_id = parts[0].strip()
|
||
source_species = parts[3].strip() if len(parts) >= 4 else ""
|
||
if selected and source_species and source_species not in selected:
|
||
continue
|
||
if source_id and source_id not in seen:
|
||
dst.write(source_id + "\n")
|
||
seen.add(source_id)
|
||
return len(seen)
|
||
|
||
|
||
def infer_string_species_from_ortholog_mapping(mapping_file: Path, fallback: str = "") -> str:
|
||
"""Infer the STRING species to query from an ortholog mapping target column."""
|
||
try:
|
||
with mapping_file.open("r", encoding="utf-8", errors="ignore") as src:
|
||
for raw in src:
|
||
line = raw.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
parts = line.split("\t")
|
||
if len(parts) >= 5:
|
||
inferred = species_to_string_code(parts[4], "")
|
||
if inferred:
|
||
return inferred
|
||
break
|
||
except Exception:
|
||
pass
|
||
return fallback
|
||
|
||
|
||
def infer_reference_species_from_ortholog_mapping(mapping_file: Path, fallback: str = "") -> str:
|
||
"""Infer the reference species label from the first data row of an ortholog mapping."""
|
||
try:
|
||
with mapping_file.open("r", encoding="utf-8", errors="ignore") as src:
|
||
for raw in src:
|
||
line = raw.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
parts = line.split("\t")
|
||
if len(parts) >= 5 and parts[4].strip():
|
||
return parts[4].strip()
|
||
break
|
||
except Exception:
|
||
pass
|
||
return fallback
|
||
|
||
|
||
def _reference_species_from_manifest_data(data: dict[str, Any]) -> str:
|
||
for key in ("reference_species", "referenceSpecies"):
|
||
value = data.get(key)
|
||
if isinstance(value, list) and value:
|
||
return str(value[0] or "").strip()
|
||
if isinstance(value, str) and value.strip():
|
||
return value.strip()
|
||
for section_key in ("parameters", "stage_context"):
|
||
section = data.get(section_key)
|
||
if not isinstance(section, dict):
|
||
continue
|
||
for key in ("reference_species", "referenceSpecies"):
|
||
value = section.get(key)
|
||
if isinstance(value, list) and value:
|
||
return str(value[0] or "").strip()
|
||
if isinstance(value, str) and value.strip():
|
||
return value.strip()
|
||
return ""
|
||
|
||
|
||
def _recent_ppi_kaks_reference_species(final_dir: Path, prefix: str, preferred: Optional[list[str]] = None) -> str:
|
||
"""Recover the technical reference species used by prior Ortholog/PPI runs."""
|
||
candidates: list[Path] = []
|
||
for stage in ("ppi", "ortholog"):
|
||
candidates.extend(final_dir.glob(f"{prefix}*.{stage}.manifest.json"))
|
||
candidates = sorted(
|
||
[path for path in candidates if path.exists() and path.stat().st_size > 0],
|
||
key=lambda path: path.stat().st_mtime,
|
||
reverse=True,
|
||
)
|
||
records: list[tuple[str, str, Path]] = []
|
||
for path in candidates:
|
||
try:
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
if str(data.get("status") or "").lower() not in {"success", "completed"}:
|
||
continue
|
||
species = _reference_species_from_manifest_data(data)
|
||
code = species_to_string_code(species, "")
|
||
if species and code:
|
||
records.append((species, code.lower(), path))
|
||
preferred_codes = [
|
||
species_to_string_code(item, item).lower()
|
||
for item in (preferred or [])
|
||
if str(item or "").strip()
|
||
]
|
||
for preferred_code in preferred_codes:
|
||
if preferred_code == "ath":
|
||
continue
|
||
for species, code, _path in records:
|
||
if code == preferred_code:
|
||
return species
|
||
for species, code, _path in records:
|
||
if code != "ath":
|
||
return species
|
||
if records:
|
||
return records[0][0]
|
||
return ""
|
||
|
||
|
||
def _downloaded_reference_bundle_species(in_dir: Path, preferred: list[str]) -> str:
|
||
preferred_codes = [species_to_string_code(item, item).lower() for item in preferred if str(item or "").strip()]
|
||
manifests = sorted(
|
||
in_dir.glob("*/*.reference_bundle_download.manifest.json"),
|
||
key=lambda path: path.stat().st_mtime if path.exists() else 0,
|
||
reverse=True,
|
||
)
|
||
records: list[tuple[str, Path]] = []
|
||
for manifest in manifests:
|
||
try:
|
||
data = json.loads(manifest.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
code = species_to_string_code(str(data.get("species_code") or manifest.parent.name), "")
|
||
if not code:
|
||
continue
|
||
pep = manifest.parent / f"{code}.pep.fasta"
|
||
if not pep.exists() or pep.stat().st_size <= 0:
|
||
continue
|
||
records.append((code, manifest))
|
||
for preferred_code in preferred_codes:
|
||
for code, _manifest in records:
|
||
if code.lower() == preferred_code:
|
||
return code
|
||
return records[0][0] if records else ""
|
||
|
||
|
||
def _normalized_dependency_stages(requested_stages: Optional[list[str]]) -> list[str]:
|
||
values = [str(x or "").strip().lower() for x in (requested_stages or []) if str(x or "").strip()]
|
||
if not values:
|
||
values = ["ppi"]
|
||
return list(dict.fromkeys(values))
|
||
|
||
|
||
def _reference_species_matches(expected: str, observed: str) -> bool:
|
||
expected_text = str(expected or "").strip()
|
||
observed_text = str(observed or "").strip()
|
||
if not expected_text or not observed_text:
|
||
return True
|
||
expected_code = species_to_string_code(expected_text, "")
|
||
observed_code = species_to_string_code(observed_text, "")
|
||
if expected_code or observed_code:
|
||
return bool(expected_code and observed_code and expected_code == observed_code)
|
||
return expected_text.lower().replace("_", " ") == observed_text.lower().replace("_", " ")
|
||
|
||
|
||
def _reference_annotation_candidate_paths(
|
||
*,
|
||
in_dir: Path,
|
||
species: str,
|
||
sequence_type: str,
|
||
roots: Optional[list[Path]] = None,
|
||
) -> list[Path]:
|
||
raw = str(species or "").strip()
|
||
code = species_to_string_code(raw, raw.lower())
|
||
tokens = []
|
||
for token in [raw, raw.lower(), raw.replace(" ", "_"), code]:
|
||
if token and token not in tokens:
|
||
tokens.append(token)
|
||
roots = roots or [in_dir, Path("/data/ref_genome"), Path("/data")]
|
||
suffixes_by_type = {
|
||
"gff3": ["{token}.gff3", "{token}.gff", "{token}.gtf", "{token}_final.gff3"],
|
||
"genome": ["{token}.genome.fasta", "{token}.genome.fa", "{token}.fasta", "{token}.fa", "{token}.fna"],
|
||
}
|
||
suffixes = suffixes_by_type.get(sequence_type, [])
|
||
candidates: list[Path] = []
|
||
for root in roots:
|
||
for token in tokens:
|
||
for suffix in suffixes:
|
||
name = suffix.format(token=token)
|
||
candidates.extend([
|
||
root / name,
|
||
root / token / name,
|
||
root / "genome_prepare" / name,
|
||
])
|
||
seen = set()
|
||
unique: list[Path] = []
|
||
for path in candidates:
|
||
text = str(path)
|
||
if text not in seen:
|
||
unique.append(path)
|
||
seen.add(text)
|
||
return unique
|
||
|
||
|
||
def resolve_reference_annotation_file(
|
||
*,
|
||
in_dir: Path,
|
||
species: str,
|
||
sequence_type: str,
|
||
) -> Optional[Path]:
|
||
for candidate in _reference_annotation_candidate_paths(in_dir=in_dir, species=species, sequence_type=sequence_type):
|
||
if candidate.exists() and candidate.stat().st_size > 0:
|
||
return candidate
|
||
return None
|
||
|
||
|
||
NCBI_DATASETS_DOWNLOAD_BASE = "https://api.ncbi.nlm.nih.gov/datasets/v2"
|
||
NCBI_REFERENCE_ANNOTATION_TYPES = {
|
||
"pep": "PROT_FASTA",
|
||
"cds": "CDS_FASTA",
|
||
"genome": "GENOME_FASTA",
|
||
"gff3": "GENOME_GFF",
|
||
}
|
||
|
||
|
||
def _reference_bundle_file_plan(requested_stages: Optional[list[str]], include_recommended: bool = False) -> list[str]:
|
||
stages = _normalized_dependency_stages(requested_stages)
|
||
files = ["pep"]
|
||
if "kaks" in stages:
|
||
files.append("cds")
|
||
if include_recommended:
|
||
for item in ("gff3", "genome"):
|
||
if item not in files:
|
||
files.append(item)
|
||
return files
|
||
|
||
|
||
def _urlopen_json_with_retry(url: str, *, timeout_sec: int = 30, attempts: int = 3) -> dict[str, Any]:
|
||
last_error: Exception | None = None
|
||
for attempt in range(1, max(1, attempts) + 1):
|
||
try:
|
||
with urllib.request.urlopen(url, timeout=timeout_sec) as response:
|
||
return json.loads(response.read().decode("utf-8"))
|
||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||
last_error = exc
|
||
if attempt < attempts:
|
||
time.sleep(min(8, 2 ** (attempt - 1)))
|
||
continue
|
||
break
|
||
raise RuntimeError(f"NCBI request failed after {attempts} attempts: {last_error}")
|
||
|
||
|
||
def _select_ncbi_reference_assembly(*, taxon_id: str = "", species_name: str = "", timeout_sec: int = 30) -> dict[str, str]:
|
||
taxon = str(taxon_id or species_name or "").strip()
|
||
if not taxon:
|
||
raise ValueError("taxon_id or species_name is required for NCBI reference bundle download")
|
||
url = (
|
||
f"{NCBI_DATASETS_DOWNLOAD_BASE}/genome/taxon/"
|
||
f"{urllib.parse.quote(taxon, safe='')}/dataset_report?page_size=10"
|
||
)
|
||
payload = _urlopen_json_with_retry(url, timeout_sec=timeout_sec, attempts=3)
|
||
reports = payload.get("reports") or []
|
||
if not reports:
|
||
raise RuntimeError(f"NCBI Datasets returned no assemblies for taxon/species: {taxon}")
|
||
|
||
def score(report: dict[str, Any]) -> tuple[int, int, int]:
|
||
source = str(report.get("source_database") or "")
|
||
accession = str(report.get("current_accession") or report.get("accession") or "")
|
||
assembly_info = report.get("assembly_info") if isinstance(report.get("assembly_info"), dict) else {}
|
||
level = str(assembly_info.get("assembly_level") or "")
|
||
annotated = bool(assembly_info.get("annotation_name"))
|
||
return (
|
||
1 if accession.startswith("GCF_") or "REFSEQ" in source else 0,
|
||
1 if annotated else 0,
|
||
1 if level.lower() in {"chromosome", "complete genome"} else 0,
|
||
)
|
||
|
||
selected = sorted(reports, key=score, reverse=True)[0]
|
||
assembly_info = selected.get("assembly_info") if isinstance(selected.get("assembly_info"), dict) else {}
|
||
organism = selected.get("organism") if isinstance(selected.get("organism"), dict) else {}
|
||
return {
|
||
"accession": str(selected.get("current_accession") or selected.get("accession") or ""),
|
||
"organism_name": str(organism.get("organism_name") or species_name or ""),
|
||
"taxon_id": str(organism.get("tax_id") or taxon_id or ""),
|
||
"assembly_name": str(assembly_info.get("assembly_name") or ""),
|
||
"assembly_level": str(assembly_info.get("assembly_level") or ""),
|
||
"annotation_name": str(assembly_info.get("annotation_name") or ""),
|
||
}
|
||
|
||
|
||
def _download_url_to_file(
|
||
url: str,
|
||
output_path: Path,
|
||
timeout_sec: int = 300,
|
||
attempts: int = 3,
|
||
progress_callback: Optional[Callable[[dict[str, Any]], None]] = None,
|
||
) -> int:
|
||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||
temp_path = output_path.with_suffix(output_path.suffix + ".part")
|
||
last_error: Exception | None = None
|
||
for attempt in range(1, max(1, attempts) + 1):
|
||
total = 0
|
||
try:
|
||
if temp_path.exists():
|
||
temp_path.unlink()
|
||
with urllib.request.urlopen(url, timeout=timeout_sec) as response, temp_path.open("wb") as dst:
|
||
total_bytes = int(response.headers.get("Content-Length") or 0)
|
||
while True:
|
||
chunk = response.read(1024 * 1024)
|
||
if not chunk:
|
||
break
|
||
dst.write(chunk)
|
||
total += len(chunk)
|
||
if progress_callback:
|
||
progress_callback({
|
||
"phase": "download",
|
||
"bytes_downloaded": total,
|
||
"total_bytes": total_bytes,
|
||
"percentage": round(total * 100 / total_bytes, 1) if total_bytes else None,
|
||
"indeterminate": not bool(total_bytes),
|
||
})
|
||
if total <= 0:
|
||
raise RuntimeError("downloaded file is empty")
|
||
temp_path.replace(output_path)
|
||
return total
|
||
except (urllib.error.URLError, TimeoutError, OSError, RuntimeError) as exc:
|
||
last_error = exc
|
||
if temp_path.exists():
|
||
try:
|
||
temp_path.unlink()
|
||
except OSError:
|
||
pass
|
||
if attempt < attempts:
|
||
time.sleep(min(15, 2 ** (attempt - 1)))
|
||
continue
|
||
break
|
||
raise RuntimeError(f"NCBI reference bundle download failed after {attempts} attempts: {last_error}")
|
||
|
||
|
||
def _ncbi_dataset_download_url(accession: str, file_types: list[str]) -> str:
|
||
params = [
|
||
("include_annotation_type", NCBI_REFERENCE_ANNOTATION_TYPES[file_type])
|
||
for file_type in file_types
|
||
if file_type in NCBI_REFERENCE_ANNOTATION_TYPES
|
||
]
|
||
if not params:
|
||
raise ValueError("No supported reference bundle file types requested")
|
||
return (
|
||
f"{NCBI_DATASETS_DOWNLOAD_BASE}/genome/accession/"
|
||
f"{urllib.parse.quote(str(accession), safe='')}/download?"
|
||
+ urllib.parse.urlencode(params)
|
||
)
|
||
|
||
|
||
def _extract_ncbi_reference_bundle_zip(
|
||
*,
|
||
zip_path: Path,
|
||
destination_dir: Path,
|
||
species_code: str,
|
||
requested_file_types: list[str],
|
||
) -> dict[str, str]:
|
||
destination_dir.mkdir(parents=True, exist_ok=True)
|
||
patterns = {
|
||
"pep": ("protein.faa", "_protein.faa", ".faa"),
|
||
"cds": ("cds_from_genomic.fna", "_cds_from_genomic.fna"),
|
||
"genome": ("genomic.fna", "_genomic.fna"),
|
||
"gff3": ("genomic.gff", "_genomic.gff", ".gff"),
|
||
}
|
||
output_names = {
|
||
"pep": f"{species_code}.pep.fasta",
|
||
"cds": f"{species_code}.cds.fasta",
|
||
"genome": f"{species_code}.genome.fasta",
|
||
"gff3": f"{species_code}.gff3",
|
||
}
|
||
extracted: dict[str, str] = {}
|
||
with zipfile.ZipFile(zip_path) as zf:
|
||
members = [name for name in zf.namelist() if not name.endswith("/")]
|
||
for file_type in requested_file_types:
|
||
suffixes = patterns.get(file_type, ())
|
||
match = next(
|
||
(
|
||
name for name in members
|
||
if any(Path(name).name.endswith(suffix) for suffix in suffixes)
|
||
),
|
||
"",
|
||
)
|
||
if not match:
|
||
continue
|
||
output_path = destination_dir / output_names[file_type]
|
||
with zf.open(match) as src, output_path.open("wb") as dst:
|
||
shutil.copyfileobj(src, dst)
|
||
if file_type == "cds":
|
||
_normalize_ncbi_cds_headers_to_protein_ids(output_path)
|
||
if output_path.exists() and output_path.stat().st_size > 0:
|
||
extracted[file_type] = str(output_path)
|
||
return extracted
|
||
|
||
|
||
def _normalize_ncbi_cds_headers_to_protein_ids(cds_path: Path) -> None:
|
||
"""Make NCBI CDS FASTA IDs match the corresponding protein FASTA accessions."""
|
||
text = cds_path.read_text(encoding="utf-8", errors="ignore")
|
||
lines = []
|
||
changed = False
|
||
for line in text.splitlines():
|
||
if not line.startswith(">"):
|
||
lines.append(line)
|
||
continue
|
||
header = line[1:].strip()
|
||
protein_id = ""
|
||
match = re.search(r"\[protein_id=([^\]\s]+)\]", header)
|
||
if match:
|
||
protein_id = match.group(1)
|
||
if not protein_id:
|
||
match = re.search(r"_cds_([A-Z]{2}_[0-9]+\.[0-9]+)", header)
|
||
if match:
|
||
protein_id = match.group(1)
|
||
if protein_id:
|
||
lines.append(f">{protein_id} {header}")
|
||
changed = True
|
||
else:
|
||
lines.append(line)
|
||
if changed:
|
||
cds_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||
|
||
|
||
def _fasta_header_index(fasta_path: Path | str | None) -> dict[str, dict[str, str]]:
|
||
path = Path(fasta_path) if fasta_path else None
|
||
if not path or not path.exists() or path.stat().st_size <= 0:
|
||
return {}
|
||
records: dict[str, dict[str, str]] = {}
|
||
with path.open("r", encoding="utf-8", errors="ignore") as handle:
|
||
for line in handle:
|
||
if not line.startswith(">"):
|
||
continue
|
||
header = line[1:].strip()
|
||
seq_id = header.split()[0] if header else ""
|
||
if not seq_id or seq_id in records:
|
||
continue
|
||
gene_match = re.search(r"\[gene=([^\]]+)\]", header)
|
||
ncbi_gene_match = re.search(r"\[db_xref=GeneID:(\d+)\]", header)
|
||
protein_match = re.search(r"\[protein_id=([^\]]+)\]", header)
|
||
records[seq_id] = {
|
||
"id": seq_id,
|
||
"header": header,
|
||
"gene_id": gene_match.group(1) if gene_match else "",
|
||
"ncbi_gene_id": ncbi_gene_match.group(1) if ncbi_gene_match else "",
|
||
"protein_id": protein_match.group(1) if protein_match else "",
|
||
}
|
||
return records
|
||
|
||
|
||
def _build_reference_bundle_id_map(
|
||
*,
|
||
species_code: str,
|
||
pep_path: Path | str | None,
|
||
cds_path: Path | str | None,
|
||
output_path: Path,
|
||
) -> dict[str, Any]:
|
||
"""Write a provenance TSV proving downloaded PEP/CDS IDs are aligned."""
|
||
pep_records = _fasta_header_index(pep_path)
|
||
cds_records = _fasta_header_index(cds_path)
|
||
all_ids = sorted(set(pep_records) | set(cds_records))
|
||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||
with output_path.open("w", encoding="utf-8", newline="") as handle:
|
||
writer = csv.DictWriter(
|
||
handle,
|
||
fieldnames=[
|
||
"species_code",
|
||
"protein_id",
|
||
"pep_id",
|
||
"cds_id",
|
||
"gene_id",
|
||
"ncbi_gene_id",
|
||
"pep_present",
|
||
"cds_present",
|
||
"pep_header",
|
||
"cds_header",
|
||
],
|
||
delimiter="\t",
|
||
)
|
||
writer.writeheader()
|
||
for seq_id in all_ids:
|
||
pep = pep_records.get(seq_id, {})
|
||
cds = cds_records.get(seq_id, {})
|
||
writer.writerow({
|
||
"species_code": species_code,
|
||
"protein_id": seq_id,
|
||
"pep_id": pep.get("id", ""),
|
||
"cds_id": cds.get("id", ""),
|
||
"gene_id": cds.get("gene_id", "") or pep.get("gene_id", ""),
|
||
"ncbi_gene_id": cds.get("ncbi_gene_id", "") or pep.get("ncbi_gene_id", ""),
|
||
"pep_present": "1" if seq_id in pep_records else "0",
|
||
"cds_present": "1" if seq_id in cds_records else "0",
|
||
"pep_header": pep.get("header", ""),
|
||
"cds_header": cds.get("header", ""),
|
||
})
|
||
overlap = sorted(set(pep_records) & set(cds_records))
|
||
pep_only = sorted(set(pep_records) - set(cds_records))
|
||
cds_only = sorted(set(cds_records) - set(pep_records))
|
||
return {
|
||
"species_code": species_code,
|
||
"pep_records": len(pep_records),
|
||
"cds_records": len(cds_records),
|
||
"pep_cds_overlap": len(overlap),
|
||
"pep_only_count": len(pep_only),
|
||
"cds_only_count": len(cds_only),
|
||
"pep_cds_overlap_ratio": (len(overlap) / len(pep_records)) if pep_records else 0,
|
||
"id_map_path": str(output_path),
|
||
"pep_only_examples": pep_only[:10],
|
||
"cds_only_examples": cds_only[:10],
|
||
}
|
||
|
||
|
||
def download_reference_annotation_bundle(
|
||
*,
|
||
job: "JobInfo",
|
||
species_code: str,
|
||
species_name: str = "",
|
||
taxon_id: str = "",
|
||
requested_stages: Optional[list[str]] = None,
|
||
include_recommended: bool = False,
|
||
accession: str = "",
|
||
timeout_sec: int = 300,
|
||
progress_callback: Optional[Callable[[dict[str, Any]], None]] = None,
|
||
) -> dict[str, Any]:
|
||
"""Download a job-local NCBI reference annotation bundle for PPI/KaKs fallback."""
|
||
def emit(progress: dict[str, Any]) -> None:
|
||
if progress_callback:
|
||
progress_callback({"stage": "reference_bundle_download", **progress})
|
||
|
||
in_dir = Path(job.input_dir)
|
||
ws = Path(job.workspace_dir)
|
||
code = species_to_string_code(species_code or species_name, species_code or "").strip() or str(species_code or "").strip()
|
||
code = code or "reference"
|
||
required_file_types = _reference_bundle_file_plan(requested_stages, include_recommended=False)
|
||
file_types = _reference_bundle_file_plan(
|
||
requested_stages,
|
||
include_recommended=include_recommended,
|
||
)
|
||
assembly = {"accession": accession}
|
||
emit({"phase": "select_assembly", "message": "正在解析推荐参考物种装配", "percentage": 5, "indeterminate": True})
|
||
if not accession:
|
||
assembly = _select_ncbi_reference_assembly(taxon_id=taxon_id, species_name=species_name, timeout_sec=min(timeout_sec, 60))
|
||
accession = assembly.get("accession", "")
|
||
if not accession:
|
||
raise RuntimeError("No NCBI assembly accession resolved for reference bundle download")
|
||
|
||
download_url = _ncbi_dataset_download_url(accession, file_types)
|
||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
work_dir = ws / "reference_bundle_download" / f"{code}_{stamp}"
|
||
zip_path = work_dir / f"{code}.ncbi_dataset.zip"
|
||
emit({"phase": "download", "message": "正在下载 NCBI 参考注释包", "percentage": 10, "indeterminate": True})
|
||
download_kwargs = {"timeout_sec": timeout_sec}
|
||
if progress_callback:
|
||
download_kwargs["progress_callback"] = lambda event: emit({"message": "正在下载 NCBI 参考注释包", **event})
|
||
bytes_downloaded = _download_url_to_file(download_url, zip_path, **download_kwargs)
|
||
if not zipfile.is_zipfile(zip_path):
|
||
raise RuntimeError(
|
||
f"NCBI reference bundle download returned an invalid zip archive: {zip_path}"
|
||
)
|
||
species_dir = in_dir / code
|
||
emit({"phase": "extract", "message": "正在解压参考 PEP/CDS", "percentage": 80, "indeterminate": True})
|
||
extracted = _extract_ncbi_reference_bundle_zip(
|
||
zip_path=zip_path,
|
||
destination_dir=species_dir,
|
||
species_code=code,
|
||
requested_file_types=file_types,
|
||
)
|
||
emit({"phase": "validate_ids", "message": "正在校验 PEP/CDS ID 对齐", "percentage": 90, "indeterminate": True})
|
||
id_validation = _build_reference_bundle_id_map(
|
||
species_code=code,
|
||
pep_path=extracted.get("pep"),
|
||
cds_path=extracted.get("cds"),
|
||
output_path=species_dir / f"{code}.pep_cds_idmap.tsv",
|
||
)
|
||
if Path(id_validation["id_map_path"]).exists():
|
||
extracted["pep_cds_idmap"] = id_validation["id_map_path"]
|
||
missing_required = [item for item in required_file_types if item not in extracted]
|
||
manifest = {
|
||
"manifest_version": "1.0",
|
||
"stage": "reference_bundle_download",
|
||
"status": "success" if not missing_required else "incomplete",
|
||
"job_id": job.job_id,
|
||
"species_code": code,
|
||
"species_name": species_name or assembly.get("organism_name", ""),
|
||
"taxon_id": taxon_id or assembly.get("taxon_id", ""),
|
||
"requested_stages": _normalized_dependency_stages(requested_stages),
|
||
"requested_file_types": file_types,
|
||
"include_recommended": include_recommended,
|
||
"assembly": assembly,
|
||
"download_url": download_url,
|
||
"zip_path": str(zip_path),
|
||
"bytes_downloaded": bytes_downloaded,
|
||
"files": extracted,
|
||
"id_validation": id_validation,
|
||
"missing_required": missing_required,
|
||
"created_at": datetime.now().isoformat(),
|
||
"provenance": {
|
||
"source": "NCBI Datasets v2",
|
||
"storage": "job_input",
|
||
"destination_dir": str(species_dir),
|
||
},
|
||
}
|
||
manifest_path = species_dir / f"{code}.reference_bundle_download.manifest.json"
|
||
manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
manifest["manifest_path"] = str(manifest_path)
|
||
emit({
|
||
"phase": "completed",
|
||
"message": "参考物种注释包下载完成",
|
||
"percentage": 100,
|
||
"indeterminate": False,
|
||
"bytes_downloaded": bytes_downloaded,
|
||
"files": extracted,
|
||
})
|
||
return manifest
|
||
|
||
|
||
def _ppi_only_preflight_response(
|
||
*,
|
||
status: str,
|
||
issue_code: str,
|
||
message: str,
|
||
decision: str,
|
||
target_species: list[str],
|
||
target_identity: dict[str, Any],
|
||
species_resolution: dict[str, Any],
|
||
requested_ppi_species: str,
|
||
selected_taxon_id: str = "",
|
||
selected_string_species: str = "",
|
||
reference_storage_key: str = "",
|
||
reference_pep: str = "",
|
||
mapping_file: Optional[Path] = None,
|
||
mapping_reference_species: str = "",
|
||
required_next_stage: str = "",
|
||
missing_required: Optional[list[str]] = None,
|
||
dependency_owner: str = "ppi",
|
||
) -> dict[str, Any]:
|
||
candidates = list(species_resolution.get("candidates") or [])
|
||
selection_mode = str(species_resolution.get("selection_mode") or "unresolved")
|
||
reference_bundle_status = {
|
||
"required_files": ["pep"],
|
||
"recommended_files": [],
|
||
"files": {"pep": reference_pep} if reference_pep else {},
|
||
"missing_required": list(missing_required or []),
|
||
"missing_recommended": [],
|
||
"mode": (
|
||
"complete"
|
||
if reference_pep
|
||
else "not_selected"
|
||
if status in {"confirmation_required", "unresolved", "conflict"}
|
||
else "unavailable"
|
||
if status == "resource_error"
|
||
else "incomplete"
|
||
),
|
||
"kaks_ready": True,
|
||
}
|
||
diagnostics = {
|
||
"issue_code": issue_code,
|
||
"severity": "info" if status == "ready" else "blocking",
|
||
"layer": "resource" if status == "resource_error" else "input_contract",
|
||
"required_next_stage": required_next_stage,
|
||
"recommendation": message,
|
||
"dependency_owner_stages": ["ppi"],
|
||
"required_bundle_files": ["pep"],
|
||
"reference_bundle_status": reference_bundle_status,
|
||
"ortholog_dependency_role": "producer" if required_next_stage == "ortholog" else "not_requested",
|
||
"reference_recommendations": candidates,
|
||
}
|
||
response = {
|
||
"status": status,
|
||
"issue_code": issue_code,
|
||
"message": message,
|
||
"target_species": target_species,
|
||
"target_identity": target_identity,
|
||
"target_string_supported": bool(species_resolution.get("target_string_supported")),
|
||
"requested_stages": ["ppi"],
|
||
"requested_ppi_species": requested_ppi_species or "auto",
|
||
"selected_string_species": selected_string_species,
|
||
"selected_taxon_id": selected_taxon_id,
|
||
"reference_species": reference_storage_key,
|
||
"reference_data_source": reference_pep,
|
||
"reference_bundle_status": reference_bundle_status,
|
||
"mapping_reference_species": mapping_reference_species,
|
||
"ortholog_mapping_source": str(mapping_file) if mapping_file else "",
|
||
"required_next_stage": required_next_stage,
|
||
"decision": decision,
|
||
"selection_mode": selection_mode,
|
||
"candidates": candidates,
|
||
"requires_user_confirmation": bool(species_resolution.get("requires_user_confirmation")),
|
||
"snapshot_id": str(species_resolution.get("snapshot_id") or ""),
|
||
"ppi_reference_policy_id": PPI_REFERENCE_POLICY_ID,
|
||
"ppi_reference_storage_key": reference_storage_key,
|
||
"dependency_owner": dependency_owner or "ppi",
|
||
"species_resolution": species_resolution,
|
||
"diagnostics": diagnostics,
|
||
}
|
||
response["ppi_provenance"] = build_ppi_provenance(
|
||
raw_target_species=target_species,
|
||
target_identity=target_identity,
|
||
species_resolution=species_resolution,
|
||
selected_string_taxon_id=selected_taxon_id,
|
||
selection_mode=selection_mode,
|
||
user_confirmed=bool(species_resolution.get("reference_user_confirmed")),
|
||
mapping_reference_species=mapping_reference_species or reference_storage_key,
|
||
mapping_path=str(mapping_file) if mapping_file else "",
|
||
normalization_species_label=(target_species[0] if target_species else selected_string_species),
|
||
snapshot_root=STRING_SPECIES_SNAPSHOT_ROOT,
|
||
status=status,
|
||
)
|
||
return response
|
||
|
||
|
||
def _effective_ppi_owner_scope_id(
|
||
*,
|
||
family: str,
|
||
selected_targets: list[str],
|
||
scope_data: dict[str, Any],
|
||
reference_storage_key: str,
|
||
) -> str:
|
||
scope_mode = str(scope_data.get("scope_mode") or "target_plus_reference")
|
||
comparative_species = [
|
||
str(x) for x in scope_data.get("comparative_species", []) if str(x)
|
||
]
|
||
if scope_mode == "target_only":
|
||
return str(scope_data.get("scope_id") or "") or build_scope_id(
|
||
family,
|
||
selected_targets,
|
||
[str(x) for x in scope_data.get("reference_species", []) if str(x)],
|
||
comparative_species,
|
||
scope_mode,
|
||
)
|
||
return build_scope_id(
|
||
family,
|
||
selected_targets,
|
||
[reference_storage_key],
|
||
comparative_species,
|
||
scope_mode,
|
||
)
|
||
|
||
|
||
def _ppi_mapping_identity_matches_current_inputs(
|
||
*,
|
||
mapping_file: Path,
|
||
final_dir: Path,
|
||
in_dir: Path,
|
||
ws: Path,
|
||
prefix: str,
|
||
family: str,
|
||
selected_targets: list[str],
|
||
scope_data: dict[str, Any],
|
||
reference_taxon_id: str,
|
||
reference_storage_key: str,
|
||
reference_pep: Path,
|
||
) -> tuple[bool, str]:
|
||
if len(selected_targets) != 1:
|
||
return False, "ppi_owner_identity_requires_single_target"
|
||
query_fasta = resolve_species_family_pep(ws, final_dir, in_dir, selected_targets[0], family)
|
||
if not query_fasta:
|
||
return False, "ppi_owner_target_pep_missing"
|
||
effective_scope_id = _effective_ppi_owner_scope_id(
|
||
family=family,
|
||
selected_targets=selected_targets,
|
||
scope_data=scope_data,
|
||
reference_storage_key=reference_storage_key,
|
||
)
|
||
from gfa_core.ortholog_materialize import build_ortholog_run_identity, ortholog_run_identity_matches
|
||
|
||
expected = build_ortholog_run_identity(
|
||
query_species=selected_targets[0],
|
||
reference_taxon_id=reference_taxon_id,
|
||
reference_storage_key=reference_storage_key,
|
||
query_fasta=query_fasta,
|
||
reference_pep=reference_pep,
|
||
scope_id=effective_scope_id,
|
||
member_set_id="",
|
||
mapping_path=mapping_file,
|
||
)
|
||
if not ortholog_run_identity_matches(mapping_file, expected):
|
||
return False, "ppi_owner_mapping_sidecar_identity_mismatch"
|
||
|
||
manifest_candidates = sorted(
|
||
final_dir.glob(f"{prefix}*.ortholog.manifest.json"),
|
||
key=lambda path: path.stat().st_mtime,
|
||
reverse=True,
|
||
)
|
||
identity_keys = (
|
||
"reference_taxon_id",
|
||
"reference_storage_key",
|
||
"reference_pep_sha256",
|
||
"target_identity",
|
||
"member_identity",
|
||
"mapping_identity",
|
||
"mapping_path",
|
||
)
|
||
for manifest_path in manifest_candidates:
|
||
try:
|
||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
params = payload.get("parameters") if isinstance(payload.get("parameters"), dict) else {}
|
||
if all(params.get(key) == expected.get(key) for key in identity_keys):
|
||
return True, ""
|
||
return False, "ppi_owner_mapping_manifest_identity_mismatch"
|
||
|
||
|
||
def _resolve_ppi_only_snapshot_preflight(
|
||
*,
|
||
job: "JobInfo",
|
||
prefix: str,
|
||
family: str,
|
||
selected_targets: list[str],
|
||
scope_data: dict[str, Any],
|
||
scope_id: str,
|
||
target_identity: dict[str, Any],
|
||
requested_ppi_species: str,
|
||
ignore_existing_ortholog: bool,
|
||
ppi_reference_taxon_id: str,
|
||
ppi_reference_storage_key: str,
|
||
ppi_reference_selection_mode: str,
|
||
ppi_reference_user_confirmed: bool,
|
||
dependency_owner: str,
|
||
) -> dict[str, Any]:
|
||
identity_status = str(target_identity.get("status") or "unresolved")
|
||
if identity_status in {"conflict", "unresolved"}:
|
||
issue_code = str(target_identity.get("issue_code") or "target_string_tree_unresolved")
|
||
species_resolution = {
|
||
"selection_mode": "unresolved",
|
||
"target_tree_taxon_id": "",
|
||
"target_tree_name": "",
|
||
"selected_string_taxon_id": "",
|
||
"selected_string_name": "",
|
||
"target_string_supported": False,
|
||
"requires_user_confirmation": False,
|
||
"resolution_source": str(target_identity.get("identity_source") or "target_identity"),
|
||
"candidate_source": "",
|
||
"candidates": [],
|
||
"issue_code": issue_code,
|
||
"string_version": "",
|
||
"snapshot_id": "",
|
||
"ppi_reference_policy_id": PPI_REFERENCE_POLICY_ID,
|
||
}
|
||
return _ppi_only_preflight_response(
|
||
status=identity_status,
|
||
issue_code=issue_code,
|
||
message=str(target_identity.get("message") or "Target STRING tree identity is unresolved."),
|
||
decision="target_identity_conflict" if identity_status == "conflict" else "target_identity_unresolved",
|
||
target_species=selected_targets,
|
||
target_identity=target_identity,
|
||
species_resolution=species_resolution,
|
||
requested_ppi_species=requested_ppi_species,
|
||
dependency_owner=dependency_owner,
|
||
)
|
||
|
||
try:
|
||
snapshot = load_string_snapshot(STRING_SPECIES_SNAPSHOT_ROOT)
|
||
species_resolution = resolve_ppi_string_species(
|
||
snapshot,
|
||
selected_targets[0] if len(selected_targets) == 1 else "",
|
||
target_scientific_name=str(target_identity.get("scientific_name") or ""),
|
||
ppi_target_tree_taxon_id=str(target_identity.get("tree_taxon_id") or ""),
|
||
)
|
||
except StringSnapshotError as exc:
|
||
species_resolution = {
|
||
"selection_mode": "resource_error",
|
||
"target_tree_taxon_id": str(target_identity.get("tree_taxon_id") or ""),
|
||
"target_tree_name": str(target_identity.get("scientific_name") or ""),
|
||
"selected_string_taxon_id": "",
|
||
"selected_string_name": "",
|
||
"target_string_supported": False,
|
||
"requires_user_confirmation": False,
|
||
"resolution_source": str(target_identity.get("identity_source") or ""),
|
||
"candidate_source": "",
|
||
"candidates": [],
|
||
"issue_code": exc.code,
|
||
"string_version": "",
|
||
"snapshot_id": "",
|
||
"ppi_reference_policy_id": PPI_REFERENCE_POLICY_ID,
|
||
}
|
||
return _ppi_only_preflight_response(
|
||
status="resource_error",
|
||
issue_code=exc.code,
|
||
message=str(exc),
|
||
decision="string_registry_error",
|
||
target_species=selected_targets,
|
||
target_identity=target_identity,
|
||
species_resolution=species_resolution,
|
||
requested_ppi_species=requested_ppi_species,
|
||
dependency_owner=dependency_owner,
|
||
)
|
||
|
||
species_resolution = {
|
||
**species_resolution,
|
||
"ppi_reference_policy_id": PPI_REFERENCE_POLICY_ID,
|
||
}
|
||
selection_mode = str(species_resolution.get("selection_mode") or "unresolved")
|
||
if selection_mode == "exact_string":
|
||
selected_taxon_id = str(species_resolution.get("selected_string_taxon_id") or "")
|
||
compatibility_code = species_to_string_code(
|
||
selected_targets[0] if len(selected_targets) == 1 else "",
|
||
"",
|
||
)
|
||
return _ppi_only_preflight_response(
|
||
status="ready",
|
||
issue_code="",
|
||
message="Target species is directly supported by the fixed STRING snapshot.",
|
||
decision="ready_target_string",
|
||
target_species=selected_targets,
|
||
target_identity=target_identity,
|
||
species_resolution=species_resolution,
|
||
requested_ppi_species=requested_ppi_species,
|
||
selected_taxon_id=selected_taxon_id,
|
||
selected_string_species=compatibility_code,
|
||
dependency_owner=dependency_owner,
|
||
)
|
||
if selection_mode != "tree_nearest":
|
||
return _ppi_only_preflight_response(
|
||
status="unresolved",
|
||
issue_code=str(species_resolution.get("issue_code") or "target_string_tree_unresolved"),
|
||
message="Target species could not be resolved in the fixed STRING species tree.",
|
||
decision="target_identity_unresolved",
|
||
target_species=selected_targets,
|
||
target_identity=target_identity,
|
||
species_resolution=species_resolution,
|
||
requested_ppi_species=requested_ppi_species,
|
||
dependency_owner=dependency_owner,
|
||
)
|
||
|
||
chosen_taxon_id = str(ppi_reference_taxon_id or "").strip()
|
||
chosen = next(
|
||
(
|
||
row
|
||
for row in species_resolution.get("candidates") or []
|
||
if str(row.get("selected_string_taxon_id") or "") == chosen_taxon_id
|
||
),
|
||
None,
|
||
)
|
||
if not (ppi_reference_user_confirmed and chosen):
|
||
return _ppi_only_preflight_response(
|
||
status="confirmation_required",
|
||
issue_code="ppi_reference_confirmation_required",
|
||
message="Select and confirm one of the nearest STRING reference species before downloading PEP or running Ortholog.",
|
||
decision="confirmation_required",
|
||
target_species=selected_targets,
|
||
target_identity=target_identity,
|
||
species_resolution=species_resolution,
|
||
requested_ppi_species=requested_ppi_species,
|
||
dependency_owner=dependency_owner,
|
||
)
|
||
|
||
storage_key = str(ppi_reference_storage_key or "").strip() or f"string_taxon_{chosen_taxon_id}"
|
||
selected_name = str(chosen.get("selected_string_name") or "")
|
||
ws = Path(job.workspace_dir)
|
||
in_dir = Path(job.input_dir)
|
||
final_dir = ws / "final"
|
||
reference_pep_path = resolve_species_sequence_file(
|
||
ws, final_dir, in_dir, scope_data, storage_key, family, "pep", family_only=False
|
||
)
|
||
reference_pep = str(reference_pep_path) if reference_pep_path else ""
|
||
mapping_candidates: list[Path] = []
|
||
if scope_id:
|
||
mapping_candidates.extend([
|
||
final_dir / f"{prefix}.{sanitize_token(scope_id)}.orthologs.txt",
|
||
final_dir / f"{prefix}.{sanitize_token(scope_id)}.ortholog_pairs.txt",
|
||
])
|
||
mapping_candidates.extend([
|
||
final_dir / f"{prefix}.orthologs.txt",
|
||
final_dir / f"{prefix}.ortholog_pairs.txt",
|
||
])
|
||
mapping_file = None if ignore_existing_ortholog else next(
|
||
(path for path in mapping_candidates if path.exists() and path.stat().st_size > 0),
|
||
None,
|
||
)
|
||
mapping_reference_species = infer_reference_species_from_ortholog_mapping(mapping_file, "") if mapping_file else ""
|
||
normalized_mapping = mapping_reference_species.lower().replace("_", " ")
|
||
allowed_mapping_references = {
|
||
chosen_taxon_id.lower(),
|
||
storage_key.lower().replace("_", " "),
|
||
selected_name.lower().replace("_", " "),
|
||
}
|
||
mapping_matches = not mapping_reference_species or normalized_mapping in allowed_mapping_references
|
||
mapping_identity_matches = False
|
||
mapping_identity_reason = ""
|
||
if mapping_file and mapping_matches and reference_pep_path:
|
||
mapping_identity_matches, mapping_identity_reason = _ppi_mapping_identity_matches_current_inputs(
|
||
mapping_file=mapping_file,
|
||
final_dir=final_dir,
|
||
in_dir=in_dir,
|
||
ws=ws,
|
||
prefix=prefix,
|
||
family=family,
|
||
selected_targets=selected_targets,
|
||
scope_data=scope_data,
|
||
reference_taxon_id=chosen_taxon_id,
|
||
reference_storage_key=storage_key,
|
||
reference_pep=reference_pep_path,
|
||
)
|
||
species_resolution = {
|
||
**species_resolution,
|
||
"selected_string_taxon_id": chosen_taxon_id,
|
||
"selected_string_name": selected_name,
|
||
"requires_user_confirmation": False,
|
||
"reference_selection_mode": str(ppi_reference_selection_mode or selection_mode),
|
||
"reference_user_confirmed": True,
|
||
"reference_storage_key": storage_key,
|
||
"reference_pep_path": reference_pep,
|
||
"ortholog_mapping_path": str(mapping_file) if mapping_file else "",
|
||
}
|
||
common = {
|
||
"target_species": selected_targets,
|
||
"target_identity": target_identity,
|
||
"species_resolution": species_resolution,
|
||
"requested_ppi_species": requested_ppi_species,
|
||
"selected_taxon_id": chosen_taxon_id,
|
||
"selected_string_species": species_to_string_code(selected_name, ""),
|
||
"reference_storage_key": storage_key,
|
||
"reference_pep": reference_pep,
|
||
"mapping_file": mapping_file,
|
||
"mapping_reference_species": mapping_reference_species,
|
||
"dependency_owner": dependency_owner,
|
||
}
|
||
if not reference_pep:
|
||
return _ppi_only_preflight_response(
|
||
status="needs_reference_data",
|
||
issue_code="ppi_reference_data_missing",
|
||
message="The confirmed STRING reference requires a protein FASTA (PEP) before Ortholog.",
|
||
decision="needs_reference_bundle",
|
||
required_next_stage="",
|
||
missing_required=["pep"],
|
||
**common,
|
||
)
|
||
if mapping_file and not mapping_matches:
|
||
return _ppi_only_preflight_response(
|
||
status="needs_ortholog",
|
||
issue_code="stale_ortholog_reference_conflict",
|
||
message="Existing Ortholog mapping does not match the confirmed STRING reference TaxID.",
|
||
decision="needs_reference_ortholog",
|
||
required_next_stage="ortholog",
|
||
**common,
|
||
)
|
||
if mapping_file and not mapping_identity_matches:
|
||
return _ppi_only_preflight_response(
|
||
status="needs_ortholog",
|
||
issue_code="stale_ortholog_identity_conflict",
|
||
message=f"Existing Ortholog mapping identity is stale or unverifiable: {mapping_identity_reason}.",
|
||
decision="needs_reference_ortholog",
|
||
required_next_stage="ortholog",
|
||
**common,
|
||
)
|
||
if not mapping_file:
|
||
return _ppi_only_preflight_response(
|
||
status="needs_ortholog",
|
||
issue_code="ppi_requires_ortholog_mapping",
|
||
message="Reference PEP is available; run Ortholog for the confirmed STRING reference.",
|
||
decision="needs_reference_ortholog",
|
||
required_next_stage="ortholog",
|
||
**common,
|
||
)
|
||
return _ppi_only_preflight_response(
|
||
status="ready",
|
||
issue_code="",
|
||
message="Confirmed STRING reference PEP and matching Ortholog mapping are available.",
|
||
decision="ready_reference_mapping",
|
||
required_next_stage="",
|
||
**common,
|
||
)
|
||
|
||
|
||
def resolve_ppi_species_resolution(
|
||
*,
|
||
job: "JobInfo",
|
||
prefix: str,
|
||
ppi_species: str = "auto",
|
||
scope_file: str = "",
|
||
target_species: Optional[list[str]] = None,
|
||
reference_species: Optional[list[str]] = None,
|
||
family: str = "",
|
||
ignore_existing_ortholog: bool = False,
|
||
requested_stages: Optional[list[str]] = None,
|
||
target_scientific_name: str = "",
|
||
ppi_target_tree_taxon_id: str = "",
|
||
ppi_reference_taxon_id: str = "",
|
||
ppi_reference_storage_key: str = "",
|
||
ppi_reference_selection_mode: str = "",
|
||
ppi_reference_user_confirmed: bool = False,
|
||
dependency_owner: str = "",
|
||
) -> Dict[str, Any]:
|
||
"""Read-only PPI/KaKs dependency preflight for UI and API callers."""
|
||
ws = Path(job.workspace_dir)
|
||
in_dir = Path(job.input_dir)
|
||
final_dir = ws / "final"
|
||
requested = str(ppi_species or "auto").strip().lower()
|
||
explicit_ppi_species = bool(requested and requested != "auto")
|
||
dependency_stages = _normalized_dependency_stages(requested_stages)
|
||
needs_ppi = "ppi" in dependency_stages
|
||
needs_kaks = "kaks" in dependency_stages
|
||
required_bundle_files = ["pep"]
|
||
if needs_kaks and "cds" not in required_bundle_files:
|
||
required_bundle_files.append("cds")
|
||
recommended_bundle_files = ["gff3", "genome"] if needs_kaks else []
|
||
selected_targets = canonicalize_species_codes_for_job(in_dir, [str(x).strip() for x in (target_species or []) if str(x).strip()])
|
||
references = canonicalize_species_codes_for_job(in_dir, [str(x).strip() for x in (reference_species or []) if str(x).strip()])
|
||
scope_data: dict[str, Any] = {}
|
||
scope_id = ""
|
||
|
||
if scope_file:
|
||
scope_path = resolve_scope_path(scope_file, in_dir, ws)
|
||
if scope_path.exists():
|
||
scope_data = read_scope(scope_path)
|
||
selected_targets = canonicalize_species_codes_for_job(
|
||
in_dir,
|
||
[str(x).strip() for x in scope_data.get("target_species", []) if str(x).strip()],
|
||
) or selected_targets
|
||
references = canonicalize_species_codes_for_job(
|
||
in_dir,
|
||
[str(x).strip() for x in scope_data.get("reference_species", []) if str(x).strip()],
|
||
) or references
|
||
scope_id = str(scope_data.get("scope_id") or "")
|
||
|
||
if not selected_targets:
|
||
prefix_species = prefix.split(".", 1)[0] if "." in prefix else ""
|
||
if prefix_species:
|
||
selected_targets = [prefix_species]
|
||
family = str(family or scope_data.get("family") or (prefix.split(".", 1)[1] if "." in prefix else prefix)).strip()
|
||
|
||
selected_target_identity = selected_targets[0] if len(selected_targets) == 1 else ""
|
||
inferred_scientific_name = (
|
||
selected_target_identity
|
||
if " " in selected_target_identity.strip() and not target_scientific_name
|
||
else target_scientific_name
|
||
)
|
||
target_identity = resolve_ppi_target_identity(
|
||
in_dir,
|
||
selected_target_identity,
|
||
explicit_scientific_name=inferred_scientific_name,
|
||
explicit_tree_taxon_id=ppi_target_tree_taxon_id,
|
||
)
|
||
|
||
if needs_ppi and not needs_kaks:
|
||
result = _resolve_ppi_only_snapshot_preflight(
|
||
job=job,
|
||
prefix=prefix,
|
||
family=family,
|
||
selected_targets=selected_targets,
|
||
scope_data=scope_data,
|
||
scope_id=scope_id,
|
||
target_identity=target_identity,
|
||
requested_ppi_species=requested or "auto",
|
||
ignore_existing_ortholog=ignore_existing_ortholog,
|
||
ppi_reference_taxon_id=ppi_reference_taxon_id,
|
||
ppi_reference_storage_key=ppi_reference_storage_key,
|
||
ppi_reference_selection_mode=ppi_reference_selection_mode,
|
||
ppi_reference_user_confirmed=ppi_reference_user_confirmed,
|
||
dependency_owner=dependency_owner or "ppi",
|
||
)
|
||
decision_path = record_ppi_preflight_decision(ws, prefix, result)
|
||
result["preflight_decision_record"] = str(decision_path)
|
||
return result
|
||
|
||
target_string_code = species_to_string_code(selected_targets[0], "") if len(selected_targets) == 1 else ""
|
||
reference_recommendations = recommended_string_reference_species(selected_targets[0]) if len(selected_targets) == 1 else []
|
||
ppi_reference_species = references[0] if references else ""
|
||
if not target_string_code:
|
||
recommended_codes = [
|
||
str(item.get("species_code") or "").strip()
|
||
for item in reference_recommendations
|
||
if str(item.get("species_code") or "").strip()
|
||
]
|
||
downloaded_reference = _downloaded_reference_bundle_species(in_dir, recommended_codes)
|
||
recent_reference = _recent_ppi_kaks_reference_species(final_dir, prefix, recommended_codes)
|
||
downloaded_reference_code = species_to_string_code(downloaded_reference, "")
|
||
recovered_reference = (
|
||
downloaded_reference
|
||
if downloaded_reference_code and downloaded_reference_code != "ath"
|
||
else recent_reference or downloaded_reference
|
||
)
|
||
current_reference_code = species_to_string_code(ppi_reference_species, "")
|
||
recovered_reference_code = species_to_string_code(recovered_reference, "")
|
||
preserve_explicit_arabidopsis_reference = bool(
|
||
explicit_ppi_species
|
||
and species_to_string_code(requested, requested) == "ath"
|
||
and current_reference_code == "ath"
|
||
)
|
||
if (
|
||
recovered_reference_code
|
||
and recovered_reference_code != "ath"
|
||
and not preserve_explicit_arabidopsis_reference
|
||
and (
|
||
not ppi_reference_species
|
||
or current_reference_code == "ath"
|
||
or current_reference_code != recovered_reference_code
|
||
)
|
||
):
|
||
ppi_reference_species = recovered_reference
|
||
selected_string_species = ""
|
||
if explicit_ppi_species:
|
||
selected_string_species = species_to_string_code(requested, requested)
|
||
elif target_string_code:
|
||
selected_string_species = target_string_code
|
||
elif ppi_reference_species:
|
||
selected_string_species = species_to_string_code(ppi_reference_species, "")
|
||
|
||
use_target_string_directly = bool(
|
||
len(selected_targets) == 1
|
||
and target_string_code
|
||
and selected_string_species == target_string_code
|
||
)
|
||
mapping_candidates: list[Path] = []
|
||
if not use_target_string_directly or needs_kaks:
|
||
if scope_id:
|
||
mapping_candidates.extend([
|
||
final_dir / f"{prefix}.{sanitize_token(scope_id)}.orthologs.txt",
|
||
final_dir / f"{prefix}.{sanitize_token(scope_id)}.ortholog_pairs.txt",
|
||
])
|
||
mapping_candidates.extend([
|
||
final_dir / f"{prefix}.orthologs.txt",
|
||
final_dir / f"{prefix}.ortholog_pairs.txt",
|
||
])
|
||
mapping_file = None if ignore_existing_ortholog else next((p for p in mapping_candidates if p.exists() and p.stat().st_size > 0), None)
|
||
|
||
reference_data_source = ""
|
||
reference_bundle_files: dict[str, str] = {}
|
||
reference_bundle_missing: list[str] = []
|
||
reference_bundle_recommended_missing: list[str] = []
|
||
if ppi_reference_species:
|
||
ref_pep = resolve_species_sequence_file(
|
||
ws,
|
||
final_dir,
|
||
in_dir,
|
||
scope_data,
|
||
ppi_reference_species,
|
||
family,
|
||
"pep",
|
||
family_only=False,
|
||
)
|
||
reference_data_source = str(ref_pep) if ref_pep else ""
|
||
if ref_pep:
|
||
reference_bundle_files["pep"] = str(ref_pep)
|
||
if needs_kaks:
|
||
ref_cds = resolve_species_sequence_file(
|
||
ws,
|
||
final_dir,
|
||
in_dir,
|
||
scope_data,
|
||
ppi_reference_species,
|
||
family,
|
||
"cds",
|
||
family_only=False,
|
||
)
|
||
if ref_cds:
|
||
reference_bundle_files["cds"] = str(ref_cds)
|
||
for optional_type in recommended_bundle_files:
|
||
optional_path = resolve_reference_annotation_file(
|
||
in_dir=in_dir,
|
||
species=ppi_reference_species,
|
||
sequence_type=optional_type,
|
||
)
|
||
if optional_path:
|
||
reference_bundle_files[optional_type] = str(optional_path)
|
||
else:
|
||
reference_bundle_recommended_missing.append(optional_type)
|
||
reference_bundle_missing = [item for item in required_bundle_files if item not in reference_bundle_files]
|
||
reference_bundle_status = {
|
||
"required_files": required_bundle_files,
|
||
"recommended_files": recommended_bundle_files,
|
||
"files": reference_bundle_files,
|
||
"missing_required": reference_bundle_missing,
|
||
"missing_recommended": reference_bundle_recommended_missing,
|
||
"mode": "complete" if not reference_bundle_missing else ("pep_only" if reference_bundle_files.get("pep") and "cds" in reference_bundle_missing else "incomplete"),
|
||
"kaks_ready": (not needs_kaks) or ("pep" in reference_bundle_files and "cds" in reference_bundle_files),
|
||
}
|
||
|
||
mapping_reference_species = infer_reference_species_from_ortholog_mapping(mapping_file, "") if mapping_file else ""
|
||
mapping_reference_conflict = bool(
|
||
mapping_file
|
||
and ppi_reference_species
|
||
and mapping_reference_species
|
||
and not _reference_species_matches(ppi_reference_species, mapping_reference_species)
|
||
)
|
||
|
||
if use_target_string_directly and not needs_kaks:
|
||
status = "ready"
|
||
issue_code = ""
|
||
message = "Target species is directly supported by STRING; PPI can query target STRING IDs."
|
||
required_next_stage = ""
|
||
decision = "ready_target_string"
|
||
elif not ppi_reference_species:
|
||
status = "needs_reference_data"
|
||
issue_code = "kaks_reference_data_missing" if needs_kaks else "ppi_reference_data_missing"
|
||
message = (
|
||
"PPI/KaKs dependency preflight requires a reference species before Ortholog can generate target-to-reference mappings."
|
||
if (needs_ppi and needs_kaks)
|
||
else "KaKs requires a reference species with CDS and PEP before Ortholog and Ka/Ks execution."
|
||
if needs_kaks
|
||
else "Target species is not directly supported by STRING; provide/select a reference protein FASTA before Ortholog."
|
||
)
|
||
required_next_stage = "ortholog"
|
||
decision = "needs_reference_species"
|
||
elif mapping_reference_conflict and not (needs_kaks and reference_bundle_missing):
|
||
status = "needs_ortholog"
|
||
issue_code = "stale_ortholog_reference_conflict"
|
||
message = (
|
||
f"Existing Ortholog mapping targets {mapping_reference_species}, but current PPI/KaKs dependency policy "
|
||
f"selects {ppi_reference_species}; rerun Ortholog with the selected reference species."
|
||
)
|
||
required_next_stage = "ortholog"
|
||
decision = "needs_reference_ortholog"
|
||
elif mapping_file:
|
||
inferred = infer_string_species_from_ortholog_mapping(mapping_file, "")
|
||
if needs_kaks and reference_bundle_missing:
|
||
status = "needs_reference_data"
|
||
issue_code = "kaks_reference_cds_missing" if needs_kaks and "cds" in reference_bundle_missing else "reference_bundle_incomplete"
|
||
message = (
|
||
"Ortholog mapping exists, but the selected reference annotation bundle is incomplete for requested PPI/KaKs stages: "
|
||
+ ", ".join(reference_bundle_missing)
|
||
)
|
||
required_next_stage = "ortholog"
|
||
decision = "needs_reference_bundle"
|
||
elif needs_ppi and not (selected_string_species or inferred):
|
||
status = "invalid_mapping"
|
||
issue_code = "ppi_invalid_ortholog_mapping"
|
||
message = "Ortholog mapping exists, but its reference species cannot be resolved to a supported STRING species."
|
||
required_next_stage = "ortholog"
|
||
decision = "failed_invalid_mapping"
|
||
else:
|
||
if inferred:
|
||
selected_string_species = selected_string_species or inferred
|
||
status = "ready"
|
||
issue_code = ""
|
||
message = (
|
||
"Ortholog mapping and reference annotation bundle are available for requested PPI/KaKs stages."
|
||
if needs_kaks
|
||
else "Ortholog mapping is available; PPI can query the mapped reference STRING species."
|
||
)
|
||
required_next_stage = ""
|
||
decision = "ready_reference_mapping"
|
||
elif reference_bundle_missing:
|
||
status = "needs_reference_data"
|
||
issue_code = "kaks_reference_cds_missing" if needs_kaks and "cds" in reference_bundle_missing else "ppi_reference_data_missing"
|
||
message = (
|
||
"Reference annotation bundle is incomplete for requested PPI/KaKs stages; missing: "
|
||
+ ", ".join(reference_bundle_missing)
|
||
+ ". Upload or download a complete reference annotation bundle before Ortholog."
|
||
)
|
||
required_next_stage = "ortholog"
|
||
decision = "needs_reference_bundle"
|
||
elif selected_string_species and (explicit_ppi_species or ppi_reference_species):
|
||
status = "needs_ortholog"
|
||
issue_code = "ppi_requires_ortholog_mapping"
|
||
message = (
|
||
"PPI/KaKs dependency data are available; run Ortholog first to generate target-to-reference mappings."
|
||
if needs_kaks
|
||
else "Reference STRING species selected; run Ortholog before PPI so target IDs are mapped to reference IDs."
|
||
)
|
||
required_next_stage = "ortholog"
|
||
decision = "needs_reference_ortholog"
|
||
elif not needs_ppi and needs_kaks:
|
||
status = "needs_ortholog"
|
||
issue_code = "ppi_requires_ortholog_mapping"
|
||
message = "KaKs reference CDS/PEP are available; run Ortholog first to generate target-to-reference pairs."
|
||
required_next_stage = "ortholog"
|
||
decision = "needs_reference_ortholog"
|
||
elif reference_data_source:
|
||
status = "needs_ortholog"
|
||
issue_code = "ppi_requires_ortholog_mapping"
|
||
message = "Reference protein data exists; run Ortholog before PPI fallback."
|
||
required_next_stage = "ortholog"
|
||
decision = "needs_reference_ortholog"
|
||
else:
|
||
status = "needs_reference_data"
|
||
issue_code = "ppi_reference_data_missing"
|
||
message = "Target species is not directly supported by STRING; provide/select a reference protein FASTA before Ortholog."
|
||
required_next_stage = "ortholog"
|
||
decision = "needs_reference_data"
|
||
|
||
selected_taxon_id = string_taxon_id(selected_string_species, "") if selected_string_species else ""
|
||
diagnostic = build_ppi_species_diagnostic(
|
||
requested_ppi_species=requested or "auto",
|
||
target_species=",".join(selected_targets),
|
||
target_string_supported=bool(target_string_code),
|
||
reference_species=ppi_reference_species,
|
||
reference_data_source=reference_data_source,
|
||
ortholog_mapping_used=bool(mapping_file),
|
||
fallback_level=status,
|
||
fallback_reason=issue_code or "",
|
||
selected_string_species=selected_string_species,
|
||
selected_taxon_id=selected_taxon_id,
|
||
prefix=prefix,
|
||
)
|
||
diagnostics = diagnostic.get("diagnostics", {})
|
||
if issue_code:
|
||
diagnostics["issue_code"] = issue_code
|
||
diagnostics["required_next_stage"] = required_next_stage
|
||
diagnostics["reference_recommendations"] = reference_recommendations
|
||
diagnostics["dependency_owner_stages"] = dependency_stages
|
||
diagnostics["required_bundle_files"] = required_bundle_files
|
||
diagnostics["reference_bundle_status"] = reference_bundle_status
|
||
diagnostics["ortholog_dependency_role"] = "producer"
|
||
if status == "ready":
|
||
diagnostics = {
|
||
"issue_code": "",
|
||
"severity": "info",
|
||
"layer": "input_contract",
|
||
"required_next_stage": "",
|
||
"recommendation": message,
|
||
"dependency_owner_stages": dependency_stages,
|
||
"required_bundle_files": required_bundle_files,
|
||
"reference_bundle_status": reference_bundle_status,
|
||
"ortholog_dependency_role": "producer",
|
||
"recommended_actions": [
|
||
{
|
||
"id": "run_ppi",
|
||
"label": "Run PPI with the resolved STRING species and mapping",
|
||
"requires_confirmation": False,
|
||
"risk": "uses_existing_ortholog_mapping" if mapping_file else "direct_target_string",
|
||
}
|
||
],
|
||
}
|
||
|
||
return {
|
||
"status": status,
|
||
"issue_code": issue_code,
|
||
"message": message,
|
||
"target_species": selected_targets,
|
||
"target_identity": target_identity,
|
||
"target_string_supported": bool(target_string_code),
|
||
"requested_stages": dependency_stages,
|
||
"requested_ppi_species": requested or "auto",
|
||
"selected_string_species": selected_string_species,
|
||
"selected_taxon_id": selected_taxon_id,
|
||
"reference_species": ppi_reference_species,
|
||
"reference_data_source": reference_data_source,
|
||
"reference_bundle_status": reference_bundle_status,
|
||
"mapping_reference_species": mapping_reference_species,
|
||
"ortholog_mapping_source": str(mapping_file) if mapping_file else "",
|
||
"required_next_stage": required_next_stage,
|
||
"decision": decision,
|
||
"diagnostics": diagnostics,
|
||
}
|
||
|
||
|
||
def write_ppi_failed_manifest(
|
||
final_dir: Path,
|
||
prefix: str,
|
||
diagnostic: Dict[str, Any],
|
||
) -> Path:
|
||
final_dir.mkdir(parents=True, exist_ok=True)
|
||
manifest_path = final_dir / f"{prefix}.ppi.manifest.json"
|
||
data = dict(diagnostic)
|
||
data["prefix"] = prefix
|
||
data["timestamp"] = datetime.now().isoformat()
|
||
manifest_path.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
return manifest_path
|
||
|
||
|
||
def resolve_species_sequence_file(
|
||
ws: Path,
|
||
final_dir: Path,
|
||
in_dir: Path,
|
||
scope_data: Optional[dict],
|
||
species: str,
|
||
family: str,
|
||
sequence_type: str,
|
||
family_only: bool = False,
|
||
) -> Optional[Path]:
|
||
# Prefer genome_prepare/standardized representative files over raw scope
|
||
# uploads. Raw NCBI CDS/PEP often use XP/NP headers while prepared family
|
||
# members and ortholog pairs use transcript-like primary IDs.
|
||
standardized_candidates: list[Path] = []
|
||
if sequence_type == "pep" and family_only:
|
||
family_pep = resolve_species_family_pep(ws, final_dir, in_dir, species, family)
|
||
if family_pep:
|
||
return family_pep
|
||
if sequence_type == "cds":
|
||
standardized_candidates.extend([
|
||
in_dir / f"{species}.cds.fasta",
|
||
in_dir / f"{species}.cds.fa",
|
||
in_dir / species / f"{species}.cds.fasta",
|
||
in_dir / species / f"{species}.cds.fa",
|
||
in_dir / "genome_prepare" / f"{species}_extracted.cds.fa",
|
||
in_dir / "genome_prepare" / f"{species}.cds.fasta",
|
||
in_dir / "genome_prepare" / f"{species}.cds.fa",
|
||
])
|
||
elif sequence_type == "pep":
|
||
standardized_candidates.extend([
|
||
in_dir / f"{species}.pep.fasta",
|
||
in_dir / f"{species}.pep.fa",
|
||
in_dir / species / f"{species}.pep.fasta",
|
||
in_dir / species / f"{species}.pep.fa",
|
||
in_dir / "genome_prepare" / f"{species}_extracted.pep.fa",
|
||
in_dir / "genome_prepare" / f"{species}.pep.fasta",
|
||
in_dir / "genome_prepare" / f"{species}.pep.fa",
|
||
])
|
||
for candidate in standardized_candidates:
|
||
if candidate.exists() and candidate.stat().st_size > 0:
|
||
return candidate
|
||
|
||
if scope_data:
|
||
file_types = ["family_pep", "pep"] if sequence_type == "pep" and family_only else [sequence_type]
|
||
scoped = first_existing_scope_file(scope_data, species, file_types)
|
||
if scoped:
|
||
return scoped
|
||
|
||
candidates = []
|
||
if sequence_type == "cds":
|
||
candidates.extend([
|
||
in_dir / species / f"{species}.cds.fasta.gz",
|
||
])
|
||
if species.lower() in {"at", "ath"}:
|
||
candidates.extend([
|
||
in_dir / "At.cds.fasta",
|
||
in_dir / "Ath.cds.fasta",
|
||
Path("/home/xzq/wutx/data/raw_genome/ath/Arabidopsis_thaliana.TAIR10.cds.all.fa"),
|
||
])
|
||
elif sequence_type == "pep":
|
||
if family_only:
|
||
candidates.extend([
|
||
in_dir / f"{species}.{family}.pep.fa",
|
||
in_dir / f"{species}.{family}.pep.fasta",
|
||
final_dir / f"{species}.{family}.pep.fa",
|
||
final_dir / f"{species}.{family}.pep.fasta",
|
||
])
|
||
candidates.extend([
|
||
in_dir / species / f"{species}.pep.fasta.gz",
|
||
])
|
||
if species.lower() in {"at", "ath"}:
|
||
candidates.extend([
|
||
in_dir / "At.pep.fasta",
|
||
in_dir / "Ath.pep.fasta",
|
||
Path("/home/xzq/wutx/data/raw_genome/ath/Arabidopsis_thaliana.TAIR10.pep.all.fa"),
|
||
])
|
||
candidates.extend(reference_protein_candidate_paths(species))
|
||
for candidate in candidates:
|
||
if candidate.exists() and candidate.stat().st_size > 0:
|
||
return candidate
|
||
return None
|
||
|
||
def write_species_filtered_pairs(
|
||
mapping_file: Path,
|
||
output_file: Path,
|
||
source_species: str,
|
||
target_species: str,
|
||
) -> int:
|
||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||
count = 0
|
||
with mapping_file.open("r", encoding="utf-8", errors="ignore") as src, output_file.open("w", encoding="utf-8") as dst:
|
||
dst.write("# Source_ID\tTarget_ID\tIdentity(%)\tSource_Species\tTarget_Species\n")
|
||
for raw in src:
|
||
line = raw.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
parts = line.split("\t")
|
||
if len(parts) < 2:
|
||
continue
|
||
src_species = parts[3] if len(parts) >= 4 else ""
|
||
tgt_species = parts[4] if len(parts) >= 5 else ""
|
||
if src_species and src_species != source_species:
|
||
continue
|
||
if tgt_species and tgt_species != target_species:
|
||
continue
|
||
dst.write("\t".join(parts[:5]) + "\n")
|
||
count += 1
|
||
return count
|
||
|
||
|
||
def annotate_stage_manifest_with_scope(
|
||
manifest_path: Path,
|
||
prefix: str,
|
||
stage_name: str,
|
||
scope_path: Path,
|
||
scope_data: dict,
|
||
extra_parameters: Optional[dict] = None,
|
||
) -> None:
|
||
if not manifest_path.exists():
|
||
return
|
||
try:
|
||
manifest_data = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return
|
||
scope_fields = {
|
||
"scope_file": str(scope_path),
|
||
"scope_id": scope_data.get("scope_id", ""),
|
||
"scope_mode": scope_data.get("scope_mode", ""),
|
||
"target_species": scope_data.get("target_species", []),
|
||
"reference_species": scope_data.get("reference_species", []),
|
||
"comparative_species": scope_data.get("comparative_species", []),
|
||
"selected_species": scope_data.get("selected_species", []),
|
||
"family": scope_data.get("family", ""),
|
||
}
|
||
manifest_data.update(scope_fields)
|
||
params = manifest_data.setdefault("parameters", {})
|
||
if isinstance(params, dict):
|
||
params.update(scope_fields)
|
||
if extra_parameters:
|
||
params.update(extra_parameters)
|
||
if extra_parameters:
|
||
manifest_data.update({k: v for k, v in extra_parameters.items() if k not in manifest_data})
|
||
route = str((extra_parameters or {}).get("analysis_route") or manifest_data.get("analysis_route") or "").strip()
|
||
member = str((extra_parameters or {}).get("member_set_id") or manifest_data.get("member_set_id") or "").strip()
|
||
try:
|
||
stage_context = build_stage_context(
|
||
stage=stage_name,
|
||
prefix=prefix,
|
||
scope_data=scope_data,
|
||
scope_file=scope_path,
|
||
analysis_route=route,
|
||
member_set_id=member,
|
||
)
|
||
manifest_data = apply_stage_context(manifest_data, stage_context)
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to apply stage context to {stage_name} manifest: {exc}")
|
||
manifest_path.write_text(json.dumps(manifest_data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
if scope_fields["scope_id"]:
|
||
scoped_manifest = manifest_path.parent / f"{prefix}.{sanitize_token(str(scope_fields['scope_id']))}.{stage_name}.manifest.json"
|
||
manifest_data["scope_manifest"] = str(scoped_manifest.absolute())
|
||
scoped_manifest.write_text(json.dumps(manifest_data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
|
||
|
||
def _read_prefix_from_manifest(path: Path) -> str:
|
||
try:
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return ""
|
||
return _clean_prefix(data.get("prefix"))
|
||
|
||
|
||
def _gene_family_from_config_xlsx(config_path: Path) -> str:
|
||
if not config_path.exists():
|
||
return ""
|
||
try:
|
||
import pandas as pd
|
||
sheets = pd.read_excel(config_path, sheet_name=None)
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to read Gene_Family from {config_path}: {exc}")
|
||
return ""
|
||
for sheet_name, df in sheets.items():
|
||
if str(sheet_name).lower() in {"analysis", "meta", "metadata"}:
|
||
for _, row in df.fillna("").iterrows():
|
||
cells = [str(v).strip() for v in row.tolist()]
|
||
for idx, cell in enumerate(cells[:-1]):
|
||
if cell.lower() in {"family", "gene_family", "analysis_family", "project_family"}:
|
||
gene_family = _clean_prefix(cells[idx + 1])
|
||
if gene_family:
|
||
return gene_family
|
||
if "Gene_Family" in df.columns:
|
||
for value in df["Gene_Family"].tolist():
|
||
gene_family = _clean_prefix(value)
|
||
if gene_family:
|
||
return gene_family
|
||
return ""
|
||
|
||
|
||
def _species_prefix_from_job_resources(in_dir: Path) -> str:
|
||
context = resolve_rnaseq_project_context(input_dir=in_dir)
|
||
return context.get("output_prefix", "")
|
||
|
||
|
||
def _is_gene_family_prefix(value: str) -> bool:
|
||
value = str(value or "").strip()
|
||
return "." in value or value.lower() in {"target", "multi"}
|
||
|
||
|
||
RNA_SEQ_STAGES = {
|
||
"rnaseq_qc",
|
||
"rnaseq_align",
|
||
"rnaseq_quant",
|
||
"rnaseq_sample_qc",
|
||
"rnaseq_de",
|
||
"annotation",
|
||
"rnaseq_enrich",
|
||
"rnaseq_go_enrich",
|
||
"rnaseq_kegg_enrich",
|
||
"rnaseq_fgsea",
|
||
"rnaseq_wgcna",
|
||
"rnaseq_candidate_family",
|
||
"rnaseq_family_score",
|
||
}
|
||
|
||
|
||
GENE_FAMILY_SCOPE_STAGES = {
|
||
"identify",
|
||
"phylogeny",
|
||
"reference_classification",
|
||
"structure",
|
||
"motif",
|
||
"domain",
|
||
"combo",
|
||
}
|
||
|
||
|
||
def resolve_effective_prefix_for_job(job, prefix: str, stage: str) -> str:
|
||
prefix = _clean_prefix(prefix)
|
||
in_dir = Path(job.input_dir)
|
||
ws = Path(job.workspace_dir)
|
||
|
||
def normalize_gene_family_prefix(value: str) -> str:
|
||
value = _clean_prefix(value)
|
||
if stage in GENE_FAMILY_SCOPE_STAGES and value.startswith("Multi."):
|
||
family = _clean_prefix(value.split(".", 1)[1])
|
||
if family:
|
||
return f"Target.{family}"
|
||
return value
|
||
|
||
if stage in RNA_SEQ_STAGES:
|
||
rnaseq_context = resolve_rnaseq_project_context(
|
||
input_dir=in_dir,
|
||
final_dir=ws / "final",
|
||
requested_prefix=prefix,
|
||
)
|
||
resolved = rnaseq_context.get("output_prefix", "")
|
||
if resolved:
|
||
if resolved != prefix:
|
||
print(
|
||
f"[INFO] Using RNA-seq output prefix {resolved} for {stage} "
|
||
f"(requested {prefix or '<empty>'}, source={rnaseq_context.get('source', '')})"
|
||
)
|
||
return resolved
|
||
if prefix:
|
||
return normalize_gene_family_prefix(prefix)
|
||
if stage in GENE_FAMILY_SCOPE_STAGES:
|
||
gene_family = _gene_family_from_config_xlsx(in_dir / "config.xlsx")
|
||
if gene_family:
|
||
return f"Target.{gene_family}"
|
||
species_config = in_dir / "genome_prepare" / "species_config.json"
|
||
try:
|
||
if species_config.exists():
|
||
data = json.loads(species_config.read_text(encoding="utf-8"))
|
||
target = data.get("target") if isinstance(data, dict) else None
|
||
if isinstance(target, dict):
|
||
resolved = _clean_prefix(target.get("prefix"))
|
||
if resolved and resolved.lower() != "multi_species":
|
||
return normalize_gene_family_prefix(resolved)
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to read prefix from {species_config}: {exc}")
|
||
for root in [in_dir / "final", ws / "final"]:
|
||
if not root.exists():
|
||
continue
|
||
for path in sorted(root.glob("*.genome_prepare.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True):
|
||
resolved = _read_prefix_from_manifest(path)
|
||
if resolved:
|
||
return normalize_gene_family_prefix(resolved)
|
||
for path in sorted(root.glob("Target.*.identify.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True):
|
||
resolved = _read_prefix_from_manifest(path) or path.name[:-len(".identify.manifest.json")]
|
||
if resolved:
|
||
return normalize_gene_family_prefix(resolved)
|
||
for path in sorted(root.glob("Multi.*.identify.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True):
|
||
resolved = _read_prefix_from_manifest(path) or path.name[:-len(".identify.manifest.json")]
|
||
if resolved:
|
||
return normalize_gene_family_prefix(resolved)
|
||
for path in sorted(root.glob("Target.*.final_members.pep.fa")):
|
||
resolved = path.name[:-len(".final_members.pep.fa")]
|
||
if resolved:
|
||
return normalize_gene_family_prefix(resolved)
|
||
for path in sorted(root.glob("Multi.*.final_members.pep.fa")):
|
||
resolved = path.name[:-len(".final_members.pep.fa")]
|
||
if resolved:
|
||
return normalize_gene_family_prefix(resolved)
|
||
gene_family = _gene_family_from_config_xlsx(in_dir / "config.xlsx")
|
||
if gene_family:
|
||
return f"Target.{gene_family}"
|
||
for root in [in_dir, in_dir / "final", ws / "final"]:
|
||
if not root.exists():
|
||
continue
|
||
for path in sorted(root.glob("Ref_combined.*.pep.fasta")):
|
||
gene_family = path.name[len("Ref_combined."):-len(".pep.fasta")]
|
||
gene_family = _clean_prefix(gene_family)
|
||
if gene_family:
|
||
return f"Target.{gene_family}"
|
||
return ""
|
||
|
||
|
||
def _has_non_comment_data(path: Path) -> bool:
|
||
"""Return True if a text table has at least one non-empty, non-comment row."""
|
||
try:
|
||
if not path or not path.exists() or path.stat().st_size == 0:
|
||
return False
|
||
with open(path, "r", encoding="utf-8", errors="ignore") as handle:
|
||
for raw in handle:
|
||
line = raw.strip()
|
||
if line and not line.startswith("#"):
|
||
return True
|
||
except Exception:
|
||
return False
|
||
return False
|
||
|
||
|
||
def _write_rnaseq_family_score_manifest(
|
||
final_dir: Path,
|
||
prefix: str,
|
||
job_id: str,
|
||
workspace: Path,
|
||
input_dir: Path,
|
||
*,
|
||
status: str,
|
||
reason: str,
|
||
missing: Optional[List[str]] = None,
|
||
log_path: Optional[Path] = None,
|
||
returncode: Optional[int] = None,
|
||
trigger: str = "",
|
||
) -> Path:
|
||
manifest_path = final_dir / f"{prefix}.rnaseq_family_score.manifest.json"
|
||
payload = {
|
||
"manifest_version": "1.0",
|
||
"stage": "rnaseq_family_score",
|
||
"status": status,
|
||
"prefix": prefix,
|
||
"job_id": job_id,
|
||
"timestamp": datetime.now().isoformat(timespec="seconds"),
|
||
"triggered_by": trigger,
|
||
"workspace": str(workspace),
|
||
"input_dir": str(input_dir),
|
||
"reason": reason,
|
||
"missing_inputs": missing or [],
|
||
"returncode": returncode,
|
||
"log": str(log_path) if log_path else "",
|
||
"artifacts": {},
|
||
}
|
||
final_dir.mkdir(parents=True, exist_ok=True)
|
||
manifest_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
return manifest_path
|
||
|
||
|
||
def _resolve_rnaseq_family_score_inputs(ws: Path, in_dir: Path, final_dir: Path, prefix: str) -> tuple[bool, Dict[str, Path], List[str]]:
|
||
de_files = [p for p in final_dir.glob(f"{prefix}.rnaseq_de.*.DE_results.txt") if p.exists() and p.stat().st_size > 0]
|
||
tpm_candidates = [
|
||
final_dir / f"{prefix}.salmon.tpm_matrix.txt",
|
||
final_dir / f"{prefix}.tpm_matrix.txt",
|
||
*sorted(final_dir.glob("*.salmon.tpm_matrix.txt")),
|
||
*sorted(final_dir.glob("*.tpm_matrix.txt")),
|
||
]
|
||
sample_candidates = [
|
||
in_dir / "samples.yaml",
|
||
*sorted(in_dir.glob("*.yaml")),
|
||
final_dir / "samples.yaml",
|
||
]
|
||
annotation_analysis_dir = ws / "annotation" / "analysis"
|
||
candidate_analysis_dir = ws / "rnaseq_candidate_family" / "analysis"
|
||
interpro_candidates = [
|
||
ws / "annotation" / "analysis" / f"{prefix}.interproscan.tsv",
|
||
ws / "rnaseq_candidate_family" / "analysis" / f"{prefix}.rnaseq_candidate_family.interproscan.tsv",
|
||
*sorted(final_dir.glob("*interproscan*.tsv")),
|
||
]
|
||
if annotation_analysis_dir.exists():
|
||
interpro_candidates.extend(sorted(annotation_analysis_dir.glob("*interproscan*.tsv")))
|
||
if candidate_analysis_dir.exists():
|
||
interpro_candidates.extend(sorted(candidate_analysis_dir.glob("*interproscan*.tsv")))
|
||
gene_annotation = final_dir / f"{prefix}.rnaseq_candidate_family.gene_annotations.tsv"
|
||
|
||
inputs: Dict[str, Path] = {}
|
||
missing: List[str] = []
|
||
if de_files:
|
||
inputs["de_results"] = de_files[0]
|
||
else:
|
||
missing.append(f"{prefix}.rnaseq_de.*.DE_results.txt")
|
||
|
||
tpm_matrix = next((p for p in tpm_candidates if p.exists() and p.stat().st_size > 0), None)
|
||
if tpm_matrix:
|
||
inputs["tpm_matrix"] = tpm_matrix
|
||
else:
|
||
missing.append(f"{prefix}.salmon.tpm_matrix.txt or {prefix}.tpm_matrix.txt")
|
||
|
||
sample_config = next((p for p in sample_candidates if p.exists() and p.stat().st_size > 0), None)
|
||
if sample_config:
|
||
inputs["sample_config"] = sample_config
|
||
else:
|
||
missing.append("samples.yaml")
|
||
|
||
interpro_tsv = next(
|
||
(
|
||
p for p in interpro_candidates
|
||
if p.exists()
|
||
and p.stat().st_size > 0
|
||
and "final_members.interproscan" not in p.name.lower()
|
||
and "curated_members.interproscan" not in p.name.lower()
|
||
and "candidate_members.interproscan" not in p.name.lower()
|
||
),
|
||
None,
|
||
)
|
||
if interpro_tsv:
|
||
inputs["interproscan_tsv"] = interpro_tsv
|
||
elif gene_annotation.exists() and gene_annotation.stat().st_size > 0:
|
||
inputs["gene_annotations"] = gene_annotation
|
||
else:
|
||
missing.append(f"{prefix}.interproscan.tsv or {prefix}.rnaseq_candidate_family.gene_annotations.tsv")
|
||
|
||
return not missing, inputs, missing
|
||
|
||
|
||
def _run_rnaseq_family_score_if_ready(
|
||
ws: Path,
|
||
in_dir: Path,
|
||
final_dir: Path,
|
||
prefix: str,
|
||
job_id: str,
|
||
cpu: int,
|
||
base_log_file: Path,
|
||
*,
|
||
trigger: str,
|
||
) -> tuple[str, Path]:
|
||
score_manifest = final_dir / f"{prefix}.rnaseq_family_score.manifest.json"
|
||
score_script = Path("apps/cli/run_rnaseq_family_score.py")
|
||
score_log = base_log_file.with_name(f"{base_log_file.stem}.rnaseq_family_score.log")
|
||
if not score_script.exists():
|
||
manifest_path = _write_rnaseq_family_score_manifest(
|
||
final_dir,
|
||
prefix,
|
||
job_id,
|
||
ws,
|
||
in_dir,
|
||
status="skipped",
|
||
reason="rnaseq_family_score script not found",
|
||
missing=[str(score_script)],
|
||
log_path=score_log,
|
||
trigger=trigger,
|
||
)
|
||
return f" Family score skipped: script not found ({score_script}).", manifest_path
|
||
|
||
ready, inputs, missing = _resolve_rnaseq_family_score_inputs(ws, in_dir, final_dir, prefix)
|
||
if not ready:
|
||
manifest_path = _write_rnaseq_family_score_manifest(
|
||
final_dir,
|
||
prefix,
|
||
job_id,
|
||
ws,
|
||
in_dir,
|
||
status="skipped",
|
||
reason="required inputs are not complete",
|
||
missing=missing,
|
||
log_path=score_log,
|
||
trigger=trigger,
|
||
)
|
||
return f" Family score skipped: missing {', '.join(missing)}.", manifest_path
|
||
|
||
score_cmd = [
|
||
"/home/xzq/miniconda3/envs/test4py311/bin/python",
|
||
str(score_script),
|
||
"--workspace", str(ws),
|
||
"--input-dir", str(in_dir),
|
||
"--final-dir", str(final_dir),
|
||
"-p", prefix,
|
||
"-j", job_id,
|
||
"-t", str(cpu),
|
||
"--sample-config", str(inputs["sample_config"]),
|
||
"--tpm-matrix", str(inputs["tpm_matrix"]),
|
||
]
|
||
if inputs.get("interproscan_tsv"):
|
||
score_cmd.extend(["--interproscan-tsv", str(inputs["interproscan_tsv"])])
|
||
print(f"[RNA-seq Family Score] Auto post-processing ({trigger}): {' '.join(score_cmd)}")
|
||
code = run_cmd_logged(score_cmd, score_log, cwd=Path("."))
|
||
if code == 0 and score_manifest.exists():
|
||
return f" Family score generated: {score_manifest.name}.", score_manifest
|
||
|
||
manifest_path = _write_rnaseq_family_score_manifest(
|
||
final_dir,
|
||
prefix,
|
||
job_id,
|
||
ws,
|
||
in_dir,
|
||
status="failed",
|
||
reason="rnaseq_family_score command failed or manifest missing",
|
||
log_path=score_log,
|
||
returncode=code,
|
||
trigger=trigger,
|
||
)
|
||
return f" Family score failed but RNA-seq results are kept; see log: {score_log}.", manifest_path
|
||
|
||
|
||
RAP_TRANSCRIPT_ID_RE = re.compile(r"^(Os\d{2})t(\d+)(?:-\d+)?$", re.IGNORECASE)
|
||
|
||
|
||
def _normalize_enrichment_gene_id_for_match(value: str) -> str:
|
||
token = (value or "").strip().strip('"').lstrip(">")
|
||
token = re.sub(r"^[a-z][a-z0-9]{2,4}:", "", token)
|
||
token = re.split(r"\s+", token, maxsplit=1)[0]
|
||
previous = None
|
||
suffix_patterns = [
|
||
re.compile(r"(?i)([._-](?:mrna|rna|transcript|cds|protein|pep|peptide)\d*[A-Za-z]*)$"),
|
||
re.compile(r"(?i)([._-][tp]\d+[A-Za-z]?)$"),
|
||
re.compile(r"(?i)([._-][rp][A-Z])$"),
|
||
re.compile(r"(\.\d+)$"),
|
||
]
|
||
while token and token != previous:
|
||
previous = token
|
||
for pattern in suffix_patterns:
|
||
token = pattern.sub("", token)
|
||
match = RAP_TRANSCRIPT_ID_RE.match(token)
|
||
if match:
|
||
token = f"{match.group(1)}g{match.group(2)}"
|
||
return token
|
||
|
||
|
||
def _read_enrichment_first_column(path: Path, *, skip_header: bool = False) -> tuple[set[str], set[str]]:
|
||
raw_ids: set[str] = set()
|
||
normalized_ids: set[str] = set()
|
||
try:
|
||
with path.open("r", encoding="utf-8", errors="ignore") as handle:
|
||
for idx, raw in enumerate(handle):
|
||
line = raw.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
if skip_header and idx == 0:
|
||
continue
|
||
gene_id = re.split(r"[\t, ]+", line, maxsplit=1)[0].strip()
|
||
normalized = _normalize_enrichment_gene_id_for_match(gene_id)
|
||
if normalized and normalized.lower() not in {"gene", "gene_id", "geneid", "id", "x"}:
|
||
raw_ids.add(gene_id)
|
||
normalized_ids.add(normalized)
|
||
except Exception:
|
||
return set(), set()
|
||
return raw_ids, normalized_ids
|
||
|
||
|
||
def _rnaseq_de_background_reference_set(final_dir: Path) -> tuple[set[str], set[str]]:
|
||
raw_ids: set[str] = set()
|
||
normalized_ids: set[str] = set()
|
||
for de_file in sorted(final_dir.glob("*.rnaseq_de.*.DE_genes.txt")):
|
||
raw, normalized = _read_enrichment_first_column(de_file, skip_header=True)
|
||
raw_ids.update(raw)
|
||
normalized_ids.update(normalized)
|
||
return raw_ids, normalized_ids
|
||
|
||
|
||
def _filter_gene2go_files_by_de_overlap(gene2go_files: list[Path], final_dir: Path) -> tuple[list[Path], list[dict[str, Any]]]:
|
||
de_raw, de_norm = _rnaseq_de_background_reference_set(final_dir)
|
||
if not de_raw:
|
||
return gene2go_files, []
|
||
compatible: list[Path] = []
|
||
reports: list[dict[str, Any]] = []
|
||
for gene2go in gene2go_files:
|
||
bg_raw, bg_norm = _read_enrichment_first_column(gene2go)
|
||
raw_overlap = len(de_raw & bg_raw)
|
||
normalized_overlap = len(de_norm & bg_norm)
|
||
report = {
|
||
"path": str(gene2go),
|
||
"de_genes": len(de_raw),
|
||
"background_genes": len(bg_raw),
|
||
"raw_overlap": raw_overlap,
|
||
"normalized_overlap": normalized_overlap,
|
||
"de_examples": sorted(de_raw)[:5],
|
||
"background_examples": sorted(bg_raw)[:5],
|
||
}
|
||
reports.append(report)
|
||
if raw_overlap > 0 or normalized_overlap > 0:
|
||
compatible.append(gene2go)
|
||
return compatible, reports
|
||
|
||
|
||
def _json_file(path: Path) -> dict:
|
||
try:
|
||
if path.exists() and path.stat().st_size > 0:
|
||
with path.open("r", encoding="utf-8") as handle:
|
||
data = json.load(handle)
|
||
if isinstance(data, dict):
|
||
return data
|
||
except Exception:
|
||
return {}
|
||
return {}
|
||
|
||
|
||
def _annotation_manifest_success(path: Path) -> bool:
|
||
return _json_file(path).get("status") == "success"
|
||
|
||
|
||
def _annotation_progress_running(path: Path) -> bool:
|
||
return _json_file(path).get("status") == "running"
|
||
|
||
|
||
def _annotation_process_active(ws: Path, scope: str, prefix: str) -> bool:
|
||
"""Best-effort check for a live local annotation process for this workspace."""
|
||
analysis_dir = ws / "annotation" / "analysis"
|
||
markers = [
|
||
str(ws.resolve()),
|
||
str(analysis_dir.resolve()),
|
||
str((analysis_dir / f"{prefix}.{scope}.annotation_safe.pep.fasta").resolve()),
|
||
]
|
||
if scope == "go":
|
||
markers.extend(["interproscan", f"{prefix}.interproscan"])
|
||
elif scope == "kegg":
|
||
markers.extend(["exec_annotation", "kofamscan", "hmmscan", f"{prefix}.kofamscan"])
|
||
else:
|
||
markers.extend(["run_annotation.py", "interproscan", "exec_annotation"])
|
||
try:
|
||
for proc in Path("/proc").iterdir():
|
||
if not proc.name.isdigit():
|
||
continue
|
||
try:
|
||
cmdline = (proc / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", errors="ignore")
|
||
except Exception:
|
||
continue
|
||
if not cmdline:
|
||
continue
|
||
if "run_annotation.py" in cmdline and any(marker in cmdline for marker in markers[:3]):
|
||
return True
|
||
if scope in {"go", "kegg"} and any(marker in cmdline for marker in markers[3:]) and any(marker in cmdline for marker in markers[:2]):
|
||
return True
|
||
except Exception:
|
||
return False
|
||
return False
|
||
|
||
|
||
def _annotation_progress_active(path: Path, ws: Path, scope: str, prefix: str) -> bool:
|
||
if not _annotation_progress_running(path):
|
||
return False
|
||
if _annotation_process_active(ws, scope, prefix):
|
||
return True
|
||
progress = _json_file(path)
|
||
updated_raw = str(progress.get("updated_at") or "")
|
||
try:
|
||
updated = datetime.fromisoformat(updated_raw)
|
||
age_sec = (datetime.now() - updated).total_seconds()
|
||
except Exception:
|
||
age_sec = 999999
|
||
return age_sec < 60
|
||
|
||
|
||
def _wait_for_annotation_reuse(
|
||
annotation_manifest: Path,
|
||
progress_file: Path,
|
||
log_file: Path,
|
||
timeout_sec: int = 172800,
|
||
scope: str = "annotation",
|
||
ws: Path | None = None,
|
||
prefix: str = "",
|
||
) -> bool:
|
||
"""Wait for an already-running background annotation instead of starting a duplicate."""
|
||
start = time.time()
|
||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||
with log_file.open("a", encoding="utf-8") as lg:
|
||
lg.write(
|
||
f"\nWAIT: detected existing {scope} annotation run for this workspace; "
|
||
"waiting to reuse its background outputs.\n"
|
||
)
|
||
lg.flush()
|
||
while time.time() - start < timeout_sec:
|
||
if _annotation_manifest_success(annotation_manifest):
|
||
lg.write(f"WAIT_DONE: {scope} annotation manifest succeeded at {datetime.now():%Y-%m-%d %H:%M:%S}\n")
|
||
return True
|
||
progress = _json_file(progress_file)
|
||
status = progress.get("status")
|
||
if status == "failed":
|
||
lg.write(f"WAIT_FAILED: {scope} annotation progress failed: {progress}\n")
|
||
return False
|
||
if status == "running" and ws is not None and not _annotation_progress_active(progress_file, ws, scope, prefix):
|
||
lg.write(f"WAIT_STALE: {scope} annotation progress is running but no live process was detected: {progress}\n")
|
||
return False
|
||
if status and status != "running":
|
||
time.sleep(5)
|
||
if _annotation_manifest_success(annotation_manifest):
|
||
lg.write(f"WAIT_DONE: {scope} annotation manifest succeeded at {datetime.now():%Y-%m-%d %H:%M:%S}\n")
|
||
return True
|
||
lg.write(f"WAIT_STOPPED: {scope} annotation no longer running, status={status}, manifest not successful\n")
|
||
return False
|
||
time.sleep(30)
|
||
lg.write(f"WAIT_TIMEOUT: {scope} annotation reuse wait timed out\n")
|
||
return False
|
||
|
||
|
||
def _annotation_state_paths(ws: Path, final_dir: Path, prefix: str, scope: str) -> dict[str, Path]:
|
||
if scope not in {"go", "kegg"}:
|
||
return {
|
||
"manifest": final_dir / f"{prefix}.annotation.manifest.json",
|
||
"progress": ws / "annotation" / "analysis" / "progress.json",
|
||
"lock": ws / "annotation" / "analysis" / ".annotation.lock",
|
||
}
|
||
return {
|
||
"manifest": final_dir / f"{prefix}.annotation_{scope}.manifest.json",
|
||
"progress": ws / "annotation" / "analysis" / f"{scope}.progress.json",
|
||
"lock": ws / "annotation" / "analysis" / f".annotation.{scope}.lock",
|
||
}
|
||
|
||
|
||
def _resolve_sequence_ids_file(job: "JobInfo", value: str) -> Optional[Path]:
|
||
text = str(value or "").strip()
|
||
if not text:
|
||
return None
|
||
path = Path(text).expanduser()
|
||
allowed_suffixes = {".ids", ".id", ".list", ".txt", ".tsv", ".csv"}
|
||
if path.suffix.lower() not in allowed_suffixes:
|
||
return None
|
||
if path.is_absolute():
|
||
return path if path.exists() and path.is_file() else None
|
||
search_roots = [Path(job.input_dir), Path(job.workspace_dir), ATH_FAMILY_ROOT]
|
||
candidate_names = [path]
|
||
compact_name = path.name.replace("-", "")
|
||
if compact_name != path.name:
|
||
candidate_names.append(path.with_name(compact_name))
|
||
for root in search_roots:
|
||
for candidate_name in candidate_names:
|
||
candidate = root / candidate_name
|
||
if candidate.exists() and candidate.is_file():
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def _resolve_job_ids_file(job: "JobInfo", prefix: str) -> Optional[Path]:
|
||
prefix = str(prefix or "").strip()
|
||
if not prefix:
|
||
return None
|
||
for suffix in (".ids", ".id", ".list", ".txt"):
|
||
candidate = _resolve_sequence_ids_file(job, f"{prefix}{suffix}")
|
||
if candidate:
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def _now_tag() -> str:
|
||
return datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
|
||
|
||
def cleanup_old_jobs(retention_days: int = RETENTION_DAYS) -> str:
|
||
"""清理 input/<job> 和 workspace/<job> 中超过保留期的任务目录"""
|
||
cutoff = datetime.now() - timedelta(days=retention_days)
|
||
removed = []
|
||
|
||
for root in [INPUT_ROOT, WORKSPACE_ROOT]:
|
||
if not root.exists():
|
||
continue
|
||
for d in root.iterdir():
|
||
if not d.is_dir():
|
||
continue
|
||
try:
|
||
mtime = datetime.fromtimestamp(d.stat().st_mtime)
|
||
except Exception:
|
||
continue
|
||
if mtime < cutoff:
|
||
try:
|
||
shutil.rmtree(d, ignore_errors=True)
|
||
removed.append(str(d))
|
||
except Exception:
|
||
pass
|
||
|
||
if not removed:
|
||
return f"✅ 清理完成：未发现超过 {retention_days} 天的任务目录"
|
||
return "✅ 清理完成，已删除：\n" + "\n".join(f"- {p}" for p in removed)
|
||
|
||
|
||
@dataclass
|
||
class JobInfo:
|
||
job_id: str
|
||
input_dir: str
|
||
workspace_dir: str
|
||
logs_dir: str
|
||
created_at: str
|
||
|
||
|
||
def new_job() -> JobInfo:
|
||
jid = f"{_now_tag()}_{os.getpid()}_{os.urandom(3).hex()}"
|
||
in_dir = (INPUT_ROOT / jid)
|
||
ws_dir = (WORKSPACE_ROOT / jid)
|
||
logs_dir = (ws_dir / "logs")
|
||
in_dir.mkdir(parents=True, exist_ok=True)
|
||
ws_dir.mkdir(parents=True, exist_ok=True)
|
||
logs_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
return JobInfo(
|
||
job_id=jid,
|
||
input_dir=str(in_dir.resolve()),
|
||
workspace_dir=str(ws_dir.resolve()),
|
||
logs_dir=str(logs_dir.resolve()),
|
||
created_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
)
|
||
|
||
|
||
def _as_path(f) -> Path:
|
||
# 兼容 gradio 不同版本的 file 返回形式
|
||
if isinstance(f, str):
|
||
return Path(f)
|
||
if isinstance(f, dict):
|
||
if "path" in f:
|
||
return Path(f["path"])
|
||
if "name" in f:
|
||
return Path(f["name"])
|
||
if hasattr(f, "name"):
|
||
return Path(f.name)
|
||
raise ValueError(f"Unknown file object: {type(f)}")
|
||
|
||
|
||
def upload_to_job(files, job: JobInfo) -> Tuple[str, List[List[str]]]:
|
||
if not files:
|
||
return "请选择文件", list_input_table(job)
|
||
|
||
in_dir = Path(job.input_dir)
|
||
added = []
|
||
for f in files:
|
||
src = _as_path(f)
|
||
dest = in_dir / src.name
|
||
if dest.exists():
|
||
dest = in_dir / f"{src.stem}_{datetime.now().strftime('%H%M%S')}{src.suffix}"
|
||
shutil.copy(src, dest)
|
||
added.append(dest.name)
|
||
|
||
return f"✅ 已上传到 input/{job.job_id}/: {', '.join(added)}", list_input_table(job)
|
||
|
||
|
||
def list_input_table(job: JobInfo) -> List[List[str]]:
|
||
"""[文件名, 大小, mtime, 完整路径]"""
|
||
in_dir = Path(job.input_dir)
|
||
rows = []
|
||
if not in_dir.exists():
|
||
return rows
|
||
for p in sorted(in_dir.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True):
|
||
if not p.is_file():
|
||
continue
|
||
size = p.stat().st_size
|
||
if size >= 1024 * 1024:
|
||
size_str = f"{size / 1024 / 1024:.1f} MB"
|
||
elif size >= 1024:
|
||
size_str = f"{size / 1024:.1f} KB"
|
||
else:
|
||
size_str = f"{size} B"
|
||
mtime = datetime.fromtimestamp(p.stat().st_mtime).strftime("%m-%d %H:%M")
|
||
rows.append([p.name, size_str, mtime, str(p.resolve())])
|
||
return rows
|
||
|
||
|
||
def _strip_gz(name: str) -> str:
|
||
return name[:-3] if name.lower().endswith(".gz") else name
|
||
|
||
|
||
def detect_role(p: Path) -> str:
|
||
name = _strip_gz(p.name.lower())
|
||
suf = Path(name).suffix.lower()
|
||
|
||
if suf in [".fa", ".fasta", ".faa", ".pep"]:
|
||
return "fasta"
|
||
if suf in [".gff", ".gff3"]:
|
||
return "gff3"
|
||
if suf in [".hmm"]:
|
||
return "hmm_db"
|
||
if name.endswith(".domtblout") or "domtblout" in name:
|
||
return "hmmer_domtblout"
|
||
if suf in [".id"]:
|
||
return "id_list"
|
||
if suf in [".tsv", ".txt", ".csv"]:
|
||
return "table_or_txt"
|
||
if suf in [".zip"]:
|
||
return "zip"
|
||
return "unknown"
|
||
|
||
|
||
def infer_species_code_from_names(files: List[Path]) -> Optional[str]:
|
||
# 尝试从文件名开头提取类似 Ft / At / Os 这样的物种缩写
|
||
# 例如 FtChromosomeV2... -> Ft
|
||
for p in files:
|
||
m = re.match(r"^([A-Z][a-z])", p.name)
|
||
if m:
|
||
return m.group(1)
|
||
return None
|
||
|
||
|
||
def l1_advise(job: JobInfo) -> str:
|
||
in_dir = Path(job.input_dir)
|
||
files = [p for p in in_dir.iterdir() if p.is_file()] if in_dir.exists() else []
|
||
if not files:
|
||
return (
|
||
"⚠️ 还没有上传文件。\n\n"
|
||
"建议先上传（至少一种）：\n"
|
||
"- 蛋白序列（*.fa/*.fasta/*.faa）\n"
|
||
"- 注释（*.gff3，可选）\n\n"
|
||
"若你希望做基因家族鉴定（identify），通常还需要：\n"
|
||
"- 候选ID列表（*.id）或 hmmer domtblout 结果（*.domtblout）\n"
|
||
"-（系统已预埋）Pfam-A.hmm 等资源\n"
|
||
)
|
||
|
||
roles: Dict[str, List[str]] = {}
|
||
for p in files:
|
||
roles.setdefault(detect_role(p), []).append(p.name)
|
||
|
||
species = infer_species_code_from_names(files) or "Unknown"
|
||
lines = []
|
||
lines.append("### 📦 已识别到的数据类型（L1）")
|
||
lines.append(f"- 推测物种缩写：**{species}**（可在前端手动改）")
|
||
for k, v in roles.items():
|
||
lines.append(f"- **{k}**: {', '.join(v)}")
|
||
|
||
#NV| # 检测表达量文件
|
||
has_expression = False
|
||
expression_files = []
|
||
for p in files:
|
||
if p.suffix == ".txt" or p.suffix == ".tsv":
|
||
name_lower = p.name.lower()
|
||
if "fpkm" in name_lower or "tpm" in name_lower or "expression" in name_lower or "exp" in name_lower:
|
||
has_expression = True
|
||
expression_files.append(p.name)
|
||
|
||
# 检测样本信息文件
|
||
has_sample_info = any(p.name.lower() == "sample.txt" for p in files)
|
||
|
||
# 检测基因ID文件
|
||
has_gene_id = any("geneid" in p.name.lower() for p in files)
|
||
|
||
|
||
has_fasta = "fasta" in roles
|
||
has_domtbl = "hmmer_domtblout" in roles
|
||
has_id = "id_list" in roles
|
||
|
||
#ZN| lines.append("\n### ✅ 基于当前上传，你可以直接做的分析")
|
||
#JB| if has_fasta:
|
||
#RT| lines.append("- **Phylogeny**：MAFFT/MUSCLE → IQ-TREE（你机器上是 iqtree）")
|
||
#YN| lines.append("- **MSA figure**：mafft 绘图法（主）+ 保守区二次绘图")
|
||
#XW| lines.append("- **Motif（MEME）**：MEME + seqlogo（若本机已装 MEME Suite）")
|
||
#ZT| lines.append("- **Summary**：汇总报告 + 关键文件索引")
|
||
#ZR| else:
|
||
#JK| lines.append("- 目前缺少 FASTA（蛋白序列），先上传 fasta 才能继续。")
|
||
#XJ|
|
||
#XZ| # Expression 差异表达分析建议
|
||
#XB| if has_expression and has_gene_id and has_sample_info:
|
||
#PQ| lines.append("- **Expression**：差异表达分析（热图、直方图、相关性分析）")
|
||
#XT| elif has_expression:
|
||
#KM| lines.append("- **Expression**：差异表达分析（还需：基因ID列表 + sample.txt）")
|
||
|
||
|
||
if has_fasta:
|
||
lines.append("- **Phylogeny**：MAFFT/MUSCLE → IQ-TREE（你机器上是 iqtree）")
|
||
lines.append("- **MSA figure**：mafft 绘图法（主）+ 保守区二次绘图")
|
||
lines.append("- **Motif（MEME）**：MEME + seqlogo（若本机已装 MEME Suite）")
|
||
lines.append("- **Summary**：汇总报告 + 关键文件索引")
|
||
else:
|
||
lines.append("- 目前缺少 FASTA（蛋白序列），先上传 fasta 才能继续。")
|
||
|
||
lines.append("\n### 🧩 若要做 Identify（基因家族鉴定）通常还需要")
|
||
if not has_fasta:
|
||
lines.append("- 必需：蛋白序列 fasta（pep/protein）")
|
||
if not (has_domtbl or has_id):
|
||
lines.append("- 候选证据：hmmer domtblout 或 候选ID列表（*.id）")
|
||
lines.append("-（项目内置）Pfam-A.hmm：data/hmm/Pfam-A.hmm")
|
||
lines.append("\n> 说明：Identify 这一步你现有代码的“输入映射规则”需要在 runner 里固定下来（见下方 runner 的 identify 入口）。")
|
||
|
||
lines.append(f"\n### ⏳ 结果保留策略\n- 本地结果默认保留 **{RETENTION_DAYS} 天**，到期会自动清理（前端会提示）。")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def identify_hmm_audit_failure(manifest_path: Path | None) -> tuple[bool, str]:
|
||
if not manifest_path or not manifest_path.exists():
|
||
return False, ""
|
||
try:
|
||
data = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||
except Exception as exc:
|
||
return False, f"identify manifest unreadable for HMM audit: {exc}"
|
||
if str(data.get("stage") or "") != "identify":
|
||
return False, ""
|
||
hmm_models = data.get("hmm_models") if isinstance(data.get("hmm_models"), dict) else {}
|
||
missing = hmm_models.get("missing_requested_pfams") or []
|
||
status = str(hmm_models.get("status") or "").lower()
|
||
if missing or status == "failed":
|
||
missing_text = ",".join(str(x) for x in missing) if missing else "unknown"
|
||
return True, f"Identify HMM audit failed: missing requested Pfam model(s): {missing_text}"
|
||
return False, ""
|
||
|
||
|
||
def _read_manifest(manifest_path: Path) -> Optional[dict]:
|
||
try:
|
||
return json.loads(manifest_path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return None
|
||
|
||
def normalize_manifest(d: dict, manifest_path: Path) -> dict:
|
||
"""标准化 manifest：补 workspace_root；统一 artifacts 格式；优先 relpath_or_hint"""
|
||
if not d:
|
||
return {}
|
||
|
||
out = dict(d)
|
||
|
||
# workspace_root：优先字段，否则从路径推断 workspace/<job_id>/
|
||
if not out.get("workspace_root"):
|
||
try:
|
||
out["workspace_root"] = str(manifest_path.parents[1].resolve())
|
||
except Exception:
|
||
out["workspace_root"] = str(manifest_path.parent.resolve())
|
||
|
||
# workspace：保留兼容（通常是 final/）
|
||
if not out.get("workspace"):
|
||
out["workspace"] = str(manifest_path.parent.resolve())
|
||
|
||
# artifacts：处理三种格式
|
||
arts = out.get("artifacts", {}) or {}
|
||
|
||
if isinstance(arts, list):
|
||
# 格式1：list[dict] 每个 dict 有 artifact 字段
|
||
arts_dict = {}
|
||
for item in arts:
|
||
if isinstance(item, dict):
|
||
# 提取 artifact 名称
|
||
art_name = item.get("artifact")
|
||
if not art_name:
|
||
# 如果没有 artifact 字段，尝试从其他字段推断
|
||
if "file" in item:
|
||
art_name = Path(item["file"]).stem
|
||
else:
|
||
art_name = f"artifact_{len(arts_dict)}"
|
||
|
||
# 确保每个条目都有 artifact 字段
|
||
v = dict(item)
|
||
if "artifact" not in v:
|
||
v["artifact"] = art_name
|
||
|
||
# 统一路径字段
|
||
if not v.get("path") and v.get("relpath_or_hint"):
|
||
v["path"] = v["relpath_or_hint"]
|
||
|
||
arts_dict[art_name] = v
|
||
arts = arts_dict
|
||
|
||
elif isinstance(arts, dict):
|
||
# 格式2：dict[art_name, dict]
|
||
arts2 = {}
|
||
for art_name, item in arts.items():
|
||
if isinstance(item, dict):
|
||
v = dict(item)
|
||
# 确保有 artifact 字段
|
||
if "artifact" not in v:
|
||
v["artifact"] = art_name
|
||
|
||
# 统一路径字段
|
||
if not v.get("path") and v.get("relpath_or_hint"):
|
||
v["path"] = v["relpath_or_hint"]
|
||
|
||
arts2[art_name] = v
|
||
else:
|
||
# 格式3：dict[art_name, str]（直接是文件路径）
|
||
arts2[art_name] = {
|
||
"artifact": art_name,
|
||
"path": item,
|
||
"full_path": str(Path(item).resolve()) if not Path(item).is_absolute() else item,
|
||
}
|
||
arts = arts2
|
||
|
||
out["artifacts"] = arts
|
||
return out
|
||
|
||
def find_latest_manifest(workspace_dir: Path) -> Optional[Path]:
|
||
cands = sorted(workspace_dir.rglob("*.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||
for path in cands:
|
||
try:
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return path
|
||
if data.get("stage") == "minimal":
|
||
continue
|
||
return path
|
||
return None
|
||
|
||
def stage_allows_minimal_manifest(stage: str) -> bool:
|
||
"""Only protein-family downstream views may use an uploaded FASTA as a minimal input."""
|
||
return stage in {"phylogeny", "motif", "domain", "structure", "combo", "pep", "msa", "msa_region_plot", "synteny_plot_revise"}
|
||
|
||
def ensure_min_manifest(workspace_dir: Path, prefix: str, final_fa: Path) -> Path:
|
||
# 用于用户只上传了 fasta，但也想跑 phylogeny/motif/msa_figure/summary 的场景
|
||
out_dir = workspace_dir / "final"
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
mp = out_dir / f"{prefix}.minimal.manifest.json"
|
||
|
||
# helper：尽量生成“可用的相对路径”，失败就退回文件名
|
||
def _safe_rel(p: Path, base: Path) -> str:
|
||
try:
|
||
return str(p.resolve().relative_to(base.resolve()))
|
||
except Exception:
|
||
return p.name
|
||
|
||
if mp.exists() and mp.stat().st_size > 0:
|
||
# 如果已存在，确保包含必要字段
|
||
try:
|
||
existing = json.loads(mp.read_text(encoding="utf-8"))
|
||
|
||
# 补齐 workspace_root / workspace / input_dir（兼容旧manifest）
|
||
if "workspace_root" not in existing:
|
||
existing["workspace_root"] = str(workspace_dir.resolve())
|
||
if "workspace" not in existing:
|
||
existing["workspace"] = str(out_dir.resolve())
|
||
if "input_dir" not in existing:
|
||
existing["input_dir"] = str(final_fa.parent.resolve())
|
||
|
||
# 兼容 artifacts 可能是 list 的情况（你前面 key_results 已做过，这里再保险一次）
|
||
artifacts = existing.get("artifacts", {}) or {}
|
||
if isinstance(artifacts, list):
|
||
artifacts_dict = {}
|
||
for item in artifacts:
|
||
if isinstance(item, dict) and item.get("artifact"):
|
||
artifacts_dict[item["artifact"]] = item
|
||
artifacts = artifacts_dict
|
||
|
||
existing["artifacts"] = artifacts
|
||
mp.write_text(json.dumps(existing, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return mp
|
||
except Exception:
|
||
pass # 如果解析失败，重新创建
|
||
|
||
# 生成 minimal manifest（注意：final_fa 在 input_dir，不在 workspace_dir）
|
||
fa_abs = final_fa.resolve()
|
||
ws_root = workspace_dir.resolve()
|
||
in_dir = final_fa.parent.resolve()
|
||
|
||
fa_art = {
|
||
"artifact": "final_fa",
|
||
"path": _safe_rel(fa_abs, in_dir), # 相对 input_dir 更合理
|
||
"full_path": str(fa_abs),
|
||
"type": "fasta",
|
||
"size": fa_abs.stat().st_size if fa_abs.exists() else 0,
|
||
"sha256": _sha256_file(fa_abs),
|
||
"source": "uploaded",
|
||
}
|
||
|
||
d = {
|
||
"manifest_version": "1.0",
|
||
"stage": "minimal",
|
||
"prefix": prefix,
|
||
"status": "created",
|
||
"timestamp": datetime.now().isoformat(),
|
||
|
||
# 关键：同时写 workspace_root + workspace
|
||
"workspace_root": str(ws_root),
|
||
"workspace": str(out_dir.resolve()),
|
||
"input_dir": str(in_dir),
|
||
|
||
# 最小也给 python 版本（后续 identify 真跑会覆盖成完整 tool_versions）
|
||
"tool_versions": {"python": f"{os.sys.version_info.major}.{os.sys.version_info.minor}.{os.sys.version_info.micro}"},
|
||
|
||
# 兼容两种 key：final_fa / final_proteins
|
||
"artifacts": {
|
||
"final_fa": fa_art,
|
||
"final_proteins": {**fa_art, "artifact": "final_proteins"},
|
||
"manifest_json": {
|
||
"artifact": "manifest_json",
|
||
"path": _safe_rel(mp.resolve(), ws_root),
|
||
"full_path": str(mp.resolve()),
|
||
"type": "json",
|
||
"size": 0,
|
||
},
|
||
},
|
||
|
||
# 兼容你旧代码可能用 params
|
||
"params": {},
|
||
"parameters": {},
|
||
|
||
"steps": [
|
||
{
|
||
"agent": "min_manifest",
|
||
"timestamp": datetime.now().isoformat(),
|
||
"operation": "create_minimal_manifest",
|
||
"out": {"final_fa": str(fa_abs)},
|
||
}
|
||
],
|
||
}
|
||
|
||
mp.write_text(json.dumps(d, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return mp
|
||
|
||
|
||
def normalize_manifest(manifest_data: dict, manifest_path: Path) -> dict:
|
||
"""标准化 manifest 数据格式"""
|
||
if not manifest_data:
|
||
return {}
|
||
|
||
# 1. 确保 workspace_root 存在
|
||
if 'workspace_root' not in manifest_data:
|
||
# 推断：final/xxx.manifest.json → workspace/<job_id>
|
||
manifest_data['workspace_root'] = str(manifest_path.parents[1])
|
||
|
||
# 2. 标准化 artifacts 为字典格式
|
||
artifacts = manifest_data.get('artifacts', {})
|
||
if isinstance(artifacts, list):
|
||
artifacts_dict = {}
|
||
for item in artifacts:
|
||
if isinstance(item, dict) and 'artifact' in item:
|
||
artifacts_dict[item['artifact']] = item
|
||
manifest_data['artifacts'] = artifacts_dict
|
||
|
||
return manifest_data
|
||
|
||
def key_results_from_manifest(manifest_path: Path) -> Tuple[List[List[str]], List[str]]:
|
||
"""返回：关键文件表格 + 图片列表（用于 gallery）"""
|
||
d = _read_manifest(manifest_path)
|
||
if not d:
|
||
return [], []
|
||
|
||
d = normalize_manifest(d, manifest_path)
|
||
|
||
ws_root = Path(d["workspace_root"])
|
||
artifacts = d.get("artifacts", {}) or {}
|
||
|
||
key_paths = []
|
||
for k, v in artifacts.items():
|
||
file_path = None
|
||
rel_hint = None
|
||
# ========== 新增：提取 caption ==========
|
||
caption = None
|
||
# ========== 新增结束 ==========
|
||
|
||
if isinstance(v, dict):
|
||
file_path = v.get("full_path") or v.get("path") or v.get("relpath_or_hint") or v.get("file")
|
||
rel_hint = v.get("relpath_or_hint") or v.get("path")
|
||
# ========== 新增：从 artifact 字典中提取 caption ==========
|
||
caption = v.get("caption")
|
||
# ========== 新增结束 ==========
|
||
elif isinstance(v, str):
|
||
file_path = v
|
||
|
||
if not file_path:
|
||
continue
|
||
|
||
p = Path(file_path)
|
||
if not p.is_absolute():
|
||
p = ws_root / file_path
|
||
|
||
if p.exists() and p.is_file():
|
||
# key_paths.append((k, str(p.resolve()), rel_hint))
|
||
# ========== 修改：传递 caption ==========
|
||
key_paths.append((k, str(p.resolve()), rel_hint, caption))
|
||
# ========== 修改结束 ==========
|
||
|
||
# 置顶（把你的 identify_report 也加进去）
|
||
priority = [
|
||
"identify_report", "report_md", "report_html",
|
||
"summary_md", "summary_files_tsv", "tree_nwk", "meme_html", "meme_txt"
|
||
]
|
||
key_paths.sort(key=lambda kv: (0 if kv[0] in priority else 1, kv[0]))
|
||
|
||
table = []
|
||
images = []
|
||
|
||
# for name, abs_path, rel_hint in key_paths:
|
||
# ========== 修改：解包时包含 caption ==========
|
||
for name, abs_path, rel_hint, caption in key_paths:
|
||
# ========== 修改结束 ==========
|
||
p = Path(abs_path)
|
||
|
||
# 优先用 relpath_or_hint（你 manifest 里已经写好了）
|
||
if rel_hint:
|
||
rel = rel_hint
|
||
else:
|
||
rel = str(p.relative_to(ws_root)) if p.is_relative_to(ws_root) else str(p)
|
||
|
||
ext = p.suffix.lower()
|
||
if ext in [".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"]:
|
||
# 使用 caption 或自动生成标题
|
||
display_caption = caption or name.replace("_", " ").title()
|
||
# Gradio Gallery 期望 List[Tuple[str, str]] 格式: (path, caption)
|
||
images.append((str(p), display_caption))
|
||
table.append([name, rel, ext])
|
||
|
||
return table, images
|
||
|
||
def list_all_results(workspace_dir: Path, limit: int = 200) -> List[List[str]]:
|
||
"""列出 workspace/<job>/ 下所有文件（完整路径）"""
|
||
rows = []
|
||
if not workspace_dir.exists():
|
||
return rows
|
||
|
||
files = [p for p in workspace_dir.rglob("*") if p.is_file() and not p.name.startswith(".")]
|
||
files.sort(key=lambda p: p.stat().st_mtime, reverse=True)
|
||
for p in files[:limit]:
|
||
size = p.stat().st_size
|
||
if size >= 1024 * 1024:
|
||
size_str = f"{size / 1024 / 1024:.1f} MB"
|
||
elif size >= 1024:
|
||
size_str = f"{size / 1024:.1f} KB"
|
||
else:
|
||
size_str = f"{size} B"
|
||
mtime = datetime.fromtimestamp(p.stat().st_mtime).strftime("%m-%d %H:%M")
|
||
rows.append([p.name, size_str, mtime, str(p.resolve())])
|
||
return rows
|
||
|
||
|
||
def preview_text(path: str, max_chars: int = 20000) -> str:
|
||
p = Path(path)
|
||
if not p.exists():
|
||
return "文件不存在"
|
||
try:
|
||
content = p.read_text(encoding="utf-8", errors="ignore")[:max_chars]
|
||
if p.stat().st_size > max_chars:
|
||
content += "\n\n... (文件较大，仅显示部分)"
|
||
return content
|
||
except Exception:
|
||
return "无法预览此文件（可能是二进制/图片/PDF）"
|
||
|
||
# ======== 新增：文件hash工具函数（给最小manifest用）========
|
||
import hashlib
|
||
|
||
def _sha256_file(p: Path, chunk_size: int = 1024 * 1024) -> Optional[str]:
|
||
try:
|
||
h = hashlib.sha256()
|
||
with p.open("rb") as f:
|
||
while True:
|
||
b = f.read(chunk_size)
|
||
if not b:
|
||
break
|
||
h.update(b)
|
||
return h.hexdigest()
|
||
except Exception:
|
||
return None
|
||
# ======== 新增结束 ========
|
||
|
||
|
||
def download_string_network_image(
|
||
string_url: str,
|
||
output_file: Path,
|
||
*,
|
||
http_client: Any = None,
|
||
rate_gate: Any = None,
|
||
) -> Optional[Path]:
|
||
"""Download a static STRING network image for frontend display."""
|
||
if not string_url or "string-db.org/cgi/network?" not in string_url:
|
||
return None
|
||
import requests
|
||
from urllib.parse import parse_qs, urlparse
|
||
|
||
parsed = urlparse(string_url)
|
||
params = {key: values[-1] for key, values in parse_qs(parsed.query, keep_blank_values=True).items()}
|
||
params.setdefault("network_flavor", "confidence")
|
||
params.setdefault("hide_disconnected_nodes", "0")
|
||
params["caller_identity"] = "PlantAI"
|
||
client = http_client or requests
|
||
shared_rate_gate = rate_gate or StringRateGate()
|
||
|
||
attempts = [
|
||
("https://string-db.org/api/highres_image/network", output_file, "STRING highres_image API", (8, 30), 40),
|
||
("https://string-db.org/api/image/network", output_file, "STRING image API", (5, 20), 30),
|
||
("https://string-db.org/api/svg/network", output_file.with_suffix(".svg"), "STRING svg API", (5, 20), 30),
|
||
]
|
||
errors: List[str] = []
|
||
for request_url, target_file, source_label, timeout_value, max_seconds in attempts:
|
||
try:
|
||
target_file.parent.mkdir(parents=True, exist_ok=True)
|
||
if target_file.exists():
|
||
target_file.unlink()
|
||
expected_svg = target_file.suffix.lower() == ".svg"
|
||
expected_token = "svg" if expected_svg else "image"
|
||
if expected_svg:
|
||
params.setdefault("block_structure_pics_in_bubbles", "0")
|
||
|
||
start_time = time.time()
|
||
shared_rate_gate.wait()
|
||
response = client.post(
|
||
request_url,
|
||
data=params,
|
||
timeout=timeout_value,
|
||
stream=True,
|
||
allow_redirects=False,
|
||
)
|
||
response_url = getattr(response, "url", None)
|
||
if not is_authoritative_string_https_url(response_url):
|
||
raise RuntimeError(f"unexpected response authority: {response_url}")
|
||
status_code = getattr(response, "status_code", None)
|
||
if (
|
||
isinstance(status_code, bool)
|
||
or not isinstance(status_code, int)
|
||
or not 200 <= status_code < 300
|
||
):
|
||
raise RuntimeError(f"unexpected HTTP status: {status_code!r}")
|
||
response.raise_for_status()
|
||
content_type = response.headers.get("content-type", "").lower()
|
||
if expected_token not in content_type:
|
||
raise RuntimeError(f"unexpected content-type: {content_type or 'unknown'}")
|
||
|
||
chunks = []
|
||
deadline = start_time + max_seconds
|
||
for chunk in response.iter_content(chunk_size=1024 * 256):
|
||
if time.time() > deadline:
|
||
raise TimeoutError(f"{source_label} download exceeded {max_seconds} seconds")
|
||
if chunk:
|
||
chunks.append(chunk)
|
||
content = b"".join(chunks)
|
||
if not content:
|
||
raise RuntimeError("empty response body")
|
||
|
||
target_file.write_bytes(content)
|
||
if target_file.exists() and target_file.stat().st_size > 0:
|
||
return target_file
|
||
except Exception as exc:
|
||
errors.append(f"{source_label}: {exc}")
|
||
continue
|
||
|
||
if errors:
|
||
print(f"[PPI] STRING network image download skipped: {'; '.join(errors)}")
|
||
return None
|
||
|
||
|
||
_UNIPROT_ACCESSION_RE = re.compile(
|
||
r"^(?:[OPQ][0-9][A-Z0-9]{3}[0-9]|[A-NR-Z][0-9][A-Z][A-Z0-9]{2}[0-9]|A0A[A-Z0-9]{7})$"
|
||
)
|
||
|
||
|
||
def _alphafold_lookup_id(identifier: str) -> Tuple[str, str]:
|
||
"""Return a conservative AlphaFold DB lookup token and link type."""
|
||
token = str(identifier or "").strip()
|
||
if not token:
|
||
return "", "unresolved"
|
||
if "." in token:
|
||
suffix = token.rsplit(".", 1)[-1]
|
||
if _UNIPROT_ACCESSION_RE.match(suffix):
|
||
return suffix, "direct_entry"
|
||
if "_" in token:
|
||
head = token.split("_", 1)[0]
|
||
if _UNIPROT_ACCESSION_RE.match(head):
|
||
return head, "direct_entry"
|
||
if _UNIPROT_ACCESSION_RE.match(token):
|
||
return token, "direct_entry"
|
||
return token, "search"
|
||
|
||
|
||
def create_alphafold2_links(
|
||
id_resolution_file: Path,
|
||
network_file: Path,
|
||
output_file: Path,
|
||
) -> Optional[Path]:
|
||
"""Create AlphaFold DB/Fold2 single-protein structure links from PPI outputs."""
|
||
from urllib.parse import quote
|
||
|
||
candidates: List[str] = []
|
||
if network_file.exists() and network_file.stat().st_size > 0:
|
||
with network_file.open("r", encoding="utf-8", errors="ignore", newline="") as handle:
|
||
reader = csv.DictReader(handle, delimiter="\t")
|
||
for row in reader:
|
||
for key in ("protein1", "protein2"):
|
||
value = str(row.get(key) or "").strip()
|
||
if value:
|
||
candidates.append(value)
|
||
|
||
if id_resolution_file.exists() and id_resolution_file.stat().st_size > 0:
|
||
with id_resolution_file.open("r", encoding="utf-8", errors="ignore", newline="") as handle:
|
||
reader = csv.DictReader(handle, delimiter="\t")
|
||
for row in reader:
|
||
for key in ("string_id", "selected_query_id", "normalized_id", "raw_id"):
|
||
value = str(row.get(key) or "").strip()
|
||
if value:
|
||
candidates.append(value)
|
||
break
|
||
|
||
seen: set[str] = set()
|
||
rows: List[Dict[str, str]] = []
|
||
for source_id in candidates:
|
||
if source_id in seen:
|
||
continue
|
||
seen.add(source_id)
|
||
lookup_id, link_type = _alphafold_lookup_id(source_id)
|
||
if not lookup_id:
|
||
continue
|
||
if link_type == "direct_entry":
|
||
url = f"https://alphafold.ebi.ac.uk/entry/{quote(lookup_id)}"
|
||
note = "direct accession-style lookup"
|
||
else:
|
||
url = f"https://alphafold.ebi.ac.uk/search/text/{quote(lookup_id)}"
|
||
note = "search link; accession not confirmed"
|
||
rows.append({
|
||
"source_id": source_id,
|
||
"lookup_id": lookup_id,
|
||
"link_type": link_type,
|
||
"alphafold_db_url": url,
|
||
"structure_type": "AlphaFold DB/Fold2 single protein",
|
||
"note": note,
|
||
})
|
||
|
||
if not rows:
|
||
return None
|
||
|
||
output_file.parent.mkdir(parents=True, exist_ok=True)
|
||
with output_file.open("w", encoding="utf-8", newline="") as handle:
|
||
writer = csv.DictWriter(
|
||
handle,
|
||
fieldnames=[
|
||
"source_id",
|
||
"lookup_id",
|
||
"link_type",
|
||
"alphafold_db_url",
|
||
"structure_type",
|
||
"note",
|
||
],
|
||
delimiter="\t",
|
||
)
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
return output_file if output_file.exists() and output_file.stat().st_size > 0 else None
|
||
|
||
|
||
def sync_ppi_manifest_final_artifacts(manifest_file: Path, final_dir: Path, prefix: str) -> None:
|
||
"""Point copied PPI skill manifest artifacts at the final/ copies."""
|
||
if not manifest_file.exists():
|
||
return
|
||
final_artifacts = {
|
||
"network": (final_dir / f"{prefix}.ppi_network.tsv", "table", "tsv"),
|
||
"report": (final_dir / f"{prefix}.ppi_report.md", "markdown", "markdown"),
|
||
"id_resolution": (final_dir / f"{prefix}.ppi_id_resolution.tsv", "table", "tsv"),
|
||
"string_url": (final_dir / f"{prefix}.ppi_url.txt", "link", "txt"),
|
||
}
|
||
try:
|
||
with manifest_file.open("r", encoding="utf-8") as handle:
|
||
data = json.load(handle)
|
||
artifacts = data.setdefault("artifacts", {})
|
||
changed = False
|
||
for key, (path, artifact_type, artifact_format) in final_artifacts.items():
|
||
if not path.exists():
|
||
continue
|
||
record = artifacts.setdefault(key, {})
|
||
record.update({
|
||
"artifact": key,
|
||
"path": f"final/{path.name}",
|
||
"full_path": str(path.absolute()),
|
||
"type": artifact_type,
|
||
"format": artifact_format,
|
||
"size": path.stat().st_size,
|
||
"sha256": _sha256_file(path),
|
||
})
|
||
changed = True
|
||
if changed:
|
||
with manifest_file.open("w", encoding="utf-8") as handle:
|
||
json.dump(data, handle, ensure_ascii=False, indent=2)
|
||
except Exception as exc:
|
||
print(f"[PPI] Could not sync PPI manifest artifact paths: {exc}")
|
||
|
||
|
||
def _read_gene_id_lines(path: Path) -> List[str]:
|
||
if not path.exists():
|
||
return []
|
||
seen: set[str] = set()
|
||
values: List[str] = []
|
||
for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||
token = line.strip()
|
||
if not token or token.startswith("#") or token in seen:
|
||
continue
|
||
seen.add(token)
|
||
values.append(token)
|
||
return values
|
||
|
||
|
||
def _next_ppi_reset_dir(final_dir: Path) -> Tuple[str, Path]:
|
||
reset_root = final_dir / "ppi_reset"
|
||
reset_root.mkdir(parents=True, exist_ok=True)
|
||
used: set[int] = set()
|
||
for path in reset_root.glob("reset_*"):
|
||
if not path.is_dir():
|
||
continue
|
||
match = re.fullmatch(r"reset_(\d{3})", path.name)
|
||
if match:
|
||
used.add(int(match.group(1)))
|
||
next_index = 1
|
||
while next_index in used:
|
||
next_index += 1
|
||
reset_id = f"reset_{next_index:03d}"
|
||
reset_dir = reset_root / reset_id
|
||
reset_dir.mkdir(parents=False, exist_ok=False)
|
||
return reset_id, reset_dir
|
||
|
||
|
||
def prepare_ppi_reset_inputs(
|
||
final_dir: Path,
|
||
prefix: str,
|
||
source_gene_id_file: Path,
|
||
submitted_gene_ids: List[str],
|
||
instruction: str = "",
|
||
ppi_species: str = "auto",
|
||
ppi_score: int = 200,
|
||
) -> Dict[str, Any]:
|
||
"""Create isolated PPI reset input/diff/manifest files without touching original PPI outputs."""
|
||
original_ids = _read_gene_id_lines(source_gene_id_file)
|
||
submitted_seen: set[str] = set()
|
||
clean_submitted: List[str] = []
|
||
for item in submitted_gene_ids:
|
||
token = str(item or "").strip()
|
||
if not token or token in submitted_seen:
|
||
continue
|
||
submitted_seen.add(token)
|
||
clean_submitted.append(token)
|
||
if not clean_submitted:
|
||
raise ValueError("PPI reset requires at least one gene ID")
|
||
|
||
reset_id, reset_dir = _next_ppi_reset_dir(final_dir)
|
||
input_file = reset_dir / f"{prefix}.ppi_reset_input.id"
|
||
diff_file = reset_dir / f"{prefix}.ppi_reset_diff.tsv"
|
||
manifest_file = reset_dir / f"{prefix}.ppi_reset.manifest.json"
|
||
|
||
input_file.write_text("\n".join(clean_submitted) + "\n", encoding="utf-8")
|
||
|
||
original_set = set(original_ids)
|
||
submitted_set = set(clean_submitted)
|
||
deleted_ids = [gid for gid in original_ids if gid not in submitted_set]
|
||
added_ids = [gid for gid in clean_submitted if gid not in original_set]
|
||
kept_ids = [gid for gid in clean_submitted if gid in original_set]
|
||
with diff_file.open("w", encoding="utf-8", newline="") as handle:
|
||
writer = csv.writer(handle, delimiter="\t")
|
||
writer.writerow(["gene_id", "action"])
|
||
for gene_id in kept_ids:
|
||
writer.writerow([gene_id, "kept"])
|
||
for gene_id in deleted_ids:
|
||
writer.writerow([gene_id, "deleted"])
|
||
for gene_id in added_ids:
|
||
writer.writerow([gene_id, "added"])
|
||
|
||
manifest = {
|
||
"manifest_version": "1.0",
|
||
"stage": "ppi_reset",
|
||
"status": "prepared",
|
||
"prefix": prefix,
|
||
"reset_id": reset_id,
|
||
"created_at": datetime.now().isoformat(),
|
||
"source_gene_id_file": str(source_gene_id_file),
|
||
"input_gene_id_file": str(input_file),
|
||
"diff_file": str(diff_file),
|
||
"instruction": str(instruction or ""),
|
||
"input_gene_count": len(clean_submitted),
|
||
"source_gene_count": len(original_ids),
|
||
"kept_count": len(kept_ids),
|
||
"deleted_count": len(deleted_ids),
|
||
"added_count": len(added_ids),
|
||
"parameters": {
|
||
"ppi_species": ppi_species,
|
||
"ppi_score": int(ppi_score),
|
||
},
|
||
"artifacts": {
|
||
"reset_input": {
|
||
"artifact": "reset_input",
|
||
"path": str(input_file),
|
||
"type": "table",
|
||
"format": "id",
|
||
"size": input_file.stat().st_size,
|
||
"sha256": _sha256_file(input_file),
|
||
},
|
||
"reset_diff": {
|
||
"artifact": "reset_diff",
|
||
"path": str(diff_file),
|
||
"type": "table",
|
||
"format": "tsv",
|
||
"size": diff_file.stat().st_size,
|
||
"sha256": _sha256_file(diff_file),
|
||
},
|
||
},
|
||
}
|
||
manifest_file.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return {
|
||
"reset_id": reset_id,
|
||
"reset_dir": reset_dir,
|
||
"input_file": input_file,
|
||
"diff_file": diff_file,
|
||
"manifest_file": manifest_file,
|
||
"gene_ids": clean_submitted,
|
||
"deleted_ids": deleted_ids,
|
||
"added_ids": added_ids,
|
||
}
|
||
|
||
|
||
def _record_reset_artifact(reset_dir: Path, reset_id: str, path: Path, artifact: str, artifact_type: str, artifact_format: str) -> Dict[str, Any]:
|
||
return {
|
||
"artifact": artifact,
|
||
"path": f"final/ppi_reset/{reset_id}/{path.name}",
|
||
"full_path": str(path.absolute()),
|
||
"type": artifact_type,
|
||
"format": artifact_format,
|
||
"size": path.stat().st_size,
|
||
"sha256": _sha256_file(path),
|
||
}
|
||
|
||
|
||
def run_ppi_reset_analysis(
|
||
job: JobInfo,
|
||
prefix: str,
|
||
source_gene_id_file: Path,
|
||
submitted_gene_ids: List[str],
|
||
instruction: str = "",
|
||
ppi_species: str = "auto",
|
||
ppi_score: int = 200,
|
||
) -> Dict[str, Any]:
|
||
"""Run a derived PPI reset analysis into final/ppi_reset/reset_###."""
|
||
ws = Path(job.workspace_dir)
|
||
final_dir = ws / "final"
|
||
logs_dir = Path(job.logs_dir)
|
||
final_dir.mkdir(parents=True, exist_ok=True)
|
||
logs_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
prepared = prepare_ppi_reset_inputs(
|
||
final_dir=final_dir,
|
||
prefix=prefix,
|
||
source_gene_id_file=source_gene_id_file,
|
||
submitted_gene_ids=submitted_gene_ids,
|
||
instruction=instruction,
|
||
ppi_species=ppi_species,
|
||
ppi_score=ppi_score,
|
||
)
|
||
reset_id = str(prepared["reset_id"])
|
||
reset_dir = Path(prepared["reset_dir"])
|
||
input_file = Path(prepared["input_file"])
|
||
skill_output_dir = reset_dir / "skill_output"
|
||
skill_output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
skill_script = Path("skills/ppi/run.py")
|
||
if not skill_script.exists():
|
||
raise FileNotFoundError("skills/ppi/run.py not found")
|
||
|
||
cmd = [
|
||
sys.executable,
|
||
str(skill_script),
|
||
"--gene-ids",
|
||
str(input_file),
|
||
"--species",
|
||
ppi_species,
|
||
"--prefix",
|
||
prefix,
|
||
"--output",
|
||
str(skill_output_dir),
|
||
"--score",
|
||
str(int(ppi_score)),
|
||
]
|
||
log_file = logs_dir / f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{prefix}.ppi_reset.{reset_id}.log"
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
manifest_file = Path(prepared["manifest_file"])
|
||
try:
|
||
manifest_data = json.loads(manifest_file.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
manifest_data = {}
|
||
manifest_data["status"] = "success" if code == 0 else "failed"
|
||
manifest_data["log_file"] = str(log_file)
|
||
manifest_data.setdefault("parameters", {})["ppi_species"] = ppi_species
|
||
manifest_data.setdefault("parameters", {})["ppi_score"] = int(ppi_score)
|
||
artifacts = manifest_data.setdefault("artifacts", {})
|
||
|
||
if code == 0:
|
||
mappings = {
|
||
f"{prefix}_network.tsv": (f"{prefix}.ppi_network.tsv", "network", "table", "tsv"),
|
||
f"{prefix}_report.md": (f"{prefix}.ppi_report.md", "report", "markdown", "markdown"),
|
||
f"{prefix}_url.txt": (f"{prefix}.ppi_url.txt", "string_url", "link", "txt"),
|
||
f"{prefix}_id_resolution.tsv": (f"{prefix}.ppi_id_resolution.tsv", "id_resolution", "table", "tsv"),
|
||
f"{prefix}.manifest.json": (f"{prefix}.ppi_skill.manifest.json", "skill_manifest", "json", "json"),
|
||
}
|
||
for src_name, (dst_name, artifact, artifact_type, artifact_format) in mappings.items():
|
||
src = skill_output_dir / src_name
|
||
if not src.exists():
|
||
continue
|
||
dst = reset_dir / dst_name
|
||
shutil.copy2(src, dst)
|
||
artifacts[artifact] = _record_reset_artifact(reset_dir, reset_id, dst, artifact, artifact_type, artifact_format)
|
||
|
||
url_file = reset_dir / f"{prefix}.ppi_url.txt"
|
||
string_url = ""
|
||
if url_file.exists():
|
||
for line in url_file.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||
match = re.search(r"(https?://\S+)", line)
|
||
if match:
|
||
string_url = match.group(1)
|
||
break
|
||
if string_url:
|
||
manifest_data["string_url"] = string_url
|
||
image_file = download_string_network_image(string_url, reset_dir / f"{prefix}.ppi_network.string.png")
|
||
if image_file:
|
||
fmt = image_file.suffix.lower().lstrip(".") or "png"
|
||
artifacts["string_network_image"] = _record_reset_artifact(reset_dir, reset_id, image_file, "string_network_image", fmt, fmt)
|
||
|
||
fold2_file = create_alphafold2_links(
|
||
id_resolution_file=reset_dir / f"{prefix}.ppi_id_resolution.tsv",
|
||
network_file=reset_dir / f"{prefix}.ppi_network.tsv",
|
||
output_file=reset_dir / f"{prefix}.ppi_alphafold2_links.tsv",
|
||
)
|
||
if fold2_file:
|
||
fold2_record = _record_reset_artifact(reset_dir, reset_id, fold2_file, "alphafold2_links", "table", "tsv")
|
||
fold2_record["source"] = "AlphaFold DB links derived from STRING PPI reset outputs"
|
||
fold2_record["structure_type"] = "AlphaFold DB/Fold2 single protein"
|
||
artifacts["alphafold2_links"] = fold2_record
|
||
|
||
manifest_file.write_text(json.dumps(manifest_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
return {
|
||
"success": code == 0,
|
||
"exit_code": code,
|
||
"reset_id": reset_id,
|
||
"reset_dir": reset_dir,
|
||
"manifest_file": manifest_file,
|
||
"log_file": log_file,
|
||
}
|
||
|
||
|
||
def _get_free_mem_gb() -> float:
|
||
# 不依赖 psutil：直接读 /proc/meminfo
|
||
try:
|
||
meminfo = Path("/proc/meminfo").read_text()
|
||
m = re.search(r"MemAvailable:\s+(\d+)\s+kB", meminfo)
|
||
if not m:
|
||
return 0.0
|
||
kb = float(m.group(1))
|
||
return kb / 1024 / 1024
|
||
except Exception:
|
||
return 0.0
|
||
|
||
|
||
def can_start_new_task(min_free_gb: float = 20.0) -> Tuple[bool, str]:
|
||
free_gb = _get_free_mem_gb()
|
||
if free_gb <= 0:
|
||
return True, "⚠️ 无法读取 MemAvailable，允许启动（请自行关注内存）"
|
||
if free_gb < min_free_gb:
|
||
return False, f"❌ 当前可用内存约 {free_gb:.1f} GB < {min_free_gb} GB，建议稍后再运行或减少并发"
|
||
return True, f"✅ 当前可用内存约 {free_gb:.1f} GB，允许启动任务"
|
||
|
||
|
||
def run_cmd_logged(cmd: List[str], log_path: Path, cwd: Optional[Path] = None) -> int:
|
||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||
with log_path.open("w", encoding="utf-8") as lg:
|
||
lg.write("CMD: " + " ".join(cmd) + "\n")
|
||
lg.write("CWD: " + (str(cwd) if cwd else os.getcwd()) + "\n")
|
||
lg.write("START: " + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n\n")
|
||
lg.flush()
|
||
|
||
# 设置环境变量，包含 conda 环境的 PATH
|
||
env = os.environ.copy()
|
||
conda_bin = "/home/xzq/miniconda3/envs/test4py311/bin"
|
||
conda_lib = "/home/xzq/miniconda3/envs/test4py311/lib"
|
||
if conda_bin not in env.get("PATH", ""):
|
||
env["PATH"] = conda_bin + ":" + env.get("PATH", "")
|
||
if conda_lib not in env.get("LD_LIBRARY_PATH", ""):
|
||
env["LD_LIBRARY_PATH"] = conda_lib + ":" + env.get("LD_LIBRARY_PATH", "")
|
||
project_root = str(PROJECT_ROOT)
|
||
pythonpath_parts = [p for p in env.get("PYTHONPATH", "").split(os.pathsep) if p]
|
||
if project_root not in pythonpath_parts:
|
||
env["PYTHONPATH"] = os.pathsep.join([project_root] + pythonpath_parts)
|
||
|
||
p = subprocess.Popen(cmd, cwd=str(cwd) if cwd else None, stdout=lg, stderr=subprocess.STDOUT, text=True, env=env)
|
||
return p.wait()
|
||
|
||
|
||
def _species_arg_list(value) -> List[str]:
|
||
if not value:
|
||
return []
|
||
if isinstance(value, str):
|
||
return [item.strip() for item in value.split(',') if item.strip()]
|
||
return [str(item).strip() for item in value if str(item).strip()]
|
||
|
||
|
||
def _workspace_url_to_path(value: str, workspace: Path) -> Optional[Path]:
|
||
raw = str(value or "").strip()
|
||
if not raw:
|
||
return None
|
||
if raw.startswith("/workspace/"):
|
||
parts = raw.strip("/").split("/", 2)
|
||
if len(parts) == 3:
|
||
return (WORKSPACE_ROOT / parts[1] / parts[2]).resolve()
|
||
path = Path(raw).expanduser()
|
||
if path.is_absolute():
|
||
return path.resolve()
|
||
if raw.startswith("final/"):
|
||
return (workspace / raw).resolve()
|
||
return (workspace / raw).resolve()
|
||
|
||
|
||
def _resolve_msa_file_for_region_plot(workspace: Path, prefix: str, msa_file: str = "", source_image: str = "") -> Optional[Path]:
|
||
explicit = _workspace_url_to_path(msa_file, workspace)
|
||
if explicit and explicit.exists() and explicit.is_file():
|
||
return explicit
|
||
|
||
image_path = _workspace_url_to_path(source_image, workspace)
|
||
final_dir = workspace / "final"
|
||
manifest_candidates: List[Path] = []
|
||
if image_path and image_path.exists():
|
||
stem = image_path.name
|
||
for suffix in ("_msa.png", "_msa.pdf", ".png", ".pdf", ".jpg", ".jpeg", ".svg"):
|
||
if stem.lower().endswith(suffix):
|
||
manifest_candidates.append(image_path.with_name(stem[: -len(suffix)] + ".manifest.json"))
|
||
if final_dir.exists():
|
||
manifest_candidates.extend(sorted(final_dir.glob(f"{prefix}.msa_region_*.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True))
|
||
|
||
if image_path and image_path.exists():
|
||
image_name = image_path.name
|
||
image_stem = image_name
|
||
for suffix in (".png", ".jpg", ".jpeg", ".svg", ".pdf"):
|
||
if image_stem.lower().endswith(suffix):
|
||
image_stem = image_stem[: -len(suffix)]
|
||
break
|
||
msa_stems = []
|
||
msa_plot_match = re.match(r"^(.+?)\.msa_plot(?:-\d+)?$", image_stem, flags=re.IGNORECASE)
|
||
if msa_plot_match:
|
||
msa_stems.append(msa_plot_match.group(1))
|
||
region_match = re.match(r"^(.+?)\.msa_region_\d+_\d+(?:_msa)?$", image_stem, flags=re.IGNORECASE)
|
||
if region_match:
|
||
msa_stems.append(region_match.group(1))
|
||
for stem_prefix in msa_stems:
|
||
candidate = final_dir / f"{stem_prefix}.pep.msa.fa"
|
||
if candidate.exists() and candidate.is_file():
|
||
return candidate.resolve()
|
||
|
||
seen = set()
|
||
for manifest_path in manifest_candidates:
|
||
if manifest_path in seen or not manifest_path.exists():
|
||
continue
|
||
seen.add(manifest_path)
|
||
try:
|
||
data = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
continue
|
||
artifacts = data.get("artifacts") or {}
|
||
source_msa = artifacts.get("source_msa") or artifacts.get("msa") or ""
|
||
if isinstance(source_msa, dict):
|
||
source_msa = source_msa.get("full_path") or source_msa.get("path") or ""
|
||
resolved = _workspace_url_to_path(str(source_msa), workspace)
|
||
if resolved and resolved.exists() and resolved.is_file():
|
||
return resolved
|
||
|
||
fallback = final_dir / f"{prefix}.pep.msa.fa"
|
||
return fallback.resolve() if fallback.exists() else None
|
||
|
||
|
||
def _tree_prefix_from_image_name(name: str) -> str:
|
||
stem = str(name or "")
|
||
for suffix in (".png", ".jpg", ".jpeg", ".svg", ".pdf"):
|
||
if stem.lower().endswith(suffix):
|
||
stem = stem[: -len(suffix)]
|
||
break
|
||
if stem.lower().endswith(".circular_tree"):
|
||
stem = stem[: -len(".circular_tree")]
|
||
lower = stem.lower()
|
||
replot_match = re.match(r"^(.+?)\.tree_replot_[^.]+$", stem, flags=re.IGNORECASE)
|
||
if replot_match:
|
||
stem = replot_match.group(1) + ".tree"
|
||
lower = stem.lower()
|
||
if lower.endswith(".motif.tree_motif"):
|
||
stem = stem[: -len(".motif.tree_motif")]
|
||
lower = stem.lower()
|
||
if lower.endswith(".combo"):
|
||
stem = stem[: -len(".combo")]
|
||
lower = stem.lower()
|
||
return stem
|
||
|
||
|
||
def _resolve_tree_file_for_replot(workspace: Path, prefix: str, tree_file: str = "", source_image: str = "") -> Optional[Path]:
|
||
explicit = _workspace_url_to_path(tree_file, workspace)
|
||
if explicit and explicit.exists() and explicit.is_file():
|
||
return explicit
|
||
final_dir = workspace / "final"
|
||
image_path = _workspace_url_to_path(source_image, workspace)
|
||
if image_path and image_path.exists():
|
||
image_stem = _tree_prefix_from_image_name(image_path.name)
|
||
candidates = [
|
||
final_dir / f"{image_stem}.nwk",
|
||
final_dir / f"{image_stem}.tree.nwk" if not image_stem.endswith(".tree") else final_dir / f"{image_stem}.nwk",
|
||
]
|
||
if image_stem.endswith(".tree"):
|
||
base_stem = image_stem[: -len(".tree")]
|
||
candidates.append(final_dir / f"{base_stem}.tree.nwk")
|
||
else:
|
||
candidates.append(final_dir / f"{image_stem}.tree.nwk")
|
||
if ".reference_classification" not in image_stem:
|
||
# Combo and tree-motif images may use a shorter public prefix; prefer
|
||
# exact tree files first, then any scoped tree derived from that prefix.
|
||
candidates.extend(sorted(final_dir.glob(f"{image_stem}*.tree.nwk"), key=lambda p: p.stat().st_mtime, reverse=True))
|
||
for candidate in candidates:
|
||
if candidate.exists() and candidate.is_file():
|
||
return candidate.resolve()
|
||
fallback = final_dir / f"{prefix}.tree.nwk"
|
||
if fallback.exists():
|
||
return fallback.resolve()
|
||
matches = sorted(final_dir.glob(f"{prefix}*.tree.nwk"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||
return matches[0].resolve() if matches else None
|
||
|
||
def normalize_combo_panels(raw_panels) -> List[str]:
|
||
allowed = ["motif", "structure", "domain"]
|
||
aliases = {
|
||
"gene_structure": "structure",
|
||
"gene-structure": "structure",
|
||
"genestructure": "structure",
|
||
"domains": "domain",
|
||
"motifs": "motif",
|
||
"all": "all",
|
||
}
|
||
if isinstance(raw_panels, str):
|
||
items = raw_panels.replace(";", ",").split(",")
|
||
elif isinstance(raw_panels, (list, tuple, set)):
|
||
items = list(raw_panels)
|
||
else:
|
||
items = []
|
||
selected: List[str] = []
|
||
for token in items:
|
||
key = str(token or "").strip().lower().replace(" ", "_")
|
||
if not key:
|
||
continue
|
||
key = aliases.get(key, key)
|
||
if key == "all":
|
||
return allowed
|
||
if key not in allowed:
|
||
raise ValueError(f"Unsupported combo panel: {token}. Allowed: {', '.join(allowed)}")
|
||
if key not in selected:
|
||
selected.append(key)
|
||
selected = selected or allowed
|
||
if len(selected) < 2:
|
||
raise ValueError("Combo redraw requires at least two data panels; single-panel views already exist in motif/structure/domain stages.")
|
||
return selected
|
||
|
||
|
||
def _manifest_combo_output_label(manifest: Dict[str, Any]) -> str:
|
||
params = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {}
|
||
return str(params.get("output_label") or manifest.get("output_label") or "").strip()
|
||
|
||
|
||
def _is_standard_combo_manifest(path: Path) -> bool:
|
||
try:
|
||
manifest = json.loads(path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return True
|
||
if _manifest_combo_output_label(manifest):
|
||
return False
|
||
artifacts = manifest.get("artifacts") if isinstance(manifest.get("artifacts"), dict) else {}
|
||
for key in artifacts:
|
||
if str(key or "").startswith("combo_replot"):
|
||
return False
|
||
return True
|
||
|
||
|
||
def infer_completed_combo_panels(
|
||
final_dir: Path,
|
||
prefix: str,
|
||
scope_data: Optional[Dict[str, Any]] = None,
|
||
analysis_route: str = "",
|
||
member_set_id: str = "",
|
||
) -> List[str]:
|
||
completed: List[str] = []
|
||
for panel in ["motif", "structure", "domain"]:
|
||
manifest_path = first_existing_stage_manifest(
|
||
final_dir,
|
||
panel,
|
||
prefix,
|
||
scope_data or {},
|
||
analysis_route,
|
||
member_set_id,
|
||
)
|
||
if manifest_path:
|
||
completed.append(panel)
|
||
return completed if len(completed) >= 2 else []
|
||
|
||
|
||
def _resolve_rnaseq_sample_config(
|
||
input_dir: Path,
|
||
workspace_dir: Path,
|
||
requested: str = "",
|
||
) -> Optional[Path]:
|
||
"""Resolve the RNA-seq sample table without inferring read roles from filenames."""
|
||
candidates: List[Path] = []
|
||
requested_text = str(requested or "").strip()
|
||
if requested_text:
|
||
requested_path = Path(requested_text).expanduser()
|
||
if requested_path.is_absolute():
|
||
candidates.append(requested_path)
|
||
else:
|
||
candidates.extend([input_dir / requested_path, workspace_dir / requested_path, requested_path])
|
||
candidates.extend([input_dir / "samples.yaml", workspace_dir / "final" / "samples.yaml"])
|
||
|
||
for candidate in candidates:
|
||
if candidate.is_file():
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def run_stage(
|
||
stage: str,
|
||
job: JobInfo,
|
||
prefix: str,
|
||
cpu: int = 8,
|
||
required_pfams: Optional[str] = None,
|
||
ref_species: Optional[List[str]] = None,
|
||
retention_days: int = 7,
|
||
# 高级参数
|
||
id_evalue: float = 0.001,
|
||
id_coverage: float = 0.3,
|
||
phy_mafft: str = None,
|
||
phy_iqtree: str = None,
|
||
phy_tree_layout: str = "both",
|
||
msa_start_col: int = 0,
|
||
msa_end_col: int = 0,
|
||
msa_auto_conserved: bool = False,
|
||
msa_window_len: int = 60,
|
||
msa_min_nongap: float = 0.6,
|
||
msa_file: str = "",
|
||
msa_source_image: str = "",
|
||
msa_reference_id: str = "",
|
||
msa_reference_start_residue: int = 0,
|
||
msa_reference_end_residue: int = 0,
|
||
tree_file: str = "",
|
||
tree_source_image: str = "",
|
||
tree_source_stage: str = "",
|
||
tree_output_label: str = "",
|
||
tree_label_offset: float = 0.4,
|
||
tree_tip_label_size: float = 3.0,
|
||
tree_branch_color: str = "black",
|
||
tree_branch_width: float = 0.5,
|
||
tree_show_bootstrap: bool = True,
|
||
tree_bootstrap_size: float = 2.0,
|
||
tree_tip_point_size: float = 2.0,
|
||
tree_target_shape: str = "triangle",
|
||
tree_reference_shape: str = "circle",
|
||
tree_marker_mode: str = "auto",
|
||
tree_subfamily_arc_enabled: bool = True,
|
||
tree_subfamily_arc_gap: float = 1.2,
|
||
tree_subfamily_arc_label_gap: float = 0.6,
|
||
tree_subfamily_arc_label_size: float = 0.0,
|
||
synteny_source_image: str = "",
|
||
synteny_source_stage: str = "synteny",
|
||
synteny_output_label: str = "synteny_replot_circos",
|
||
synteny_gene_label_size: float = 38.0,
|
||
synteny_gene_track_r0: float = 1.03,
|
||
synteny_gene_track_r1: float = 1.30,
|
||
synteny_chrom_label_radius: str = "1r - 60p",
|
||
synteny_chrom_label_size: float = 40.0,
|
||
synteny_ideogram_thickness: float = 90.0,
|
||
tfbs_upstream_length: int = 2000,
|
||
tfbs_motif_db_mode: str = "auto",
|
||
tfbs_fimo_pvalue: float = 1e-4,
|
||
tfbs_fimo_qvalue: float = 0.05,
|
||
tfbs_background_mode: str = "none",
|
||
tfbs_max_nearest_rank: str = "order",
|
||
tfbs_top_motifs: int = 30,
|
||
tfbs_scan_both_strands: bool = True,
|
||
combo_panels: Optional[List[str]] = None,
|
||
combo_output_label: str = "",
|
||
reference_subfamily_file: str = "",
|
||
subfamily_confirm_file: str = "",
|
||
reference_target_prefix: str = "",
|
||
reference_prefix: str = "At",
|
||
reference_overwrite_gene_subfamily: bool = False,
|
||
motif_nmotifs: int = 10,
|
||
motif_minw: int = 6,
|
||
motif_maxw: int = 200,
|
||
dom_evalue: float = 1e-5,
|
||
dom_coverage: float = None,
|
||
blast_evalue: float = 1e-5,
|
||
blast_identity: float = 30.0,
|
||
id_transform: str = "none",
|
||
geo_id: str = "",
|
||
# RNA-seq 参数
|
||
rnaseq_mode: str = "upload",
|
||
species_name: str = "",
|
||
top_pathways: int = 20,
|
||
enrich_background_source: str = "auto",
|
||
enrich_species: str = "",
|
||
enrich_key_type: str = "auto",
|
||
enrich_type: str = "go",
|
||
kegg_organism: str = "",
|
||
rnaseq_foreground_mode: str = "auto",
|
||
annotation_applications: Optional[List[str]] = None,
|
||
rnaseq_sample_config: str = "",
|
||
rnaseq_sample_parallel_jobs: int = 0,
|
||
rnaseq_threads_per_sample: int = 0,
|
||
rnaseq_project_max_cpu: int = 0,
|
||
rnaseq_resource_decision: Optional[Dict[str, Any]] = None,
|
||
rnaseq_qc_tool: str = "fastp",
|
||
rnaseq_fastp_bin: str = "tools/fastp",
|
||
rnaseq_fastp_detect_adapter_for_pe: bool = True,
|
||
rnaseq_fastp_cut_front: bool = True,
|
||
rnaseq_fastp_cut_tail: bool = True,
|
||
rnaseq_fastp_cut_window_size: int = 4,
|
||
rnaseq_fastp_cut_mean_quality: int = 20,
|
||
rnaseq_fastp_qualified_quality_phred: int = 20,
|
||
rnaseq_fastp_unqualified_percent_limit: int = 30,
|
||
rnaseq_fastp_n_base_limit: int = 5,
|
||
rnaseq_fastp_length_required: int = 30,
|
||
rnaseq_splice_site_source: str = "auto",
|
||
rnaseq_known_splice_site: str = "",
|
||
rnaseq_quant_mode: str = "dual",
|
||
rnaseq_abundance_engine: str = "auto",
|
||
rnaseq_transcriptome_fasta: str = "",
|
||
rnaseq_tx2gene: str = "",
|
||
rnaseq_salmon_libtype: str = "A",
|
||
rnaseq_tximport_counts_from_abundance: str = "no",
|
||
rnaseq_kallisto_bias: bool = False,
|
||
rnaseq_de_padj: float = 0.05,
|
||
rnaseq_de_module: str = "",
|
||
rnaseq_de_log2fc: float = 1.0,
|
||
enrich_pvalue_cutoff: float = 0.05,
|
||
enrich_qvalue_cutoff: float = 0.05,
|
||
go_obo: str = "data/go/go-basic.obo",
|
||
kegg_level: str = "pathway",
|
||
kegg_filter_scope: str = "plant",
|
||
kegg_highlight_mode: str = "auto",
|
||
kegg_no_network: bool = False,
|
||
kegg_skip_pathview: bool = False,
|
||
enrich_plot_type: str = "r_plot",
|
||
rnaseq_count_matrix: str = "",
|
||
rnaseq_expression_matrix: str = "",
|
||
rnaseq_sample_table: str = "",
|
||
fgsea_comparison: str = "",
|
||
fgsea_min_size: int = 15,
|
||
fgsea_max_size: int = 500,
|
||
fgsea_nperm: int = 10000,
|
||
fgsea_seed: int = 1,
|
||
wgcna_min_mean: float = 1.0,
|
||
wgcna_max_genes: int = 8000,
|
||
wgcna_min_module_size: int = 30,
|
||
wgcna_merge_cut_height: float = 0.25,
|
||
wgcna_powers: str = "1,2,3,4,5,6,7,8,9,10,12,14,16,18,20",
|
||
candidate_max_deg_proteins: int = 500,
|
||
candidate_run_interproscan: str = "auto",
|
||
# NCBI fetch 参数
|
||
ncbi_query: str = "",
|
||
ncbi_gene_ids: str = "",
|
||
ncbi_mode: str = "gene",
|
||
ncbi_id_type: str = "auto",
|
||
ncbi_taxid: str = "",
|
||
ncbi_species: str = "",
|
||
ncbi_seq_types: str = "both",
|
||
ncbi_transcript_policy: str = "refseq_preferred",
|
||
ncbi_allow_family_download: bool = False,
|
||
ncbi_email: str = "",
|
||
# Local sequence extraction parameters
|
||
sequence_extract_fasta: str = "",
|
||
sequence_extract_ids_file: str = "",
|
||
sequence_extract_ids: str = "",
|
||
sequence_extract_seq_type: str = "auto",
|
||
sequence_extract_rename_mode: str = "keep",
|
||
sequence_extract_clean_headers: bool = False,
|
||
# PPI 参数
|
||
ppi_species: str = "auto",
|
||
normalization_species_label: str = "",
|
||
ppi_reference_taxon_id: str = "",
|
||
ppi_reference_storage_key: str = "",
|
||
ppi_reference_selection_mode: str = "",
|
||
ppi_reference_user_confirmed: bool = False,
|
||
dependency_owner: str = "",
|
||
ppi_score: int = 200,
|
||
# Genome Prepare 参数
|
||
genome_prepare_type: str = "auto",
|
||
# KaKs 参数
|
||
kaks_neutral_rate: float = 7.08e-9,
|
||
**kwargs,
|
||
) -> Tuple[str, Optional[Path]]:
|
||
# 函数入口debug
|
||
print(f"[FUNC ENTRY DEBUG] stage={stage}, rnaseq_mode={rnaseq_mode}, species_name={species_name}")
|
||
ok, msg = can_start_new_task()
|
||
if not ok:
|
||
return msg, None
|
||
|
||
ws = Path(job.workspace_dir)
|
||
in_dir = Path(job.input_dir)
|
||
logs = Path(job.logs_dir)
|
||
final_dir = ws / "final" # 定义 final_dir 供所有阶段使用
|
||
|
||
valid_annotation_apps = {"Pfam", "PANTHER", "SMART", "ProSiteProfiles", "ProSitePatterns"}
|
||
selected_annotation_apps = annotation_applications or kwargs.get("annotationApplications") or ["Pfam", "ProSiteProfiles"]
|
||
if isinstance(selected_annotation_apps, str):
|
||
selected_annotation_apps = [x.strip() for x in selected_annotation_apps.split(",") if x.strip()]
|
||
selected_annotation_apps = [x for x in selected_annotation_apps if x in valid_annotation_apps]
|
||
if not selected_annotation_apps:
|
||
selected_annotation_apps = ["Pfam", "ProSiteProfiles"]
|
||
annotation_applications_arg = ",".join(selected_annotation_apps)
|
||
|
||
target_species = kwargs.get("target_species")
|
||
interspecies_ref_species = kwargs.get("interspecies_ref_species")
|
||
scope_file = str(kwargs.get("scope_file") or kwargs.get("scopeFile") or "").strip()
|
||
analysis_route = str(kwargs.get("analysis_route") or kwargs.get("analysisRoute") or "").strip()
|
||
analysis_mode = str(kwargs.get("analysis_mode") or kwargs.get("analysisMode") or "").strip()
|
||
member_set_id = str(kwargs.get("member_set_id") or kwargs.get("memberSetId") or "").strip()
|
||
combo_output_label = str(combo_output_label or kwargs.get("comboOutputLabel") or kwargs.get("combo_output_label") or "").strip()
|
||
subfamily_label_source = str(kwargs.get("subfamily_label_source") or kwargs.get("subfamilyLabelSource") or "").strip().lower()
|
||
requires_confirmed_subfamily = _truthy(kwargs.get("requires_confirmed_subfamily") or kwargs.get("requiresConfirmedSubfamily"))
|
||
if requires_confirmed_subfamily and subfamily_label_source in {"", "auto"}:
|
||
subfamily_label_source = "confirmed_or_uploaded_subfamily"
|
||
interspecies_blast_engine = str(kwargs.get("interspecies_blast_engine") or "diamond").strip().lower()
|
||
if interspecies_blast_engine not in {"diamond", "blastp"}:
|
||
interspecies_blast_engine = "diamond"
|
||
interspecies_plot_mode = str(kwargs.get("interspecies_plot_mode") or kwargs.get("interspeciesPlotMode") or "both").strip().lower()
|
||
if interspecies_plot_mode not in {"multi", "pairwise", "both"}:
|
||
interspecies_plot_mode = "both"
|
||
|
||
resolved_prefix = resolve_effective_prefix_for_job(job, prefix, stage)
|
||
if resolved_prefix != _clean_prefix(prefix):
|
||
print(f"[INFO] Resolved empty prefix for {stage}: {resolved_prefix or '<empty>'}")
|
||
prefix = resolved_prefix
|
||
|
||
# 初始化可复现性管理器
|
||
repro_manager = None
|
||
try:
|
||
repro_scope_id = ""
|
||
if scope_file:
|
||
try:
|
||
scope_path_for_repro = Path(scope_file)
|
||
if not scope_path_for_repro.is_absolute():
|
||
scope_path_for_repro = Path(__file__).resolve().parents[2] / scope_path_for_repro
|
||
if scope_path_for_repro.exists():
|
||
repro_scope_id = str(read_scope(scope_path_for_repro).get("scope_id") or "")
|
||
except Exception:
|
||
repro_scope_id = ""
|
||
repro_manager = ReproducibilityManager(
|
||
job_id=job.job_id,
|
||
prefix=prefix,
|
||
workspace_root=str(WORKSPACE_ROOT),
|
||
scope_file=scope_file,
|
||
scope_id=repro_scope_id,
|
||
)
|
||
except Exception as e:
|
||
print(f"[WARN] ReproducibilityManager init failed: {e}")
|
||
|
||
manifest = find_latest_manifest(ws)
|
||
if not manifest:
|
||
fasta_files = []
|
||
if in_dir.exists():
|
||
for p in sorted(in_dir.iterdir()):
|
||
if p.is_file() and detect_role(p) == "fasta":
|
||
fasta_files.append(p)
|
||
if fasta_files and stage_allows_minimal_manifest(stage):
|
||
manifest = ensure_min_manifest(ws, prefix=prefix, final_fa=fasta_files[0])
|
||
elif stage in {"expression", "genome_prepare"} or stage.startswith("rnaseq"):
|
||
manifest = None
|
||
elif not stage_allows_minimal_manifest(stage):
|
||
manifest = None
|
||
else:
|
||
return "No fasta found to create minimal manifest for this stage", None
|
||
|
||
log_file = logs / f"{_now_tag()}_{stage}.log"
|
||
|
||
if stage == "promoter_plot_revise":
|
||
elements = str(kwargs.get("promoter_elements") or kwargs.get("promoterElements") or "").strip()
|
||
categories = str(kwargs.get("promoter_categories") or kwargs.get("promoterCategories") or "").strip()
|
||
match_mode = str(kwargs.get("promoter_match_mode") or kwargs.get("promoterMatchMode") or "exact").strip().lower()
|
||
plot_modes = str(kwargs.get("promoter_plot_modes") or kwargs.get("promoterPlotModes") or "position_distribution,heatmap").strip()
|
||
color_by = str(kwargs.get("promoter_color_by") or kwargs.get("promoterColorBy") or "auto").strip().lower()
|
||
output_label = str(kwargs.get("promoter_output_label") or kwargs.get("promoterOutputLabel") or "").strip()
|
||
top_n = str(kwargs.get("promoter_top_n") or kwargs.get("promoterTopN") or "30").strip()
|
||
promoter_file = str(kwargs.get("promoter_file") or kwargs.get("promoterFile") or "").strip()
|
||
promoter_tree_file = str(kwargs.get("promoter_tree_file") or kwargs.get("promoterTreeFile") or tree_file or "").strip()
|
||
promoter_subfamily_file = str(kwargs.get("promoter_subfamily_file") or kwargs.get("promoterSubfamilyFile") or "").strip()
|
||
if not elements and not categories:
|
||
return "❌ 启动子自定义绘图需要至少选择一个元件或类别。", manifest
|
||
if match_mode not in {"exact", "contains", "regex"}:
|
||
return f"❌ 启动子自定义绘图 match_mode 无效: {match_mode}", manifest
|
||
if color_by not in {"auto", "element", "category"}:
|
||
return f"❌ 启动子自定义绘图 color_by 无效: {color_by}", manifest
|
||
script = Path("apps/cli/run_promoter_plot_revise.py")
|
||
if not script.exists():
|
||
return "❌ Promoter custom plot script not found: apps/cli/run_promoter_plot_revise.py", manifest
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix,
|
||
"--elements", elements,
|
||
"--categories", categories,
|
||
"--match-mode", match_mode,
|
||
"--plot-modes", plot_modes,
|
||
"--top-n", top_n,
|
||
"--color-by", color_by,
|
||
]
|
||
if output_label:
|
||
cmd.extend(["--output-label", output_label])
|
||
if promoter_file:
|
||
cmd.extend(["--promoter-file", promoter_file])
|
||
if promoter_tree_file:
|
||
cmd.extend(["--tree", promoter_tree_file])
|
||
if promoter_subfamily_file:
|
||
cmd.extend(["--subfamily", promoter_subfamily_file])
|
||
if scope_file:
|
||
cmd.extend(["--scope-file", scope_file])
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
promoter_replot_manifest = final_dir / f"{prefix}.promoter_plot_revise.manifest.json"
|
||
if code == 0 and promoter_replot_manifest.exists():
|
||
return f"✅ 启动子自定义绘图完成。Log: {log_file}", promoter_replot_manifest
|
||
if promoter_replot_manifest.exists():
|
||
return f"❌ 启动子自定义绘图失败 (exit code: {code}). Log: {log_file}", promoter_replot_manifest
|
||
return f"❌ 启动子自定义绘图失败 (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
if stage == "ncbi_fetch_sequences":
|
||
ncbi_defaults = _ncbi_species_defaults_from_prefix(prefix)
|
||
if not ncbi_species:
|
||
ncbi_species = ncbi_defaults.get("species", "")
|
||
if not ncbi_taxid:
|
||
ncbi_taxid = ncbi_defaults.get("taxid", "")
|
||
ncbi_cmd = [
|
||
"/home/xzq/miniconda3/envs/test4py311/bin/python",
|
||
"apps/cli/run_ncbi_fetch_sequences.py",
|
||
"--workspace", str(ws),
|
||
"-p", prefix or "NCBI.fetch",
|
||
"-j", job.job_id,
|
||
"--mode", str(ncbi_mode or "gene"),
|
||
"--id-type", str(ncbi_id_type or "auto"),
|
||
"--seq-types", str(ncbi_seq_types or "both"),
|
||
"--transcript-policy", str(ncbi_transcript_policy or "refseq_preferred"),
|
||
]
|
||
if ncbi_taxid:
|
||
ncbi_cmd.extend(["--taxid", str(ncbi_taxid)])
|
||
if ncbi_species:
|
||
ncbi_cmd.extend(["--species", str(ncbi_species)])
|
||
if ncbi_query:
|
||
ncbi_cmd.extend(["--query", str(ncbi_query)])
|
||
resolved_ids_file = None
|
||
if ncbi_gene_ids:
|
||
ncbi_gene_ids_text = str(ncbi_gene_ids).strip()
|
||
provided_gene_id_file = Path(ncbi_gene_ids_text).expanduser()
|
||
if not provided_gene_id_file.is_absolute():
|
||
for candidate in (Path(job.input_dir) / provided_gene_id_file, ws / provided_gene_id_file):
|
||
if candidate.exists() and candidate.is_file():
|
||
provided_gene_id_file = candidate
|
||
break
|
||
if provided_gene_id_file.exists() and provided_gene_id_file.is_file():
|
||
resolved_ids_file = provided_gene_id_file
|
||
else:
|
||
gene_id_file = ws / "ncbi_fetch_sequences" / "analysis" / f"{prefix or 'NCBI.fetch'}.input_ids.txt"
|
||
gene_id_file.parent.mkdir(parents=True, exist_ok=True)
|
||
items = [item.strip() for item in re.split(r"[\s,;]+", ncbi_gene_ids_text) if item.strip()]
|
||
gene_id_file.write_text("\n".join(items) + "\n", encoding="utf-8")
|
||
resolved_ids_file = gene_id_file
|
||
elif prefix:
|
||
resolved_ids_file = _resolve_job_ids_file(job, prefix)
|
||
if resolved_ids_file:
|
||
ncbi_cmd.extend(["--gene-ids", str(resolved_ids_file)])
|
||
if ncbi_allow_family_download:
|
||
ncbi_cmd.append("--allow-family-download")
|
||
if ncbi_email:
|
||
ncbi_cmd.extend(["--email", str(ncbi_email)])
|
||
print(f"[CMD] {' '.join(ncbi_cmd)}")
|
||
code = run_cmd_logged(ncbi_cmd, log_file, cwd=Path("."))
|
||
ncbi_manifest = final_dir / f"{prefix or 'NCBI.fetch'}.ncbi_fetch.manifest.json"
|
||
if code == 0:
|
||
return "✅ NCBI sequence fetch completed.", ncbi_manifest
|
||
return f"❌ NCBI sequence fetch failed (exit code: {code}). Log: {log_file}", ncbi_manifest if ncbi_manifest.exists() else manifest
|
||
|
||
if stage == "sequence_extract_by_id":
|
||
extract_cmd = [
|
||
"/home/xzq/miniconda3/envs/test4py311/bin/python",
|
||
"apps/cli/run_sequence_extract_by_id.py",
|
||
"--workspace", str(ws),
|
||
"-p", prefix or "sequence_extract",
|
||
"-j", job.job_id,
|
||
"--seq-type", str(sequence_extract_seq_type or "auto"),
|
||
"--rename-mode", str(sequence_extract_rename_mode or "keep"),
|
||
]
|
||
if sequence_extract_clean_headers:
|
||
extract_cmd.append("--clean-all-headers")
|
||
if sequence_extract_fasta:
|
||
for item in re.split(r"[,;，；]+", str(sequence_extract_fasta)):
|
||
item = item.strip()
|
||
if item:
|
||
extract_cmd.extend(["--fasta", item])
|
||
ids_file_text = str(sequence_extract_ids_file or "").strip()
|
||
if ids_file_text:
|
||
ids_path = _resolve_sequence_ids_file(job, ids_file_text)
|
||
if ids_path:
|
||
extract_cmd.extend(["--ids-file", str(ids_path)])
|
||
else:
|
||
# Treat a non-existing ids_file value as inline IDs; this avoids
|
||
# failing when the parser captured a single gene ID into this field.
|
||
sequence_extract_ids = f"{sequence_extract_ids} {ids_file_text}".strip()
|
||
elif prefix:
|
||
resolved_ids_file = _resolve_job_ids_file(job, prefix)
|
||
if resolved_ids_file:
|
||
extract_cmd.extend(["--ids-file", str(resolved_ids_file)])
|
||
if sequence_extract_ids:
|
||
extract_cmd.extend(["--ids", str(sequence_extract_ids)])
|
||
print(f"[CMD] {' '.join(extract_cmd)}")
|
||
code = run_cmd_logged(extract_cmd, log_file, cwd=Path("."))
|
||
extract_manifest = final_dir / f"{prefix or 'sequence_extract'}.sequence_extract.manifest.json"
|
||
if code == 0:
|
||
return "✅ Sequence extraction by ID completed.", extract_manifest
|
||
if extract_manifest.exists():
|
||
try:
|
||
data = json.loads(extract_manifest.read_text(encoding="utf-8"))
|
||
if data.get("status") == "needs_user_confirmation":
|
||
return f"❌ 序列提取需要用户确认源FASTA文件。候选文件已写入manifest。Log: {log_file}", extract_manifest
|
||
except Exception:
|
||
pass
|
||
return f"❌ Sequence extraction by ID failed (exit code: {code}). Log: {log_file}", extract_manifest if extract_manifest.exists() else manifest
|
||
|
||
if stage == "msa_region_plot":
|
||
auto_conserved = bool(msa_auto_conserved or kwargs.get("msaAutoConserved"))
|
||
|
||
def _as_int(value, default=0):
|
||
try:
|
||
return int(value)
|
||
except (TypeError, ValueError):
|
||
return default
|
||
|
||
def _as_float(value, default=0.6):
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return default
|
||
|
||
start_col = _as_int(msa_start_col or kwargs.get("msaStartCol"), 0)
|
||
end_col = _as_int(msa_end_col or kwargs.get("msaEndCol"), 0)
|
||
window_len = _as_int(msa_window_len or kwargs.get("msaWindowLen"), 60)
|
||
min_nongap = _as_float(msa_min_nongap or kwargs.get("msaMinNongap"), 0.6)
|
||
reference_id = (msa_reference_id or kwargs.get("msaReferenceId") or "").strip()
|
||
reference_start = _as_int(msa_reference_start_residue or kwargs.get("msaReferenceStartResidue"), 0)
|
||
reference_end = _as_int(msa_reference_end_residue or kwargs.get("msaReferenceEndResidue"), 0)
|
||
reference_mode = bool(reference_id or reference_start or reference_end)
|
||
|
||
region_cmd = [
|
||
"/home/xzq/miniconda3/envs/test4py311/bin/python",
|
||
"apps/cli/run_msa_region_plot.py",
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix,
|
||
"--force",
|
||
]
|
||
resolved_msa = _resolve_msa_file_for_region_plot(
|
||
workspace=ws,
|
||
prefix=prefix,
|
||
msa_file=msa_file or kwargs.get("msaFile", ""),
|
||
source_image=msa_source_image or kwargs.get("msaSourceImage", ""),
|
||
)
|
||
if resolved_msa:
|
||
region_cmd.extend(["--msa-file", str(resolved_msa)])
|
||
if scope_file:
|
||
region_cmd.extend(["--scope-file", scope_file])
|
||
if analysis_route:
|
||
region_cmd.extend(["--analysis-route", analysis_route])
|
||
if member_set_id:
|
||
region_cmd.extend(["--member-set-id", member_set_id])
|
||
if auto_conserved:
|
||
region_cmd.extend(["--auto-conserved", "--window-len", str(window_len), "--min-nongap", str(min_nongap)])
|
||
elif reference_mode:
|
||
if not reference_id or reference_start < 1 or reference_end < 1:
|
||
return "❌ 参考序列编号绘图需要填写参考基因、起始残基和终止残基。", manifest
|
||
region_cmd.extend([
|
||
"--reference-id", reference_id,
|
||
"--reference-start-residue", str(reference_start),
|
||
"--reference-end-residue", str(reference_end),
|
||
])
|
||
else:
|
||
if start_col < 1 or end_col < 1:
|
||
return "❌ MSA区间绘图需要填写 alignment 起止列，或选择自动保守窗口。", manifest
|
||
region_cmd.extend(["--start-col", str(start_col), "--end-col", str(end_col)])
|
||
|
||
print(f"[CMD] {' '.join(region_cmd)}")
|
||
code = run_cmd_logged(region_cmd, log_file, cwd=Path("."))
|
||
region_manifest_candidates = list(final_dir.glob(f"{prefix}.msa_region_*.manifest.json"))
|
||
region_manifest_candidates.extend(final_dir.glob(f"{prefix}.msa_conserved_region_*.manifest.json"))
|
||
region_manifests = sorted(
|
||
region_manifest_candidates,
|
||
key=lambda p: p.stat().st_mtime,
|
||
reverse=True,
|
||
)
|
||
region_manifest = region_manifests[0] if region_manifests else manifest
|
||
if code == 0:
|
||
return "✅ MSA区间绘图完成。", region_manifest
|
||
return f"❌ MSA区间绘图失败 (exit code: {code}). Log: {log_file}", region_manifest
|
||
|
||
if stage == "synteny_plot_revise":
|
||
from apps.cli.run_synteny import run_circos_prepare, compute_file_hash
|
||
|
||
def _as_float_synteny(value, default):
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return default
|
||
|
||
def _clean_label(value: str, default: str) -> str:
|
||
label_value = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(value or default).strip())
|
||
return label_value.strip("._-") or default
|
||
|
||
def _infer_synteny_prefix_from_image_name(name: str) -> str:
|
||
stem = Path(str(name or "")).name
|
||
stem = re.sub(r"\.(png|jpg|jpeg|svg|pdf)$", "", stem, flags=re.I)
|
||
for suffix in (".synteny_replot_circos", ".synteny_replot.circos", ".synteny_replot", ".circos_png", ".circos_svg", ".circos.circos", ".circos"):
|
||
if stem.lower().endswith(suffix):
|
||
stem = stem[: -len(suffix)]
|
||
break
|
||
scoped = re.match(r"^([^.]+\.[^.]+)\.[^.]+\.target-", stem, flags=re.I)
|
||
if scoped:
|
||
stem = scoped.group(1)
|
||
return _clean_prefix(stem or prefix)
|
||
|
||
def _first_existing(paths):
|
||
for candidate in paths:
|
||
if candidate and Path(candidate).exists() and Path(candidate).is_file():
|
||
return Path(candidate).resolve()
|
||
return None
|
||
|
||
source_image_value = synteny_source_image or kwargs.get("syntenySourceImage") or ""
|
||
source_stage = str(synteny_source_stage or kwargs.get("syntenySourceStage") or "synteny").strip() or "synteny"
|
||
source_image_path = _workspace_url_to_path(source_image_value, ws) if source_image_value else None
|
||
inferred_prefix = _infer_synteny_prefix_from_image_name(Path(source_image_path).name if source_image_path else source_image_value)
|
||
if inferred_prefix:
|
||
prefix = inferred_prefix
|
||
species_prefix = prefix.split(".", 1)[0] if prefix else ""
|
||
analysis_dir = ws / "synteny" / "analysis"
|
||
if not analysis_dir.exists():
|
||
return "❌ 未找到 synteny/analysis 目录，无法重绘种内共线性 Circos 图。", manifest
|
||
|
||
gff_file = _first_existing([
|
||
analysis_dir / f"{species_prefix}.gff",
|
||
analysis_dir / f"{prefix}.gff",
|
||
*(sorted(analysis_dir.glob("*.gff"), key=lambda p: p.stat().st_mtime, reverse=True)),
|
||
])
|
||
collinearity_file = _first_existing([
|
||
analysis_dir / f"{species_prefix}.collinearity",
|
||
analysis_dir / f"{prefix}.collinearity",
|
||
*(sorted(analysis_dir.glob("*.collinearity"), key=lambda p: p.stat().st_mtime, reverse=True)),
|
||
])
|
||
gene_id_file = _first_existing([
|
||
final_dir / f"{prefix}.final_members.id",
|
||
final_dir / f"{species_prefix}.final_members.id",
|
||
*(sorted(final_dir.glob(f"{prefix}*.final_members.id"), key=lambda p: p.stat().st_mtime, reverse=True)),
|
||
*(sorted(final_dir.glob("*.final_members.id"), key=lambda p: p.stat().st_mtime, reverse=True)),
|
||
])
|
||
genome_file = _first_existing([
|
||
in_dir / species_prefix / f"{species_prefix}.genome.fasta",
|
||
in_dir / species_prefix / f"{species_prefix}.genome.fa",
|
||
in_dir / f"{species_prefix}.genome.fasta",
|
||
in_dir / f"{species_prefix}.genome.fa",
|
||
*(sorted(in_dir.glob(f"**/{species_prefix}.genome.fasta"), key=lambda p: p.stat().st_mtime, reverse=True) if in_dir.exists() and species_prefix else []),
|
||
*(sorted(in_dir.glob(f"**/{species_prefix}.genome.fa"), key=lambda p: p.stat().st_mtime, reverse=True) if in_dir.exists() and species_prefix else []),
|
||
])
|
||
missing = []
|
||
for label_name, path_value in (("GFF", gff_file), ("collinearity", collinearity_file), ("gene ID", gene_id_file), ("genome", genome_file)):
|
||
if not path_value:
|
||
missing.append(label_name)
|
||
if missing:
|
||
return f"❌ 缺少重绘所需文件: {', '.join(missing)}。请先完成 synteny 阶段。", manifest
|
||
|
||
output_label = _clean_label(synteny_output_label or kwargs.get("syntenyOutputLabel"), "synteny_replot_circos")
|
||
analysis_output_prefix = analysis_dir / f"{prefix}.synteny_replot"
|
||
requested_gene_label_size = _as_float_synteny(synteny_gene_label_size or kwargs.get("syntenyGeneLabelSize"), 38.0)
|
||
applied_gene_label_size = max(12.0, min(requested_gene_label_size, 80.0))
|
||
requested_gene_track_r1 = _as_float_synteny(synteny_gene_track_r1 or kwargs.get("syntenyGeneTrackR1"), 1.30)
|
||
# Large Circos text labels need a taller text track; otherwise Circos may
|
||
# silently mark all labels as not_placed. Keep r0 as the user's distance
|
||
# control and expand only the outer boundary when the font is enlarged.
|
||
min_gene_track_r1 = 1.30 + max(0.0, applied_gene_label_size - 38.0) * 0.02
|
||
applied_gene_track_r1 = max(requested_gene_track_r1, min(min_gene_track_r1, 2.10))
|
||
plot_params = {
|
||
"gene_label_size": applied_gene_label_size,
|
||
"gene_track_r0": _as_float_synteny(synteny_gene_track_r0 or kwargs.get("syntenyGeneTrackR0"), 1.03),
|
||
"gene_track_r1": applied_gene_track_r1,
|
||
"chrom_label_radius": str(synteny_chrom_label_radius or kwargs.get("syntenyChromLabelRadius") or "1r - 60p"),
|
||
"chrom_label_size": max(20.0, min(_as_float_synteny(synteny_chrom_label_size or kwargs.get("syntenyChromLabelSize"), 40.0), 80.0)),
|
||
"ideogram_thickness": _as_float_synteny(synteny_ideogram_thickness or kwargs.get("syntenyIdeogramThickness"), 90.0),
|
||
}
|
||
start_time = time.time()
|
||
result = run_circos_prepare(
|
||
gff_file=gff_file,
|
||
collinearity_file=collinearity_file,
|
||
gene_id_file=gene_id_file,
|
||
output_prefix=analysis_output_prefix,
|
||
genome_file=genome_file,
|
||
**plot_params,
|
||
)
|
||
final_outputs = []
|
||
for suffix in ("svg", "png"):
|
||
src = analysis_dir / f"{analysis_output_prefix.name}.circos.{suffix}"
|
||
if src.exists():
|
||
dest = final_dir / f"{prefix}.{output_label}.{suffix}"
|
||
shutil.copy2(src, dest)
|
||
final_outputs.append(dest)
|
||
success = bool(result.get("success")) and bool(final_outputs)
|
||
artifacts = {
|
||
"source_gff": str(gff_file),
|
||
"source_collinearity": str(collinearity_file),
|
||
"source_gene_ids": str(gene_id_file),
|
||
"source_genome": str(genome_file),
|
||
}
|
||
for out in final_outputs:
|
||
suffix = out.suffix.lower().lstrip(".")
|
||
artifacts[f"synteny_replot_{suffix}"] = {
|
||
"artifact": "synteny_replot_circos" if suffix in {"png", "svg"} else "synteny_replot_file",
|
||
"path": str(out.relative_to(ws)),
|
||
"full_path": str(out.resolve()),
|
||
"type": "image" if suffix in {"png", "svg", "jpg", "jpeg"} else "file",
|
||
"format": suffix,
|
||
"size": out.stat().st_size,
|
||
"sha256": compute_file_hash(out),
|
||
"description": "种内共线性 Circos 重绘图",
|
||
}
|
||
circos_conf = result.get("circos_conf")
|
||
if circos_conf and Path(circos_conf).exists():
|
||
conf_path = Path(circos_conf)
|
||
artifacts["synteny_replot_circos_conf"] = {
|
||
"artifact": "synteny_replot_config",
|
||
"path": str(conf_path.relative_to(ws)) if conf_path.is_relative_to(ws) else str(conf_path),
|
||
"full_path": str(conf_path.resolve()),
|
||
"type": "config",
|
||
"format": "conf",
|
||
"size": conf_path.stat().st_size,
|
||
"sha256": compute_file_hash(conf_path),
|
||
}
|
||
metrics = {
|
||
"circos_plotting_success": success,
|
||
"elapsed_sec": round(time.time() - start_time, 3),
|
||
"circos_gene_label_size": plot_params["gene_label_size"],
|
||
"circos_gene_label_size_requested": requested_gene_label_size,
|
||
"circos_gene_label_size_was_clamped": requested_gene_label_size != plot_params["gene_label_size"],
|
||
"circos_gene_track_r1_requested": requested_gene_track_r1,
|
||
"circos_gene_track_r1_auto_expanded": requested_gene_track_r1 != plot_params["gene_track_r1"],
|
||
"circos_gene_track_r0": plot_params["gene_track_r0"],
|
||
"circos_gene_track_r1": plot_params["gene_track_r1"],
|
||
"circos_chrom_label_radius": plot_params["chrom_label_radius"],
|
||
"circos_chrom_label_size": plot_params["chrom_label_size"],
|
||
"circos_ideogram_thickness": plot_params["ideogram_thickness"],
|
||
}
|
||
for key in ("circos_chromosomes_total", "circos_chromosomes_plotted", "circos_chromosomes_filtered", "circos_chromosomes_filter_mode", "circos_chromosomes_filtered_names"):
|
||
if key in result:
|
||
metrics[key] = result.get(key)
|
||
safe_result = json.loads(json.dumps(result, ensure_ascii=False, default=str))
|
||
manifest_data = {
|
||
"manifest_version": "1.0",
|
||
"stage": "synteny_plot_revise",
|
||
"status": "success" if success else "failed",
|
||
"prefix": prefix,
|
||
"source_stage": source_stage,
|
||
"display_stage": "synteny",
|
||
"created_at": datetime.now().isoformat(),
|
||
"parameters": {
|
||
"output_label": output_label,
|
||
"source_stage": source_stage,
|
||
"display_stage": "synteny",
|
||
"source_image": source_image_value,
|
||
**plot_params,
|
||
},
|
||
"artifacts": artifacts,
|
||
"metrics": metrics,
|
||
"plot": safe_result,
|
||
"note": "圈图保留主核染色体背景，过滤掉疑似细胞器/非主序列。",
|
||
}
|
||
synteny_manifest = final_dir / f"{prefix}.{output_label}.manifest.json"
|
||
canonical_manifest = final_dir / f"{prefix}.synteny_plot_revise.manifest.json"
|
||
synteny_manifest.write_text(json.dumps(manifest_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
canonical_manifest.write_text(json.dumps(manifest_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
if success:
|
||
return "✅ 种内共线性 Circos 图修订完成。", canonical_manifest
|
||
return f"❌ 种内共线性 Circos 图修订失败：{result.get('stderr') or '未生成SVG/PNG'}", canonical_manifest
|
||
|
||
if stage == "tree_plot_revise":
|
||
layout = (phy_tree_layout or kwargs.get("treeLayout") or "both").lower()
|
||
if layout not in {"rectangular", "circular", "both"}:
|
||
return f"❌ 树图布局无效：{layout}", manifest
|
||
resolved_tree = _resolve_tree_file_for_replot(
|
||
workspace=ws,
|
||
prefix=prefix,
|
||
tree_file=tree_file or kwargs.get("treeFile", ""),
|
||
source_image=tree_source_image or kwargs.get("treeSourceImage", ""),
|
||
)
|
||
if not resolved_tree:
|
||
return "❌ 未找到可重绘的 Newick 树文件。", manifest
|
||
from apps.cli.run_phylogeny import run_tree_plotting, compute_file_hash
|
||
source_stage = str(tree_source_stage or kwargs.get("treeSourceStage") or "").strip()
|
||
source_image_name = Path(str(tree_source_image or kwargs.get("treeSourceImage") or "")).name.lower()
|
||
allowed_display_stages = {
|
||
"phylogeny",
|
||
"reference_classification",
|
||
"motif",
|
||
"domain",
|
||
"structure",
|
||
"interspecies_synteny",
|
||
"combo",
|
||
"tree_plot_revise",
|
||
}
|
||
display_stage = source_stage if source_stage in allowed_display_stages else ""
|
||
if display_stage == "tree_plot_revise":
|
||
display_stage = ""
|
||
if not display_stage:
|
||
if ".motif.tree_motif." in source_image_name:
|
||
display_stage = "motif"
|
||
elif ".reference_classification.tree" in source_image_name:
|
||
display_stage = "reference_classification"
|
||
elif ".interspecies_tree" in source_image_name:
|
||
display_stage = "interspecies_synteny"
|
||
elif ".combo." in source_image_name:
|
||
display_stage = "combo"
|
||
else:
|
||
display_stage = "phylogeny"
|
||
base = resolved_tree.stem
|
||
if base.endswith(".tree"):
|
||
base = base[:-5]
|
||
def _as_float_tree(value, default):
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return default
|
||
|
||
def _as_bool_tree(value, default=True):
|
||
if isinstance(value, bool):
|
||
return value
|
||
if value is None or value == "":
|
||
return default
|
||
return str(value).lower() in {"1", "true", "yes", "y"}
|
||
|
||
plot_params = {
|
||
"label_offset": _as_float_tree(tree_label_offset or kwargs.get("treeLabelOffset"), 0.4),
|
||
"tip_label_size": _as_float_tree(tree_tip_label_size or kwargs.get("treeTipLabelSize"), 3.0),
|
||
"branch_color": str(tree_branch_color or kwargs.get("treeBranchColor") or "black"),
|
||
"branch_width": _as_float_tree(tree_branch_width or kwargs.get("treeBranchWidth"), 0.5),
|
||
"show_bootstrap": _as_bool_tree(tree_show_bootstrap if tree_show_bootstrap is not None else kwargs.get("treeShowBootstrap"), True),
|
||
"bootstrap_size": _as_float_tree(tree_bootstrap_size or kwargs.get("treeBootstrapSize"), 2.0),
|
||
"tip_point_size": _as_float_tree(tree_tip_point_size or kwargs.get("treeTipPointSize"), 2.0),
|
||
"target_shape": str(tree_target_shape or kwargs.get("treeTargetShape") or "triangle"),
|
||
"reference_shape": str(tree_reference_shape or kwargs.get("treeReferenceShape") or "circle"),
|
||
"gene_shape": str(tree_target_shape or kwargs.get("treeTargetShape") or "triangle"),
|
||
"marker_mode": str(tree_marker_mode or kwargs.get("treeMarkerMode") or "auto"),
|
||
"subfamily_arc_enabled": _as_bool_tree(tree_subfamily_arc_enabled if tree_subfamily_arc_enabled is not None else kwargs.get("treeSubfamilyArcEnabled"), True),
|
||
"subfamily_arc_gap": _as_float_tree(tree_subfamily_arc_gap or kwargs.get("treeSubfamilyArcGap"), 1.2),
|
||
"subfamily_arc_label_gap": _as_float_tree(tree_subfamily_arc_label_gap or kwargs.get("treeSubfamilyArcLabelGap"), 0.6),
|
||
"subfamily_arc_label_size": _as_float_tree(tree_subfamily_arc_label_size or kwargs.get("treeSubfamilyArcLabelSize"), 0.0),
|
||
}
|
||
label = (tree_output_label or kwargs.get("treeOutputLabel") or f"tree_replot_{layout}").strip()
|
||
safe_label = re.sub(r"[^A-Za-z0-9_.-]+", "_", label)
|
||
output_prefix = f"{base}.{safe_label}"
|
||
|
||
def _first_existing(paths):
|
||
for candidate in paths:
|
||
if candidate and Path(candidate).exists() and Path(candidate).is_file():
|
||
return Path(candidate)
|
||
return None
|
||
|
||
def _manifest_species_values(manifest_data: dict, key: str) -> list[str]:
|
||
params_data = manifest_data.get("parameters") if isinstance(manifest_data.get("parameters"), dict) else {}
|
||
value = params_data.get(key) or manifest_data.get(key) or (manifest_data.get("metrics") or {}).get(key)
|
||
if isinstance(value, list):
|
||
return [str(v) for v in value if str(v)]
|
||
if isinstance(value, str) and value:
|
||
return [v]
|
||
return []
|
||
|
||
def _read_interspecies_manifest() -> dict:
|
||
candidates = [final_dir / f"{prefix}.interspecies_synteny.manifest.json"]
|
||
candidates.extend(sorted(final_dir.glob(f"{prefix}*.interspecies_synteny.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True))
|
||
tree_base = resolved_tree.name.replace(".nwk", "")
|
||
if tree_base.endswith(".interspecies_tree"):
|
||
root_prefix = tree_base[: -len(".interspecies_tree")]
|
||
candidates.append(final_dir / f"{root_prefix}.interspecies_synteny.manifest.json")
|
||
candidates.extend(sorted(final_dir.glob(f"{root_prefix}*.interspecies_synteny.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True))
|
||
seen_paths = set()
|
||
for candidate in candidates:
|
||
if candidate in seen_paths or not candidate.exists():
|
||
continue
|
||
seen_paths.add(candidate)
|
||
try:
|
||
data = json.loads(candidate.read_text(encoding="utf-8"))
|
||
if isinstance(data, dict):
|
||
return data
|
||
except Exception:
|
||
continue
|
||
return {}
|
||
|
||
def _run_interspecies_tree_replot() -> dict:
|
||
inter_manifest = _read_interspecies_manifest()
|
||
artifacts_data = inter_manifest.get("artifacts") if isinstance(inter_manifest.get("artifacts"), dict) else {}
|
||
gene_map_value = artifacts_data.get("gene_to_species")
|
||
gene_map_path = None
|
||
if isinstance(gene_map_value, dict):
|
||
gene_map_path = _workspace_url_to_path(gene_map_value.get("full_path") or gene_map_value.get("path") or "", ws)
|
||
if not gene_map_path or not gene_map_path.exists():
|
||
tree_base = resolved_tree.name.replace(".interspecies_tree.nwk", "")
|
||
gene_map_path = _first_existing([
|
||
ws / "interspecies_synteny" / "analysis" / f"{prefix}.gene_to_species.tsv",
|
||
ws / "interspecies_synteny" / "analysis" / f"{tree_base}.gene_to_species.tsv",
|
||
])
|
||
|
||
target_species_values = _manifest_species_values(inter_manifest, "target_species")
|
||
ref_species_values = _manifest_species_values(inter_manifest, "ref_species")
|
||
if gene_map_path and gene_map_path.exists() and (not target_species_values or not ref_species_values):
|
||
try:
|
||
with gene_map_path.open("r", encoding="utf-8", errors="ignore") as handle:
|
||
header = handle.readline().rstrip("\n").split("\t")
|
||
idx_species = header.index("species") if "species" in header else 1
|
||
idx_group = header.index("group") if "group" in header else 2
|
||
target_seen, ref_seen = [], []
|
||
for raw in handle:
|
||
parts = raw.rstrip("\n").split("\t")
|
||
if len(parts) <= max(idx_species, idx_group):
|
||
continue
|
||
species_value = parts[idx_species]
|
||
group_value = parts[idx_group]
|
||
if group_value == "target" and species_value not in target_seen:
|
||
target_seen.append(species_value)
|
||
if group_value == "ref" and species_value not in ref_seen:
|
||
ref_seen.append(species_value)
|
||
target_species_values = target_species_values or target_seen
|
||
ref_species_values = ref_species_values or ref_seen
|
||
except Exception:
|
||
pass
|
||
target_species_label = ",".join(target_species_values) if target_species_values else "Target"
|
||
ref_species_label = ",".join(ref_species_values) if ref_species_values else "Reference"
|
||
|
||
temp_dir = ws / "tmp" / f"tree_plot_revise_{safe_label}"
|
||
if temp_dir.exists():
|
||
shutil.rmtree(temp_dir)
|
||
temp_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
def _read_tree_tip_labels(tree_path: Path) -> set[str]:
|
||
try:
|
||
text = tree_path.read_text(encoding="utf-8", errors="ignore")
|
||
except Exception:
|
||
return set()
|
||
return {m.group(1).strip().strip("'").strip('"') for m in re.finditer(r"(?<=[(,])([^():,;]+):", text) if m.group(1).strip()}
|
||
|
||
def _merge_matching_subfamily_files(candidates: list[Path], tips: set[str]) -> Path | None:
|
||
if not tips:
|
||
return None
|
||
merged: dict[str, str] = {}
|
||
used_files: list[str] = []
|
||
seen_candidates: set[Path] = set()
|
||
for candidate in candidates:
|
||
if candidate in seen_candidates or not candidate.exists() or not candidate.is_file():
|
||
continue
|
||
seen_candidates.add(candidate)
|
||
matched_here = False
|
||
try:
|
||
with candidate.open("r", encoding="utf-8", errors="ignore") as handle:
|
||
for raw in handle:
|
||
parts = raw.strip().split()
|
||
if len(parts) < 2:
|
||
continue
|
||
gene_id, subfamily = parts[0], parts[1]
|
||
if gene_id in tips and gene_id not in merged:
|
||
merged[gene_id] = subfamily
|
||
matched_here = True
|
||
except Exception:
|
||
continue
|
||
if matched_here:
|
||
used_files.append(str(candidate))
|
||
if not merged:
|
||
return None
|
||
merged_path = temp_dir / f"{tree_root_prefix}.merged.geneID.subfamily"
|
||
with merged_path.open("w", encoding="utf-8") as handle:
|
||
for gene_id in sorted(merged):
|
||
handle.write(f"{gene_id}\t{merged[gene_id]}\n")
|
||
return merged_path
|
||
|
||
tree_root_prefix = resolved_tree.name.replace(".interspecies_tree.nwk", "")
|
||
subfamily_candidates = []
|
||
for stem in (prefix, tree_root_prefix):
|
||
subfamily_candidates.extend([
|
||
final_dir / f"{stem}.geneID.subfamily",
|
||
final_dir / f"{stem}.target_plus_reference.geneID.subfamily",
|
||
in_dir / f"{stem}.geneID.subfamily",
|
||
])
|
||
subfamily_candidates.extend(sorted(final_dir.glob(f"{stem}*.geneID.subfamily"), key=lambda p: p.stat().st_mtime, reverse=True))
|
||
for species_value in [*target_species_values, *ref_species_values]:
|
||
species_stem = str(species_value).strip()
|
||
if not species_stem:
|
||
continue
|
||
subfamily_candidates.extend([
|
||
final_dir / f"{species_stem}.SPL.target_plus_reference.geneID.subfamily",
|
||
final_dir / f"{species_stem}.SPL.geneID.subfamily",
|
||
])
|
||
subfamily_candidates.extend(sorted(final_dir.glob(f"{species_stem}.*.geneID.subfamily"), key=lambda p: p.stat().st_mtime, reverse=True))
|
||
subfamily_file = _first_existing(subfamily_candidates)
|
||
tip_labels = _read_tree_tip_labels(resolved_tree)
|
||
merged_subfamily_file = _merge_matching_subfamily_files(subfamily_candidates, tip_labels)
|
||
if merged_subfamily_file:
|
||
subfamily_file = merged_subfamily_file
|
||
r_script = Path("scripts/interspecies_tree.R")
|
||
cmd = [
|
||
"/home/xzq/miniconda3/envs/test4py311/bin/Rscript",
|
||
str(r_script),
|
||
str(resolved_tree),
|
||
str(temp_dir),
|
||
target_species_label,
|
||
ref_species_label,
|
||
str(subfamily_file) if subfamily_file else "",
|
||
"",
|
||
str(gene_map_path) if gene_map_path and gene_map_path.exists() else "",
|
||
str(plot_params["label_offset"]),
|
||
str(plot_params["tip_label_size"]),
|
||
str(plot_params["branch_color"]),
|
||
str(plot_params["branch_width"]),
|
||
str(plot_params["target_shape"]),
|
||
str(plot_params["reference_shape"]),
|
||
str(plot_params["tip_point_size"]),
|
||
str(plot_params["bootstrap_size"]),
|
||
"true" if plot_params.get("subfamily_arc_enabled", True) else "false",
|
||
str(plot_params.get("subfamily_arc_gap", 1.2)),
|
||
str(plot_params.get("subfamily_arc_label_gap", 0.6)),
|
||
str(plot_params.get("subfamily_arc_label_size", 0.0) or ""),
|
||
]
|
||
start_time = time.time()
|
||
proc = subprocess.run(cmd, capture_output=True, text=True, check=False, timeout=180)
|
||
output_files = []
|
||
outputs = {
|
||
"rect_pdf": temp_dir / "interspecies_tree.pdf",
|
||
"rect_png": temp_dir / "interspecies_tree.png",
|
||
"circ_pdf": temp_dir / "interspecies_tree.circular_tree.pdf",
|
||
"circ_png": temp_dir / "interspecies_tree.circular_tree.png",
|
||
}
|
||
wanted = []
|
||
if layout in {"rectangular", "both"}:
|
||
wanted.extend(["rect_pdf", "rect_png"])
|
||
if layout in {"circular", "both"}:
|
||
wanted.extend(["circ_pdf", "circ_png"])
|
||
for key in wanted:
|
||
src = outputs.get(key)
|
||
if not src or not src.exists():
|
||
continue
|
||
suffix = ".circular_tree" if key.startswith("circ_") else ""
|
||
dest = final_dir / f"{output_prefix}{suffix}{src.suffix}"
|
||
shutil.copy2(src, dest)
|
||
output_files.append(dest)
|
||
return {
|
||
"success": proc.returncode == 0 and bool(output_files),
|
||
"output_files": [str(p) for p in output_files],
|
||
"elapsed_sec": round(time.time() - start_time, 3),
|
||
"stdout": proc.stdout,
|
||
"stderr": proc.stderr,
|
||
"returncode": proc.returncode,
|
||
"command": cmd,
|
||
"interspecies_tree_replot": True,
|
||
"gene_to_species": str(gene_map_path) if gene_map_path else "",
|
||
"subfamily_file": str(subfamily_file) if subfamily_file else "",
|
||
"target_species": target_species_values,
|
||
"ref_species": ref_species_values,
|
||
}
|
||
|
||
if display_stage == "interspecies_synteny" or ".interspecies_tree" in source_image_name or resolved_tree.name.endswith(".interspecies_tree.nwk"):
|
||
result = _run_interspecies_tree_replot()
|
||
else:
|
||
result = run_tree_plotting(
|
||
tree_file=resolved_tree,
|
||
output_prefix=output_prefix,
|
||
tree_layout=layout,
|
||
**plot_params,
|
||
)
|
||
tree_manifest = final_dir / f"{output_prefix}.manifest.json"
|
||
artifacts = {}
|
||
for out in result.get("output_files", []) or []:
|
||
path = Path(out)
|
||
if not path.exists():
|
||
continue
|
||
suffix = path.suffix.lower().lstrip(".")
|
||
artifacts[path.name] = {
|
||
"artifact": "tree_replot" if suffix == "png" else "tree_replot_file",
|
||
"path": str(path.relative_to(ws)),
|
||
"full_path": str(path.resolve()),
|
||
"type": "image" if suffix == "png" else "document" if suffix == "pdf" else "file",
|
||
"format": suffix,
|
||
"size": path.stat().st_size,
|
||
"sha256": compute_file_hash(path),
|
||
}
|
||
manifest_data = {
|
||
"manifest_version": "1.0",
|
||
"stage": "tree_plot_revise",
|
||
"status": "success" if result.get("success") else "failed",
|
||
"prefix": prefix,
|
||
"source_stage": source_stage,
|
||
"display_stage": display_stage,
|
||
"created_at": datetime.now().isoformat(),
|
||
"parameters": {
|
||
"tree_layout": layout,
|
||
"output_prefix": output_prefix,
|
||
"source_stage": source_stage,
|
||
"display_stage": display_stage,
|
||
"source_image": tree_source_image or kwargs.get("treeSourceImage", ""),
|
||
**plot_params,
|
||
},
|
||
"artifacts": {"source_tree": str(resolved_tree), **artifacts},
|
||
"metrics": {"tree_plotting_success": bool(result.get("success")), "elapsed_sec": result.get("elapsed_sec")},
|
||
"plot": result,
|
||
}
|
||
tree_manifest.write_text(json.dumps(manifest_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
canonical_manifest = final_dir / f"{prefix}.tree_plot_revise.manifest.json"
|
||
canonical_manifest.write_text(json.dumps(manifest_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
if result.get("success"):
|
||
return "✅ 进化树图修订完成。", canonical_manifest
|
||
return f"❌ 进化树图修订失败：{result.get('error') or result.get('stderr') or 'unknown'}", tree_manifest
|
||
|
||
if stage in PREPARE_SUBSTAGES:
|
||
script = Path("apps/cli/run_prepare_substep.py")
|
||
if not script.exists():
|
||
return "Prepare substep script not found: apps/cli/run_prepare_substep.py", manifest
|
||
prep_cmd = [
|
||
"/home/xzq/miniconda3/envs/test4py311/bin/python",
|
||
str(script),
|
||
"--input-dir", str(in_dir),
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix or "data_prep",
|
||
"--stage", stage,
|
||
"--operation", PREPARE_SUBSTAGES[stage],
|
||
"--cpu", str(cpu),
|
||
]
|
||
print(f"[CMD] {' '.join(prep_cmd)}")
|
||
code = run_cmd_logged(prep_cmd, log_file, cwd=Path("."))
|
||
prep_manifest = ws / "final" / f"{prefix or 'data_prep'}.{stage}.manifest.json"
|
||
if code == 0 and prep_manifest.exists():
|
||
return f"✅ Prepare substep complete: {stage}. Log: {log_file}", prep_manifest
|
||
return f"❌ Prepare substep failed: {stage} (exit code: {code}). Log: {log_file}", prep_manifest if prep_manifest.exists() else manifest
|
||
|
||
if stage == "identify_refine":
|
||
script = Path("apps/cli/run_identify_refine.py")
|
||
if not script.exists():
|
||
return "Identify refine script not found: apps/cli/run_identify_refine.py", manifest
|
||
cmd = [
|
||
"/home/xzq/miniconda3/envs/test4py311/bin/python",
|
||
str(script),
|
||
"--input-dir", str(in_dir),
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix,
|
||
]
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
identify_manifest = ws / "final" / f"{prefix}.identify.manifest.json"
|
||
refine_manifest = ws / "final" / f"{prefix}.identify_refine.manifest.json"
|
||
if code == 0 and identify_manifest.exists():
|
||
return f"✅ Identify成员修订完成。后续亚家族证据阶段将使用 curated_members；final_members 将在亚家族确认后生成。日志：{log_file}", refine_manifest if refine_manifest.exists() else identify_manifest
|
||
return f"❌ Identify成员修订失败 (exit code: {code}). 请上传 {prefix}.curated_members.id 或 {prefix}.final_members.edited.id。日志：{log_file}", refine_manifest if refine_manifest.exists() else identify_manifest if identify_manifest.exists() else manifest
|
||
|
||
if stage == "identify":
|
||
# ... 现有identify代码保持不变 ...
|
||
script = Path("apps/cli/run_identify.py")
|
||
if not script.exists():
|
||
return (
|
||
"❌ Identify 入口缺失：apps/cli/run_identify.py 不存在",
|
||
manifest,
|
||
)
|
||
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--input-dir", str(in_dir),
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix,
|
||
"--cpu", str(cpu),
|
||
"--retention-days", str(retention_days),
|
||
"--log-file", str(log_file),
|
||
"--dom-i-evalue-max", str(id_evalue),
|
||
"--hmm-cov-min", str(id_coverage),
|
||
"--blast-evalue", str(blast_evalue),
|
||
"--blast-identity-min", str(blast_identity),
|
||
]
|
||
|
||
# 直接传参（用户填了就传；没填就不传，让脚本内部推断/默认）
|
||
if required_pfams:
|
||
cmd += ["--required-pfams", required_pfams]
|
||
if ref_species:
|
||
cmd += ["--ref-species", ",".join(ref_species)]
|
||
if scope_file:
|
||
scope_path = Path(scope_file).expanduser()
|
||
if not scope_path.is_absolute():
|
||
for candidate in (in_dir / scope_path, ws / scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
scope_path = candidate
|
||
break
|
||
if not scope_path.exists():
|
||
return f"❌ Identify scope 文件不存在: {scope_file}", manifest
|
||
try:
|
||
scope_data = read_scope(scope_path)
|
||
print(f"[INFO] Identify scope: {scope_data.get('scope_id', scope_path.name)}")
|
||
except Exception as exc:
|
||
return f"❌ Identify scope 文件无法读取: {scope_file}; {exc}", manifest
|
||
cmd += ["--scope-file", str(scope_path)]
|
||
selected_targets = _species_arg_list(target_species)
|
||
if selected_targets:
|
||
cmd += ["--target-species", ",".join(selected_targets)]
|
||
|
||
# 记录命令到可复现性管理器
|
||
if repro_manager:
|
||
try:
|
||
repro_manager.start_stage(stage, cmd, f"Run {stage} analysis")
|
||
except Exception as e:
|
||
print(f"[WARN] Failed to record command: {e}")
|
||
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
# 标记阶段完成
|
||
if repro_manager:
|
||
try:
|
||
status = "success" if code == 0 else "failed"
|
||
repro_manager.end_stage(stage, status=status, log_file=str(log_file))
|
||
except Exception as e:
|
||
print(f"[WARN] Failed to end stage recording: {e}")
|
||
|
||
if code != 0:
|
||
# 读取日志文件最后50行，提取详细错误信息
|
||
error_detail = ""
|
||
try:
|
||
if log_file.exists():
|
||
with open(log_file, 'r', encoding='utf-8', errors='ignore') as f:
|
||
lines = f.readlines()
|
||
# 获取最后50行
|
||
last_lines = lines[-50:] if len(lines) > 50 else lines
|
||
# 查找包含错误关键字的行
|
||
error_lines = []
|
||
for line in last_lines:
|
||
line = line.strip()
|
||
if any(keyword in line for keyword in ['Error', 'ERROR', 'Exception', 'Traceback', 'RuntimeError', '❌', 'failed', '失败']):
|
||
error_lines.append(line)
|
||
# 如果有错误行，使用最后5行；否则使用最后5行日志
|
||
if error_lines:
|
||
error_detail = "\\n".join(error_lines[-5:])
|
||
else:
|
||
error_detail = "\\n".join([l.strip() for l in last_lines[-5:]])
|
||
except Exception as e:
|
||
error_detail = f"(无法读取日志: {e})"
|
||
|
||
error_msg = f"❌ Identify 运行失败"
|
||
if error_detail:
|
||
error_msg += f"\\n详细错误:\\n{error_detail}"
|
||
error_msg += f"\\n完整日志: {log_file}"
|
||
return error_msg, manifest
|
||
|
||
identify_manifest = first_existing_stage_manifest(ws / "final", "identify", prefix) or find_latest_manifest(ws)
|
||
audit_failed, audit_reason = identify_hmm_audit_failure(identify_manifest)
|
||
if audit_failed:
|
||
return f"❌ Identify HMM模型审计失败，已阻断下游：{audit_reason}。日志：{log_file}", identify_manifest or manifest
|
||
manifest = identify_manifest or manifest
|
||
|
||
# 生成可复现性文件（仅在最后一个阶段或 all 阶段完成时）
|
||
if repro_manager and stage == "all":
|
||
try:
|
||
repro_manager.finalize()
|
||
except Exception as e:
|
||
print(f"[WARN] Failed to finalize reproducibility: {e}")
|
||
|
||
return f"✅ Identify 完成。日志：{log_file}", manifest
|
||
|
||
# ==================== 修改从这里开始 ====================
|
||
if stage == "phylogeny":
|
||
script = Path("apps/cli/run_phylogeny.py")
|
||
if not script.exists():
|
||
return (
|
||
"❌ Phylogeny 入口缺失：apps/cli/run_phylogeny.py 不存在",
|
||
manifest,
|
||
)
|
||
|
||
protein_fasta = None
|
||
if not scope_file:
|
||
protein_fasta, member_error = resolve_member_protein_for_stage_route(
|
||
ws=ws,
|
||
final_dir=final_dir,
|
||
in_dir=in_dir,
|
||
prefix=prefix,
|
||
analysis_route=analysis_route,
|
||
member_set_id=member_set_id,
|
||
)
|
||
if member_error:
|
||
return member_error, manifest
|
||
|
||
# 构建phylogeny命令
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--prefix", prefix,
|
||
"--workspace", str(ws), # workspace/<job_id> 目录
|
||
"--cpu", str(cpu),
|
||
"--log-file", str(log_file),
|
||
]
|
||
if scope_file:
|
||
scope_path = Path(scope_file).expanduser()
|
||
if not scope_path.is_absolute():
|
||
for candidate in (in_dir / scope_path, ws / scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
scope_path = candidate
|
||
break
|
||
if not scope_path.exists():
|
||
return f"❌ Phylogeny scope 文件不存在: {scope_file}", manifest
|
||
cmd += ["--scope-file", str(scope_path)]
|
||
elif protein_fasta:
|
||
cmd += ["--proteins", str(protein_fasta)]
|
||
if analysis_route:
|
||
cmd += ["--analysis-route", analysis_route]
|
||
if member_set_id:
|
||
cmd += ["--member-set-id", member_set_id]
|
||
phylogeny_scope_data = {}
|
||
phylogeny_scope_id_for_labels = ""
|
||
if scope_file:
|
||
try:
|
||
phylogeny_scope_data = read_scope(scope_path)
|
||
phylogeny_scope_id_for_labels = str(phylogeny_scope_data.get("scope_id") or "")
|
||
except Exception:
|
||
phylogeny_scope_data = {}
|
||
phylogeny_scope_id_for_labels = ""
|
||
subfamily_file = None
|
||
if should_disable_cli_subfamily_auto(analysis_route, subfamily_label_source):
|
||
cmd += ["--subfamily", "NONE"]
|
||
else:
|
||
subfamily_file = resolve_subfamily_for_plotting(ws / "final", in_dir, prefix, phylogeny_scope_id_for_labels, subfamily_label_source)
|
||
if not subfamily_file and phylogeny_scope_data:
|
||
subfamily_file = merge_subfamily_for_scope(ws / "final", in_dir, prefix, phylogeny_scope_data, subfamily_label_source)
|
||
if subfamily_file:
|
||
cmd += ["--subfamily", str(subfamily_file)]
|
||
|
||
# 添加高级参数
|
||
if phy_mafft:
|
||
cmd += [f"--mafft-params={phy_mafft}"]
|
||
if phy_iqtree:
|
||
cmd += [f"--iqtree-params={phy_iqtree}"]
|
||
if phy_tree_layout:
|
||
cmd += ["--tree-layout", phy_tree_layout]
|
||
cmd += [
|
||
"--tree-subfamily-arc-enabled", "true" if tree_subfamily_arc_enabled else "false",
|
||
"--tree-subfamily-arc-gap", str(tree_subfamily_arc_gap),
|
||
"--tree-subfamily-arc-label-gap", str(tree_subfamily_arc_label_gap),
|
||
"--tree-subfamily-arc-label-size", str(tree_subfamily_arc_label_size or 0.0),
|
||
]
|
||
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
# 严格检查：必须生成 manifest 文件才算成功。scope 运行只认 scoped manifest，
|
||
# 避免旧的 {prefix}.phylogeny.manifest.json 让新物种组合误判完成。
|
||
phylogeny_scope_id = ""
|
||
if scope_file:
|
||
try:
|
||
phylogeny_scope_id = str(read_scope(scope_path).get("scope_id") or "")
|
||
except Exception:
|
||
phylogeny_scope_id = ""
|
||
output_prefix_token = stage_output_prefix("phylogeny", prefix, phylogeny_scope_data, analysis_route, member_set_id)
|
||
new_manifest = first_existing_stage_manifest(ws / "final", "phylogeny", prefix, phylogeny_scope_data, analysis_route, member_set_id)
|
||
expected_manifest = ws / "final" / f"{output_prefix_token}.phylogeny.manifest.json"
|
||
|
||
if code == 0 and new_manifest and new_manifest.exists():
|
||
# 成功：检查输出文件完整性
|
||
expected_files = [
|
||
ws / "final" / f"{output_prefix_token}.pep.msa.fa",
|
||
ws / "final" / f"{output_prefix_token}.tree.nwk",
|
||
ws / "final" / f"{output_prefix_token}.phylogeny_report.md",
|
||
new_manifest
|
||
]
|
||
|
||
existing = [f for f in expected_files if f.exists()]
|
||
|
||
if len(existing) >= 3: # 至少3个主要文件
|
||
return f"✅ Phylogeny 完成。生成{len(existing)}个文件。日志：{log_file}", new_manifest
|
||
else:
|
||
# manifest 存在但其他文件不全，视为部分成功但标记为失败
|
||
return f"❌ Phylogeny 失败：输出文件不全（{len(existing)}/{len(expected_files)}）。日志：{log_file}", manifest
|
||
|
||
elif code == 0 and not new_manifest:
|
||
# 返回码0但没有manifest，视为失败
|
||
return f"❌ Phylogeny 失败：未生成 manifest 文件（期望 {expected_manifest.name}）。日志：{log_file}", manifest
|
||
|
||
elif new_manifest and new_manifest.exists():
|
||
# If the CLI wrote a phylogeny manifest, return it so the frontend shows the real failure.
|
||
return f"Phylogeny failed (exit code: {code}). Log: {log_file}", new_manifest
|
||
|
||
elif code == 1:
|
||
return f"Phylogeny failed: MAFFT succeeded but complete outputs were not generated. Log: {log_file}", manifest
|
||
|
||
else:
|
||
return f"Phylogeny failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
######## END PHYLOGENY #########
|
||
if stage == "reference_classification":
|
||
script = Path("apps/cli/run_reference_classification.py")
|
||
if not script.exists():
|
||
return "❌ Reference classification script not found: apps/cli/run_reference_classification.py", manifest
|
||
|
||
scope_path = None
|
||
reference_scope_id = ""
|
||
if scope_file:
|
||
scope_path = Path(scope_file).expanduser()
|
||
if not scope_path.is_absolute():
|
||
for candidate in (in_dir / scope_path, ws / scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
scope_path = candidate
|
||
break
|
||
if not scope_path.exists():
|
||
return f"❌ Reference classification scope 文件不存在: {scope_file}", manifest
|
||
try:
|
||
reference_scope_id = str(read_scope(scope_path).get("scope_id") or "")
|
||
except Exception:
|
||
reference_scope_id = ""
|
||
if scope_path:
|
||
try:
|
||
parent_scope_data = read_scope(scope_path)
|
||
except Exception as exc:
|
||
return f"❌ Reference classification scope 文件无法读取: {scope_file}; {exc}", manifest
|
||
family = str(parent_scope_data.get("family") or (prefix.split(".")[-1] if "." in prefix else "")).strip()
|
||
config_family = _gene_family_from_config_xlsx(in_dir / "config.xlsx")
|
||
analysis_family = str(((parent_scope_data.get("analysis") or {}).get("Family") if isinstance(parent_scope_data.get("analysis"), dict) else "") or "").strip()
|
||
target_list = [str(x) for x in parent_scope_data.get("target_species") or [] if str(x).strip()]
|
||
comparative_list = [str(x) for x in parent_scope_data.get("comparative_species") or [] if str(x).strip()]
|
||
reference_list = [str(x) for x in parent_scope_data.get("reference_species") or [] if str(x).strip()]
|
||
selected_codes = {item.lower() for item in [*target_list, *comparative_list, *reference_list]}
|
||
authoritative_family = config_family or analysis_family
|
||
if authoritative_family and family != authoritative_family:
|
||
return f"❌ Reference classification scope identity conflict: scope family={family}, config family={authoritative_family}", manifest
|
||
if family and family.lower() in selected_codes:
|
||
return f"❌ Reference classification scope identity conflict: {family} 是物种代码，不是 gene family", manifest
|
||
if not reference_list:
|
||
return f"❌ Reference classification 需要 referenceSpecies；当前 scope={reference_scope_id or '<empty>'} 未包含参考物种", manifest
|
||
classifiable_species = []
|
||
for item in [*target_list, *comparative_list]:
|
||
if item and item not in classifiable_species:
|
||
classifiable_species.append(item)
|
||
if classifiable_species and family:
|
||
parent_evidence_member_set = f"{reference_scope_id}:subfamily_evidence:domain_refined" if reference_scope_id else ""
|
||
parent_evidence_tree = None
|
||
parent_evidence_ready = False
|
||
if analysis_mode != "per_species" and len(classifiable_species) > 1 and parent_evidence_member_set:
|
||
parent_evidence_prefix = stage_output_prefix(
|
||
"phylogeny",
|
||
prefix,
|
||
parent_scope_data,
|
||
"subfamily_evidence",
|
||
parent_evidence_member_set,
|
||
)
|
||
parent_evidence_tree = final_dir / f"{parent_evidence_prefix}.tree.nwk"
|
||
parent_evidence_manifests = {
|
||
stage_name: first_existing_stage_manifest(
|
||
final_dir,
|
||
stage_name,
|
||
prefix,
|
||
parent_scope_data,
|
||
"subfamily_evidence",
|
||
parent_evidence_member_set,
|
||
)
|
||
for stage_name in ("phylogeny", "motif", "domain", "structure", "combo")
|
||
}
|
||
parent_evidence_ready = (
|
||
parent_evidence_tree.exists()
|
||
and parent_evidence_tree.stat().st_size > 0
|
||
and all(path and path.exists() for path in parent_evidence_manifests.values())
|
||
)
|
||
joint_mode = analysis_mode != "per_species" and len(classifiable_species) > 1
|
||
if parent_evidence_ready and parent_evidence_tree is not None and scope_path is not None:
|
||
ref_member_set = f"{reference_scope_id}:subfamily_prediction:all_members"
|
||
ref_cmd = [
|
||
sys.executable,
|
||
str(script),
|
||
"--prefix", prefix,
|
||
"--workspace", str(ws),
|
||
"--input-dir", str(in_dir),
|
||
"--tree", str(parent_evidence_tree),
|
||
"--target-prefix", prefix.split(".")[0] if "." in prefix else prefix,
|
||
"--reference-prefix", reference_prefix or (reference_list[0] if reference_list else "At"),
|
||
"--log-file", str(log_file),
|
||
"--scope-file", str(scope_path),
|
||
"--analysis-route", "subfamily_prediction",
|
||
"--member-set-id", ref_member_set,
|
||
]
|
||
if reference_subfamily_file:
|
||
ref_cmd.extend(["--reference-subfamily", reference_subfamily_file])
|
||
if reference_overwrite_gene_subfamily:
|
||
ref_cmd.append("--overwrite-gene-subfamily")
|
||
print(f"[CMD] {' '.join(ref_cmd)}")
|
||
ref_code = run_cmd_logged(ref_cmd, log_file, cwd=Path("."))
|
||
ref_manifest = first_existing_stage_manifest(
|
||
final_dir,
|
||
"reference_classification",
|
||
prefix,
|
||
parent_scope_data,
|
||
"subfamily_prediction",
|
||
ref_member_set,
|
||
)
|
||
if ref_code == 0 and ref_manifest:
|
||
try:
|
||
manifest_data = json.loads(Path(ref_manifest).read_text(encoding="utf-8"))
|
||
manifest_data["analysis_mode"] = "joint_multi_species"
|
||
manifest_data["classifiable_species"] = classifiable_species
|
||
manifest_data["target_species"] = target_list
|
||
manifest_data["comparative_species"] = comparative_list
|
||
manifest_data["reference_species"] = reference_list
|
||
manifest_data["policy"] = "joint_multi_species; one aggregate subfamily tree/combo/review table"
|
||
manifest_data.setdefault("metrics", {})["classifiable_species_count"] = len(classifiable_species)
|
||
stage_context = manifest_data.setdefault("stage_context", {})
|
||
if isinstance(stage_context, dict):
|
||
stage_context["analysis_mode"] = "joint_multi_species"
|
||
stage_context["classifiable_species"] = classifiable_species
|
||
Path(ref_manifest).write_text(json.dumps(manifest_data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
except Exception as exc:
|
||
print(f"[Reference classification] failed to annotate joint manifest: {exc}")
|
||
return (
|
||
f"✅ Reference classification 使用联合 evidence 生成一张联合亚家族分类结果：{', '.join(classifiable_species)}。日志：{log_file}",
|
||
ref_manifest,
|
||
)
|
||
return f"❌ Reference classification 使用联合 evidence 失败; log={log_file}", ref_manifest or manifest
|
||
|
||
if joint_mode:
|
||
missing_joint = []
|
||
if not parent_evidence_tree or not parent_evidence_tree.exists() or parent_evidence_tree.stat().st_size <= 0:
|
||
missing_joint.append("phylogeny_tree")
|
||
try:
|
||
missing_joint.extend([
|
||
stage_name for stage_name, manifest_path in parent_evidence_manifests.items()
|
||
if not manifest_path or not manifest_path.exists()
|
||
])
|
||
except Exception:
|
||
missing_joint.extend(["phylogeny", "motif", "domain", "structure", "combo"])
|
||
return (
|
||
"❌ Reference classification 联合模式需要同一 scope 的联合 evidence，不能自动拆成 per_species。"
|
||
f"缺失：{', '.join(dict.fromkeys(missing_joint)) or 'unknown'}。日志：{log_file}",
|
||
manifest,
|
||
)
|
||
|
||
aggregate_results = []
|
||
aggregate_errors = []
|
||
for species in classifiable_species:
|
||
species_prefix = f"{species}.{family}"
|
||
evidence_scope = build_gene_family_scope(
|
||
input_dir=in_dir,
|
||
workspace=ws,
|
||
prefix=species_prefix,
|
||
family=family,
|
||
target_species=[species],
|
||
reference_species=reference_list,
|
||
comparative_species=[],
|
||
scope_mode="target_plus_reference",
|
||
)
|
||
evidence_scope_path = final_dir / f"{species_prefix}.target_reference_evidence.scope.json"
|
||
if not evidence_scope_path.exists():
|
||
write_scope(evidence_scope, evidence_scope_path)
|
||
evidence_scope_data = read_scope(evidence_scope_path)
|
||
evidence_sid = sanitize_token(str(evidence_scope.scope_id))
|
||
evidence_member_set = f"{evidence_sid}:subfamily_evidence:domain_refined"
|
||
evidence_output_prefix = stage_output_prefix(
|
||
"phylogeny",
|
||
species_prefix,
|
||
evidence_scope_data,
|
||
"subfamily_evidence",
|
||
evidence_member_set,
|
||
)
|
||
evidence_tree = final_dir / f"{evidence_output_prefix}.tree.nwk"
|
||
evidence_manifests = {
|
||
stage_name: first_existing_stage_manifest(
|
||
final_dir,
|
||
stage_name,
|
||
species_prefix,
|
||
evidence_scope_data,
|
||
"subfamily_evidence",
|
||
evidence_member_set,
|
||
)
|
||
for stage_name in ("phylogeny", "motif", "domain", "structure", "combo")
|
||
}
|
||
missing_evidence = [
|
||
stage_name for stage_name, manifest_path in evidence_manifests.items()
|
||
if not manifest_path or not manifest_path.exists()
|
||
]
|
||
if not evidence_tree.exists() or evidence_tree.stat().st_size <= 0:
|
||
missing_evidence.append("phylogeny_tree")
|
||
if missing_evidence:
|
||
aggregate_errors.append(
|
||
f"{species_prefix} reference_classification missing explicit upstream subfamily_evidence stages: "
|
||
f"{', '.join(dict.fromkeys(missing_evidence))}. "
|
||
f"Expected evidence tree: {evidence_tree}. "
|
||
"PlanningAgentV2 must schedule target+reference phylogeny/motif/domain/structure/combo with analysisRoute=subfamily_evidence first; "
|
||
"the regular target-only phylogeny tree is not a valid substitute."
|
||
)
|
||
continue
|
||
|
||
combo_manifest = evidence_manifests.get("combo")
|
||
compact_combo_outputs = materialize_compact_combo_outputs(final_dir, species_prefix, combo_manifest)
|
||
|
||
ref_scope = build_gene_family_scope(
|
||
input_dir=in_dir,
|
||
workspace=ws,
|
||
prefix=species_prefix,
|
||
family=family,
|
||
target_species=[species],
|
||
reference_species=reference_list,
|
||
comparative_species=[],
|
||
scope_mode="target_plus_reference",
|
||
)
|
||
ref_scope_path = final_dir / f"{species_prefix}.target_plus_reference.scope.json"
|
||
if not ref_scope_path.exists():
|
||
write_scope(ref_scope, ref_scope_path)
|
||
ref_scope_id = str(ref_scope.scope_id)
|
||
ref_sid = sanitize_token(ref_scope_id)
|
||
ref_member_set = f"{ref_sid}:subfamily_prediction:all_members"
|
||
ref_cmd = [
|
||
sys.executable,
|
||
str(script),
|
||
"--prefix", species_prefix,
|
||
"--workspace", str(ws),
|
||
"--input-dir", str(in_dir),
|
||
"--tree", str(evidence_tree),
|
||
"--target-prefix", species,
|
||
"--reference-prefix", reference_prefix or (reference_list[0] if reference_list else "At"),
|
||
"--log-file", str(log_file),
|
||
"--scope-file", str(ref_scope_path),
|
||
"--analysis-route", "subfamily_prediction",
|
||
"--member-set-id", ref_member_set,
|
||
]
|
||
if reference_subfamily_file:
|
||
ref_cmd.extend(["--reference-subfamily", reference_subfamily_file])
|
||
if reference_overwrite_gene_subfamily:
|
||
ref_cmd.append("--overwrite-gene-subfamily")
|
||
print(f"[CMD] {' '.join(ref_cmd)}")
|
||
ref_code = run_cmd_logged(ref_cmd, log_file, cwd=Path("."))
|
||
ref_scope_data = read_scope(ref_scope_path)
|
||
ref_manifest = first_existing_stage_manifest(
|
||
final_dir,
|
||
"reference_classification",
|
||
species_prefix,
|
||
ref_scope_data,
|
||
"subfamily_prediction",
|
||
ref_member_set,
|
||
)
|
||
if ref_code == 0 and ref_manifest:
|
||
result_item = {
|
||
"species": species,
|
||
"prefix": species_prefix,
|
||
"evidence_scope_file": str(evidence_scope_path),
|
||
"reference_scope_file": str(ref_scope_path),
|
||
"manifest": str(ref_manifest),
|
||
"tree": str(final_dir / f"{species_prefix}.target_plus_reference.reference_classification.tree.png"),
|
||
"circular_tree": str(final_dir / f"{species_prefix}.target_plus_reference.reference_classification.tree.circular_tree.png"),
|
||
}
|
||
if compact_combo_outputs.get(".png"):
|
||
result_item["evidence_combo"] = str(compact_combo_outputs[".png"])
|
||
if combo_manifest:
|
||
result_item["evidence_combo_manifest"] = str(combo_manifest)
|
||
aggregate_results.append(result_item)
|
||
else:
|
||
aggregate_errors.append(f"{species_prefix} reference_classification failed; log={log_file}")
|
||
|
||
|
||
if (
|
||
len(aggregate_results) == 1
|
||
and not aggregate_errors
|
||
and aggregate_results[0].get("prefix") == prefix
|
||
and aggregate_results[0].get("manifest")
|
||
):
|
||
return (
|
||
f"✅ Reference classification 完成 {prefix}。日志：{log_file}",
|
||
Path(aggregate_results[0]["manifest"]),
|
||
)
|
||
|
||
aggregate_output_prefix = stage_output_prefix(
|
||
"reference_classification",
|
||
prefix,
|
||
parent_scope_data,
|
||
"subfamily_prediction",
|
||
f"{reference_scope_id}:subfamily_prediction:all_members",
|
||
)
|
||
aggregate_manifest = final_dir / f"{aggregate_output_prefix}.reference_classification.manifest.json"
|
||
aggregate_payload = {
|
||
"manifest_version": "1.0",
|
||
"stage": "reference_classification",
|
||
"status": "success" if aggregate_results and not aggregate_errors else "failed",
|
||
"prefix": prefix,
|
||
"output_prefix": aggregate_output_prefix,
|
||
"legacy_output_prefix": f"{prefix}.{sanitize_token(reference_scope_id)}" if reference_scope_id else prefix,
|
||
"scope_id": reference_scope_id,
|
||
"scope_mode": parent_scope_data.get("scope_mode", ""),
|
||
"target_species": target_list,
|
||
"comparative_species": comparative_list,
|
||
"classifiable_species": classifiable_species,
|
||
"reference_species": reference_list,
|
||
"analysis_route": "subfamily_prediction",
|
||
"analysis_mode": "per_species",
|
||
"member_set_id": f"{reference_scope_id}:subfamily_prediction:all_members",
|
||
"result_group": "subfamily_confirmation",
|
||
"stage_context": {
|
||
"stage": "reference_classification",
|
||
"prefix": prefix,
|
||
"scope_id": reference_scope_id,
|
||
"scope_mode": parent_scope_data.get("scope_mode", ""),
|
||
"analysis_route": "subfamily_prediction",
|
||
"member_set_id": f"{reference_scope_id}:subfamily_prediction:all_members",
|
||
"result_group": "subfamily_confirmation",
|
||
"analysis_mode": "per_species",
|
||
"family": family,
|
||
"target_species": target_list,
|
||
"reference_species": reference_list,
|
||
"comparative_species": comparative_list,
|
||
"selected_species": [*target_list, *reference_list, *comparative_list],
|
||
},
|
||
"policy": "per_species; each classifiable species has its own tree/combo/review table",
|
||
"artifacts": {
|
||
f"{item['species']}_reference_classification_manifest": {
|
||
"path": f"final/{Path(item['manifest']).name}",
|
||
"type": "json",
|
||
}
|
||
for item in aggregate_results
|
||
},
|
||
"parameters": {
|
||
"analysis_route": "subfamily_prediction",
|
||
"member_set_id": f"{reference_scope_id}:subfamily_prediction:all_members",
|
||
"result_group": "subfamily_confirmation",
|
||
},
|
||
"metrics": {
|
||
"per_species_count": len(aggregate_results),
|
||
"classifiable_species_count": len(classifiable_species),
|
||
"failed_species_count": len(aggregate_errors),
|
||
"errors": aggregate_errors,
|
||
},
|
||
"per_species_results": aggregate_results,
|
||
"created_at": datetime.now().isoformat(),
|
||
}
|
||
aggregate_manifest.write_text(json.dumps(aggregate_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
if reference_scope_id:
|
||
legacy_aggregate_manifest = final_dir / f"{prefix}.{sanitize_token(reference_scope_id)}.reference_classification.manifest.json"
|
||
if legacy_aggregate_manifest != aggregate_manifest:
|
||
legacy_aggregate_manifest.write_text(aggregate_manifest.read_text(encoding="utf-8"), encoding="utf-8")
|
||
if aggregate_errors:
|
||
return f"❌ Reference classification 部分/全部物种失败: {'; '.join(aggregate_errors)}。日志：{log_file}", aggregate_manifest
|
||
return f"✅ Reference classification 按单物种完成 {len(aggregate_results)} 个待分类物种。日志：{log_file}", aggregate_manifest
|
||
if scope_path:
|
||
try:
|
||
tree_file, _scope_data_for_tree = resolve_scoped_tree_from_phylogeny(ws, final_dir, prefix, scope_path)
|
||
except Exception as exc:
|
||
return f"❌ Reference classification 找不到同 scope 的 phylogeny 树文件: {exc}", manifest
|
||
else:
|
||
tree_file = final_dir / f"{prefix}.tree.nwk"
|
||
if not tree_file.exists():
|
||
return "❌ Reference classification requires phylogeny results. Please run phylogeny first.", manifest
|
||
|
||
target_prefix_arg = reference_target_prefix or prefix.split(".")[0]
|
||
cmd = [
|
||
sys.executable,
|
||
str(script),
|
||
"--prefix",
|
||
prefix,
|
||
"--workspace",
|
||
str(ws),
|
||
"--input-dir",
|
||
str(in_dir),
|
||
"--tree",
|
||
str(tree_file),
|
||
"--target-prefix",
|
||
target_prefix_arg,
|
||
"--reference-prefix",
|
||
reference_prefix or "At",
|
||
"--log-file",
|
||
str(log_file),
|
||
]
|
||
if reference_subfamily_file:
|
||
cmd.extend(["--reference-subfamily", reference_subfamily_file])
|
||
if scope_path:
|
||
cmd.extend(["--scope-file", str(scope_path)])
|
||
if analysis_route:
|
||
cmd.extend(["--analysis-route", analysis_route])
|
||
if member_set_id:
|
||
cmd.extend(["--member-set-id", member_set_id])
|
||
if reference_overwrite_gene_subfamily:
|
||
cmd.append("--overwrite-gene-subfamily")
|
||
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
reference_scope_data = read_scope(scope_path) if scope_path else {}
|
||
reference_manifest = first_existing_stage_manifest(
|
||
final_dir,
|
||
"reference_classification",
|
||
prefix,
|
||
reference_scope_data,
|
||
analysis_route or "subfamily_prediction",
|
||
member_set_id,
|
||
) or (final_dir / f"{stage_output_prefix('reference_classification', prefix, reference_scope_data, analysis_route or 'subfamily_prediction', member_set_id)}.reference_classification.manifest.json")
|
||
if code == 0 and reference_manifest.exists():
|
||
return f"✅ Reference classification 完成。日志：{log_file}", reference_manifest
|
||
if reference_manifest.exists():
|
||
return f"❌ Reference classification 失败。日志：{log_file}", reference_manifest
|
||
return f"❌ Reference classification failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
if stage == "subfamily_confirm":
|
||
script = Path("apps/cli/run_subfamily_confirm.py")
|
||
if not script.exists():
|
||
return "❌ Subfamily confirm script not found: apps/cli/run_subfamily_confirm.py", manifest
|
||
|
||
cmd = [
|
||
sys.executable,
|
||
str(script),
|
||
"--prefix",
|
||
prefix,
|
||
"--workspace",
|
||
str(ws),
|
||
"--input-dir",
|
||
str(in_dir),
|
||
]
|
||
edited_file = (subfamily_confirm_file or str(kwargs.get("subfamilyConfirmFile") or "")).strip()
|
||
if edited_file:
|
||
cmd.extend(["--edited-subfamily", edited_file])
|
||
confirm_scope_data = {}
|
||
if scope_file:
|
||
cmd.extend(["--scope-file", scope_file])
|
||
try:
|
||
confirm_scope_data = read_scope(resolve_scope_path(scope_file, in_dir, ws))
|
||
except Exception:
|
||
confirm_scope_data = {}
|
||
source_prediction_route = str(kwargs.get("sourcePredictionRoute") or kwargs.get("source_prediction_route") or "").strip()
|
||
if not source_prediction_route and analysis_route and analysis_route != "confirmed_subfamily":
|
||
source_prediction_route = analysis_route
|
||
confirm_route = "confirmed_subfamily"
|
||
expected_member_set = member_set_id_for(prefix, confirm_scope_data, confirm_route) if confirm_scope_data.get("scope_id") else ""
|
||
if expected_member_set and (not member_set_id or f":{confirm_route}:" not in member_set_id):
|
||
member_set_id = expected_member_set
|
||
cmd.extend(["--analysis-route", confirm_route])
|
||
if member_set_id:
|
||
cmd.extend(["--member-set-id", member_set_id])
|
||
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
confirm_manifest = first_existing_stage_manifest(
|
||
final_dir, "subfamily_confirm", prefix, confirm_scope_data, confirm_route, member_set_id
|
||
) or (final_dir / f"{stage_output_prefix('subfamily_confirm', prefix, confirm_scope_data, confirm_route, member_set_id)}.subfamily_confirm.manifest.json")
|
||
if code == 0 and confirm_manifest.exists():
|
||
return f"✅ Subfamily confirm 完成。日志：{log_file}", confirm_manifest
|
||
if confirm_manifest.exists():
|
||
return f"❌ Subfamily confirm 失败。日志：{log_file}", confirm_manifest
|
||
return f"❌ Subfamily confirm failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
if stage == "structure":
|
||
# 检查GFF3来源：scope 多物种解析优先；无 scope 时沿用单文件上传逻辑
|
||
gff3_file = None
|
||
structure_scope_path = None
|
||
structure_tree_file = None
|
||
structure_scope_id = ""
|
||
if not scope_file:
|
||
# 1. 检查用户是否上传了GFF3
|
||
uploaded_gff3 = list(Path(job.input_dir).glob("*.gff3")) + list(Path(job.input_dir).glob("*.gff"))
|
||
if uploaded_gff3:
|
||
gff3_file = uploaded_gff3[0]
|
||
print(f"[INFO] Using uploaded GFF3: {gff3_file}")
|
||
else:
|
||
# 2. TODO: 从数据库/参考基因组提取
|
||
# 暂时报错
|
||
return "❌ Structure stage requires GFF3 file. Please upload genome annotation (GFF3).", manifest
|
||
else:
|
||
print(f"[INFO] Structure will resolve target GFF3 files from scope: {scope_file}")
|
||
structure_scope_path = resolve_scope_path(scope_file, in_dir, ws)
|
||
if not structure_scope_path.exists():
|
||
return f"❌ Structure scope 文件不存在: {scope_file}", manifest
|
||
try:
|
||
structure_tree_file, _structure_scope_data = resolve_scoped_tree_from_phylogeny(
|
||
ws, final_dir, prefix, structure_scope_path
|
||
)
|
||
structure_scope_id = str(_structure_scope_data.get("scope_id") or "")
|
||
except Exception as exc:
|
||
return f"❌ Structure 找不到同 scope 的 phylogeny 树文件: {exc}", manifest
|
||
if not structure_tree_file:
|
||
structure_tree_file = final_dir / f"{prefix}.tree.nwk"
|
||
if not structure_tree_file.exists():
|
||
return "❌ Structure stage requires phylogeny results. Please run /run phylogeny first.", manifest
|
||
|
||
script = Path("apps/cli/run_structure.py")
|
||
if not script.exists():
|
||
return "❌ Structure script not found", manifest
|
||
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--prefix", prefix,
|
||
"--workspace", str(ws),
|
||
"--cpu", str(cpu),
|
||
"--log-file", str(log_file)
|
||
]
|
||
if gff3_file:
|
||
cmd.extend(["--gff3", str(gff3_file)])
|
||
if structure_tree_file:
|
||
cmd.extend(["--tree", str(structure_tree_file)])
|
||
if structure_scope_path:
|
||
cmd.extend(["--scope-file", str(structure_scope_path)])
|
||
if analysis_route:
|
||
cmd.extend(["--analysis-route", analysis_route])
|
||
if member_set_id:
|
||
cmd.extend(["--member-set-id", member_set_id])
|
||
subfamily_file = None
|
||
if should_disable_cli_subfamily_auto(analysis_route, subfamily_label_source):
|
||
cmd.extend(["--subfamily", "NONE"])
|
||
else:
|
||
subfamily_file = resolve_subfamily_for_plotting(final_dir, in_dir, prefix, structure_scope_id, subfamily_label_source)
|
||
if subfamily_file:
|
||
cmd.extend(["--subfamily", str(subfamily_file)])
|
||
|
||
code = run_cmd_logged(cmd, log_file)
|
||
structure_scope_data_for_manifest = _structure_scope_data if structure_scope_id else {}
|
||
structure_manifest = first_existing_stage_manifest(
|
||
final_dir, "structure", prefix, structure_scope_data_for_manifest, analysis_route, member_set_id
|
||
) or (final_dir / f"{stage_output_prefix('structure', prefix, structure_scope_data_for_manifest, analysis_route, member_set_id)}.structure.manifest.json")
|
||
|
||
if code == 0 and structure_manifest.exists():
|
||
return f"✅ Structure visualization complete. Log: {log_file}", structure_manifest
|
||
if code == 0:
|
||
return f"✅ Structure visualization complete. Log: {log_file}", find_latest_manifest(ws)
|
||
else:
|
||
return f"❌ Structure failed. Log: {log_file}", structure_manifest if structure_manifest.exists() else manifest
|
||
########结束#########
|
||
|
||
if stage == "domain":
|
||
# Scoped domain runs use the same member set as scoped phylogeny/motif.
|
||
# Do not require a legacy bare {prefix}.identify.manifest.json: single-target
|
||
# members may come from Target.{species}.{family}.final_members.* or from the
|
||
# scoped phylogeny input FASTA recorded in {prefix}.{scope_id}.phylogeny.manifest.json.
|
||
domain_member_protein = None
|
||
if not scope_file:
|
||
domain_member_protein, member_error = resolve_member_protein_for_stage_route(
|
||
ws=ws,
|
||
final_dir=final_dir,
|
||
in_dir=in_dir,
|
||
prefix=prefix,
|
||
analysis_route=analysis_route,
|
||
member_set_id=member_set_id,
|
||
)
|
||
if member_error:
|
||
return member_error, manifest
|
||
|
||
phylogeny_manifest = final_dir / f"{prefix}.phylogeny.manifest.json"
|
||
if not phylogeny_manifest.exists():
|
||
return "❌ Domain stage requires phylogeny results. Please run phylogeny first.", None
|
||
|
||
pfam_base = Path(__file__).resolve().parents[2] / "data" / "hmm"
|
||
pfam_hmm = pfam_base / "Pfam-A.hmm"
|
||
if not pfam_hmm.exists():
|
||
return f"❌ Pfam database not found: {pfam_hmm}", None
|
||
|
||
required_indices = [".h3f", ".h3i", ".h3m", ".h3p"]
|
||
missing_indices = [f"Pfam-A.hmm{ext}" for ext in required_indices if not (pfam_base / f"Pfam-A.hmm{ext}").exists()]
|
||
if missing_indices:
|
||
return f"❌ Missing Pfam index files: {', '.join(missing_indices)}. Run 'hmmpress Pfam-A.hmm' first.", None
|
||
|
||
script = Path("apps/cli/run_domain.py")
|
||
if not script.exists():
|
||
return "❌ Domain script not found: apps/cli/run_domain.py", None
|
||
|
||
domain_scope_id = ""
|
||
|
||
domain_manifest = final_dir / f"{prefix}.domain.manifest.json"
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--prefix", prefix,
|
||
"--workspace", str(ws),
|
||
"--cpu", str(cpu),
|
||
"--evalue", str(dom_evalue),
|
||
"--log-file", str(log_file),
|
||
]
|
||
if scope_file:
|
||
scope_path = Path(scope_file).expanduser()
|
||
if not scope_path.is_absolute():
|
||
for candidate in (in_dir / scope_path, ws / scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
scope_path = candidate
|
||
break
|
||
if not scope_path.exists():
|
||
return f"❌ Domain scope 文件不存在: {scope_file}", manifest
|
||
cmd.extend(["--scope-file", str(scope_path)])
|
||
if analysis_route:
|
||
cmd.extend(["--analysis-route", analysis_route])
|
||
if member_set_id:
|
||
cmd.extend(["--member-set-id", member_set_id])
|
||
try:
|
||
domain_tree_file, _domain_scope_data = resolve_scoped_tree_from_phylogeny(ws, final_dir, prefix, scope_path)
|
||
cmd.extend(["--tree", str(domain_tree_file)])
|
||
domain_scope_id = str(_domain_scope_data.get("scope_id") or "")
|
||
except Exception as exc:
|
||
return f"❌ Domain 找不到同 scope 的 phylogeny 树文件: {exc}", manifest
|
||
else:
|
||
protein_file = domain_member_protein
|
||
if protein_file:
|
||
cmd.extend(["--proteins", str(protein_file)])
|
||
subfamily_file = None
|
||
if should_disable_cli_subfamily_auto(analysis_route, subfamily_label_source):
|
||
cmd.extend(["--subfamily", "NONE"])
|
||
else:
|
||
subfamily_file = resolve_subfamily_for_plotting(final_dir, in_dir, prefix, domain_scope_id, subfamily_label_source)
|
||
if subfamily_file:
|
||
cmd.extend(["--subfamily", str(subfamily_file)])
|
||
if dom_coverage is not None:
|
||
cmd.extend(["--coverage", str(dom_coverage)])
|
||
domain_scope_data_for_manifest = _domain_scope_data if domain_scope_id else {}
|
||
if domain_scope_id:
|
||
domain_manifest = final_dir / f"{stage_output_prefix('domain', prefix, domain_scope_data_for_manifest, analysis_route, member_set_id)}.domain.manifest.json"
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
domain_manifest = first_existing_stage_manifest(
|
||
final_dir, "domain", prefix, domain_scope_data_for_manifest, analysis_route, member_set_id
|
||
) or domain_manifest
|
||
|
||
if domain_manifest.exists():
|
||
if code == 0:
|
||
return f"Domain complete. Log: {log_file}", domain_manifest
|
||
return f"Domain failed. Log: {log_file}", domain_manifest
|
||
if code == 0:
|
||
return f"Domain finished but manifest not found. Log: {log_file}", manifest
|
||
return f"Domain failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
# if stage == "msa_figure":
|
||
# cmd = ["python", "apps/cli/run_msa_figure.py", "--manifest", str(manifest)]
|
||
# code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
# if code != 0:
|
||
# return f"❌ MSA Figure 失败，日志：{log_file}", manifest
|
||
# return f"✅ MSA Figure 完成。日志：{log_file}", manifest
|
||
|
||
if stage == "structure":
|
||
# 查找subfamily文件
|
||
subfamily_file = resolve_subfamily_for_plotting(final_dir, in_dir, prefix, label_source=subfamily_label_source)
|
||
cmd = [sys.executable, "apps/cli/run_structure.py", "--manifest", str(manifest), "--cpu", str(cpu)]
|
||
if subfamily_file and subfamily_file.exists():
|
||
cmd += ["--subfamily", str(subfamily_file)]
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
if code != 0:
|
||
return f"❌ Structure 失败，日志：{log_file}", manifest
|
||
return f"✅ Structure 完成。日志：{log_file}", manifest
|
||
|
||
if stage == "motif":
|
||
motif_scope_id = ""
|
||
motif_manifest = final_dir / f"{prefix}.motif.manifest.json"
|
||
candidate_manifests = [
|
||
final_dir / f"{prefix}.pep.manifest.json",
|
||
final_dir / f"{prefix}.phylogeny.manifest.json",
|
||
final_dir / f"{prefix}.identify.manifest.json",
|
||
]
|
||
input_manifest = next((p for p in candidate_manifests if p.exists()), manifest)
|
||
cmd = [sys.executable, "apps/cli/run_motif.py", "--manifest", str(input_manifest), "--cpu", str(cpu)]
|
||
# Advanced parameters
|
||
cmd += ["--nmotifs", str(motif_nmotifs), "--minw", str(motif_minw), "--maxw", str(motif_maxw)]
|
||
if scope_file:
|
||
scope_path = Path(scope_file).expanduser()
|
||
if not scope_path.is_absolute():
|
||
for candidate in (in_dir / scope_path, ws / scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
scope_path = candidate
|
||
break
|
||
if not scope_path.exists():
|
||
return f"❌ Motif scope 文件不存在: {scope_file}", manifest
|
||
cmd += ["--scope-file", str(scope_path)]
|
||
if analysis_route:
|
||
cmd += ["--analysis-route", analysis_route]
|
||
if member_set_id:
|
||
cmd += ["--member-set-id", member_set_id]
|
||
try:
|
||
motif_tree_file, _motif_scope_data = resolve_scoped_tree_from_phylogeny(ws, final_dir, prefix, scope_path)
|
||
cmd += ["--tree", str(motif_tree_file)]
|
||
motif_scope_id = str(_motif_scope_data.get("scope_id") or "")
|
||
except Exception as exc:
|
||
return f"❌ Motif 找不到同 scope 的 phylogeny 树文件: {exc}", manifest
|
||
else:
|
||
protein_file, member_error = resolve_member_protein_for_stage_route(
|
||
ws=ws,
|
||
final_dir=final_dir,
|
||
in_dir=in_dir,
|
||
prefix=prefix,
|
||
analysis_route=analysis_route,
|
||
member_set_id=member_set_id,
|
||
)
|
||
if member_error:
|
||
return member_error, manifest
|
||
if protein_file:
|
||
cmd += ["--fasta", str(protein_file)]
|
||
subfamily_file = None
|
||
if should_disable_cli_subfamily_auto(analysis_route, subfamily_label_source):
|
||
cmd += ["--subfamily", "NONE"]
|
||
else:
|
||
subfamily_file = resolve_subfamily_for_plotting(final_dir, in_dir, prefix, motif_scope_id, subfamily_label_source)
|
||
if subfamily_file:
|
||
cmd += ["--subfamily", str(subfamily_file)]
|
||
motif_scope_data_for_manifest = _motif_scope_data if motif_scope_id else {}
|
||
if motif_scope_id:
|
||
motif_manifest = final_dir / f"{stage_output_prefix('motif', prefix, motif_scope_data_for_manifest, analysis_route, member_set_id)}.motif.manifest.json"
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
motif_manifest = first_existing_stage_manifest(
|
||
final_dir, "motif", prefix, motif_scope_data_for_manifest, analysis_route, member_set_id
|
||
) or motif_manifest
|
||
if code == 0 and motif_manifest.exists():
|
||
return f"Motif complete. Log: {log_file}", motif_manifest
|
||
if motif_manifest.exists():
|
||
return f"Motif failed. Log: {log_file}", motif_manifest
|
||
return f"Motif failed. Log: {log_file}", manifest
|
||
if stage == "pep":
|
||
# PEP 阶段 - 蛋白特征分析 (技能化版本)
|
||
# 输入：scope模式优先复用同一scope的phylogeny蛋白集合；非scope模式沿用identify输出
|
||
pep_scope_path = None
|
||
pep_scope_data = None
|
||
if scope_file:
|
||
pep_scope_path = resolve_scope_path(scope_file, in_dir, ws)
|
||
if not pep_scope_path.exists():
|
||
return f"❌ PEP scope 文件不存在: {scope_file}", manifest
|
||
try:
|
||
protein_file, pep_scope_data = resolve_scoped_protein_fasta_from_phylogeny(ws, final_dir, prefix, pep_scope_path, in_dir=in_dir)
|
||
except Exception as exc:
|
||
return f"❌ PEP stage cannot resolve scoped protein FASTA: {exc}", manifest
|
||
else:
|
||
protein_file = final_dir / f"{prefix}.final_members.pep.fa"
|
||
if not protein_file.exists():
|
||
return "❌ PEP stage requires protein FASTA file. Please run Identify stage first.", manifest
|
||
protein_file = ensure_protein_fasta_without_stop_markers(protein_file, final_dir, prefix)
|
||
|
||
# PEP 的完整发表级流程固定使用 apps/cli/run_pep.py：
|
||
# 理化性质 + DeepTMHMM + WoLF PSORT。Plant-mPLoc代码保留但默认关闭；旧 skills/pep 已移入备份，避免误走轻量实现。
|
||
cli_script = Path("apps/cli/run_pep.py")
|
||
if not cli_script.exists():
|
||
return "❌ PEP script not found: apps/cli/run_pep.py", manifest
|
||
script = cli_script
|
||
use_skill = False
|
||
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--input-dir", str(in_dir),
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix,
|
||
"--fasta", str(protein_file),
|
||
"--cpu", str(cpu),
|
||
"--log-file", str(log_file),
|
||
]
|
||
if pep_scope_path:
|
||
cmd += ["--scope-file", str(pep_scope_path)]
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
# 检查输出结果
|
||
pep_manifest = final_dir / f"{prefix}.pep.manifest.json"
|
||
if code == 0 and pep_scope_path and pep_scope_data:
|
||
annotate_pep_manifest_with_scope(pep_manifest, prefix, protein_file, pep_scope_path, pep_scope_data)
|
||
|
||
if code == 0 and pep_manifest.exists():
|
||
if pep_scope_path and pep_scope_data:
|
||
scoped_pep_manifest = final_dir / f"{prefix}.{sanitize_token(str(pep_scope_data.get('scope_id')))}.pep.manifest.json"
|
||
if scoped_pep_manifest.exists():
|
||
pep_manifest = scoped_pep_manifest
|
||
annotate_pep_manifest_with_scope(
|
||
pep_manifest, prefix, protein_file, pep_scope_path, pep_scope_data,
|
||
{"analysis_route": analysis_route, "member_set_id": member_set_id} if (analysis_route or member_set_id) else None,
|
||
)
|
||
return f"✅ PEP analysis complete. Log: {log_file}", pep_manifest
|
||
elif code == 0:
|
||
return f"⚠️ PEP finished but manifest not found. Log: {log_file}", manifest
|
||
else:
|
||
return f"❌ PEP failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
if stage == "summary":
|
||
summary_manifest_seed = (
|
||
ws / "final" / f"{prefix}.identify.manifest.json"
|
||
if (ws / "final" / f"{prefix}.identify.manifest.json").exists()
|
||
else find_latest_manifest(ws)
|
||
)
|
||
cmd = [sys.executable, "apps/cli/run_summary.py", "--workspace", str(ws), "--prefix", prefix]
|
||
if summary_manifest_seed and summary_manifest_seed.exists():
|
||
cmd += ["--manifest", str(summary_manifest_seed)]
|
||
summary_manifest = ws / "final" / f"{prefix}.summary.manifest.json"
|
||
if scope_file:
|
||
summary_scope_path = resolve_scope_path(scope_file, in_dir, ws)
|
||
if not summary_scope_path.exists():
|
||
return f"❌ Summary scope 文件不存在: {scope_file}", manifest
|
||
try:
|
||
summary_scope_data = read_scope(summary_scope_path)
|
||
summary_scope_id = str(summary_scope_data.get("scope_id") or "")
|
||
if summary_scope_id:
|
||
summary_manifest = ws / "final" / f"{prefix}.{sanitize_token(summary_scope_id)}.summary.manifest.json"
|
||
except Exception as exc:
|
||
return f"❌ Summary scope 文件无法读取: {scope_file}; {exc}", manifest
|
||
cmd += ["--scope-file", str(summary_scope_path)]
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
if code != 0:
|
||
return f"❌ Summary 失败，日志：{log_file}", summary_manifest if summary_manifest.exists() else manifest
|
||
return f"✅ Summary 完成。日志：{log_file}", summary_manifest if summary_manifest.exists() else manifest
|
||
|
||
# ========== TFBS Stage (MEME/FIMO) ==========
|
||
if stage == "tfbs":
|
||
tfbs_scope_path = None
|
||
tfbs_scope_data = {}
|
||
if scope_file:
|
||
tfbs_scope_path = Path(scope_file).expanduser()
|
||
if not tfbs_scope_path.exists():
|
||
for candidate in (in_dir / tfbs_scope_path, ws / tfbs_scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
tfbs_scope_path = candidate
|
||
break
|
||
if not tfbs_scope_path.exists():
|
||
return f"❌ TFBS scope 文件不存在: {scope_file}", manifest
|
||
try:
|
||
tfbs_scope_data = read_scope(tfbs_scope_path)
|
||
except Exception as exc:
|
||
return f"❌ TFBS scope 文件无法读取: {scope_file}; {exc}", manifest
|
||
|
||
script = Path("apps/cli/run_tfbs.py")
|
||
if not script.exists():
|
||
return "❌ TFBS script not found: apps/cli/run_tfbs.py", manifest
|
||
|
||
def _tfbs_int(value, default, min_value=1):
|
||
try:
|
||
return max(min_value, int(value))
|
||
except (TypeError, ValueError):
|
||
return default
|
||
|
||
def _tfbs_float(value, default):
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return default
|
||
|
||
motif_mode = str(kwargs.get("tfbsMotifDbMode") or kwargs.get("tfbs_motif_db_mode") or tfbs_motif_db_mode or "auto").strip().lower()
|
||
if motif_mode not in {"auto", "planttfdb", "jaspar", "both"}:
|
||
motif_mode = "auto"
|
||
background_mode = str(kwargs.get("tfbsBackgroundMode") or kwargs.get("tfbs_background_mode") or tfbs_background_mode or "none").strip().lower()
|
||
if background_mode not in {"all_annotated_promoters", "family_promoters", "none"}:
|
||
background_mode = "none"
|
||
max_rank = str(kwargs.get("tfbsMaxNearestRank") or kwargs.get("tfbs_max_nearest_rank") or tfbs_max_nearest_rank or "order").strip().lower()
|
||
upstream_len = _tfbs_int(kwargs.get("tfbsUpstreamLength") or kwargs.get("tfbs_upstream_length") or tfbs_upstream_length, 2000)
|
||
top_motifs = _tfbs_int(kwargs.get("tfbsTopMotifs") or kwargs.get("tfbs_top_motifs") or tfbs_top_motifs, 30)
|
||
pvalue = _tfbs_float(kwargs.get("tfbsFimoPvalue") or kwargs.get("tfbs_fimo_pvalue") or tfbs_fimo_pvalue, 1e-4)
|
||
qvalue = _tfbs_float(kwargs.get("tfbsFimoQvalue") or kwargs.get("tfbs_fimo_qvalue") or tfbs_fimo_qvalue, 0.05)
|
||
scan_both = kwargs.get("tfbsScanBothStrands")
|
||
if scan_both is None:
|
||
scan_both = kwargs.get("tfbs_scan_both_strands", tfbs_scan_both_strands)
|
||
scan_both = bool(scan_both)
|
||
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--input-dir", str(in_dir),
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix,
|
||
"--cpu", str(cpu),
|
||
"--upstream-length", str(upstream_len),
|
||
"--motif-db-mode", motif_mode,
|
||
"--fimo-pvalue", str(pvalue),
|
||
"--fimo-qvalue", str(qvalue),
|
||
"--background-mode", background_mode,
|
||
"--max-nearest-rank", max_rank,
|
||
"--top-motifs", str(top_motifs),
|
||
"--analysis-route", analysis_route,
|
||
"--member-set-id", member_set_id,
|
||
]
|
||
if not scan_both:
|
||
cmd.append("--no-scan-both-strands")
|
||
if tfbs_scope_path:
|
||
cmd.extend(["--scope-file", str(tfbs_scope_path)])
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
tfbs_scope_id = str(tfbs_scope_data.get("scope_id") or "") if tfbs_scope_data else ""
|
||
tfbs_manifest = ws / "final" / f"{prefix}.tfbs.manifest.json"
|
||
if tfbs_scope_id:
|
||
scoped_tfbs_manifest = ws / "final" / f"{prefix}.{sanitize_token(tfbs_scope_id)}.tfbs.manifest.json"
|
||
if scoped_tfbs_manifest.exists():
|
||
tfbs_manifest = scoped_tfbs_manifest
|
||
if code == 0 and tfbs_manifest.exists():
|
||
if tfbs_scope_path and tfbs_scope_data:
|
||
annotate_stage_manifest_with_scope(
|
||
tfbs_manifest, prefix, "tfbs", tfbs_scope_path, tfbs_scope_data,
|
||
{"analysis_route": analysis_route, "member_set_id": member_set_id} if (analysis_route or member_set_id) else None,
|
||
)
|
||
return f"✅ TFBS prediction complete. Log: {log_file}", tfbs_manifest
|
||
if code == 0:
|
||
return f"⚠️ TFBS finished but manifest not found. Log: {log_file}", manifest
|
||
return f"❌ TFBS failed (exit code: {code}). Log: {log_file}", tfbs_manifest if tfbs_manifest.exists() else manifest
|
||
|
||
# ========== Promoter Stage (PlantCARE) ==========
|
||
if stage == "promoter":
|
||
promoter_scope_path = None
|
||
promoter_scope_data = {}
|
||
if scope_file:
|
||
promoter_scope_path = Path(scope_file).expanduser()
|
||
if not promoter_scope_path.exists():
|
||
for candidate in (in_dir / promoter_scope_path, ws / promoter_scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
promoter_scope_path = candidate
|
||
break
|
||
if not promoter_scope_path.exists():
|
||
return f"❌ Promoter scope 文件不存在: {scope_file}", manifest
|
||
try:
|
||
promoter_scope_data = read_scope(promoter_scope_path)
|
||
except Exception as exc:
|
||
return f"❌ Promoter scope 文件无法读取: {scope_file}; {exc}", manifest
|
||
else:
|
||
# Promoter 阶段需要 GFF3 和基因组文件；无 scope 时保持 legacy 自动检测。
|
||
gff3_files = list(in_dir.glob("*.gff3")) + list(in_dir.glob("*.gff"))
|
||
genome_files = []
|
||
for p in in_dir.iterdir():
|
||
if p.is_file():
|
||
n = p.name.lower()
|
||
if n.endswith(('.fasta', '.fa')) and 'genome' in n:
|
||
genome_files.append(p)
|
||
|
||
if not gff3_files:
|
||
return "❌ Promoter stage requires GFF3 file. Please upload genome annotation (GFF3).", manifest
|
||
|
||
if not genome_files:
|
||
# 尝试排除蛋白和pep文件
|
||
for p in in_dir.iterdir():
|
||
if p.is_file() and p.suffix.lower() in ('.fasta', '.fa'):
|
||
n = p.name.lower()
|
||
if 'pep' not in n and 'protein' not in n and 'cds' not in n:
|
||
if 'ath' not in n and 'arabidopsis' not in n:
|
||
genome_files.append(p)
|
||
|
||
if not genome_files:
|
||
return "❌ Promoter stage requires genome FASTA file. Please upload genome sequence.", manifest
|
||
|
||
gff3_file = gff3_files[0]
|
||
genome_file = genome_files[0]
|
||
|
||
# 查找树文件（用于可视化）。Promoter 图是 tree + cis-element
|
||
# 组合图，scope 模式下必须等同 scope phylogeny 可用后再执行。
|
||
tree_file = ws / "final" / f"{prefix}.tree.nwk"
|
||
if promoter_scope_path:
|
||
try:
|
||
tree_file, _promoter_scope_data = resolve_scoped_tree_from_phylogeny(
|
||
ws, final_dir, prefix, promoter_scope_path
|
||
)
|
||
except Exception as exc:
|
||
return (
|
||
"❌ Promoter requires the same-scope phylogeny tree for visualization. "
|
||
f"Run phylogeny first or fix the scoped phylogeny manifest. Detail: {exc}"
|
||
), manifest
|
||
elif not tree_file.exists():
|
||
return (
|
||
"❌ Promoter requires phylogeny tree for visualization. "
|
||
f"Missing: {tree_file}. Run phylogeny first."
|
||
), manifest
|
||
|
||
# 查找亚家族文件（可选）
|
||
promoter_scope_id = str(promoter_scope_data.get("scope_id") or "") if promoter_scope_data else ""
|
||
subfamily_file = resolve_subfamily_for_plotting(ws / "final", in_dir, prefix, promoter_scope_id, subfamily_label_source)
|
||
|
||
script = Path("apps/cli/run_promoter.py")
|
||
if not script.exists():
|
||
return "❌ Promoter script not found: apps/cli/run_promoter.py", manifest
|
||
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--input-dir", str(in_dir),
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix,
|
||
"--cpu", str(cpu),
|
||
"--log-file", str(log_file),
|
||
"--force", # 强制重新提交，不使用缓存结果
|
||
]
|
||
if promoter_scope_path:
|
||
cmd.extend(["--scope-file", str(promoter_scope_path)])
|
||
else:
|
||
cmd.extend(["--gff3", str(gff3_file), "--genome", str(genome_file)])
|
||
|
||
if tree_file.exists():
|
||
cmd.extend(["--tree", str(tree_file)])
|
||
|
||
if subfamily_file.exists():
|
||
cmd.extend(["--subfamily", str(subfamily_file)])
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
# 检查输出结果
|
||
promoter_manifest = ws / "final" / f"{prefix}.promoter.manifest.json"
|
||
if promoter_scope_path and promoter_scope_data.get("scope_id"):
|
||
scoped_promoter_manifest = ws / "final" / f"{prefix}.{sanitize_token(str(promoter_scope_data.get('scope_id')))}.promoter.manifest.json"
|
||
if scoped_promoter_manifest.exists():
|
||
promoter_manifest = scoped_promoter_manifest
|
||
if code == 0 and promoter_manifest.exists():
|
||
if promoter_scope_path and promoter_scope_data:
|
||
annotate_stage_manifest_with_scope(
|
||
promoter_manifest, prefix, "promoter", promoter_scope_path, promoter_scope_data,
|
||
{"analysis_route": analysis_route, "member_set_id": member_set_id} if (analysis_route or member_set_id) else None,
|
||
)
|
||
return f"✅ Promoter analysis complete. Log: {log_file}", promoter_manifest
|
||
elif code == 0:
|
||
return f"⚠️ Promoter finished but manifest not found. Log: {log_file}", manifest
|
||
else:
|
||
return f"❌ Promoter failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
# ========== Synteny 阶段 ==========
|
||
if stage == "synteny":
|
||
synteny_scope_path = None
|
||
if scope_file:
|
||
synteny_scope_path = Path(scope_file).expanduser()
|
||
if not synteny_scope_path.exists():
|
||
for candidate in (in_dir / synteny_scope_path, ws / synteny_scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
synteny_scope_path = candidate
|
||
break
|
||
if not synteny_scope_path.exists():
|
||
return f"❌ Synteny scope 文件不存在: {scope_file}", manifest
|
||
else:
|
||
# Synteny 阶段需要 GFF3 和基因组文件；无 scope 时保持 legacy 自动检测。
|
||
gff3_files = list(in_dir.glob("*.gff3")) + list(in_dir.glob("*.gff"))
|
||
genome_files = []
|
||
for p in in_dir.iterdir():
|
||
if p.is_file():
|
||
n = p.name.lower()
|
||
if n.endswith(('.fasta', '.fa')) and ('genome' in n or 'chromosome' in n):
|
||
genome_files.append(p)
|
||
|
||
if not gff3_files:
|
||
return "❌ Synteny stage requires GFF3 file. Please upload genome annotation (GFF3).", manifest
|
||
|
||
if not genome_files:
|
||
for p in in_dir.iterdir():
|
||
if p.is_file() and p.suffix.lower() in ('.fasta', '.fa'):
|
||
n = p.name.lower()
|
||
if 'pep' not in n and 'protein' not in n and 'cds' not in n:
|
||
genome_files.append(p)
|
||
|
||
if not genome_files:
|
||
return "❌ Synteny stage requires genome FASTA file. Please upload genome sequence.", manifest
|
||
|
||
gff3_file = gff3_files[0]
|
||
genome_file = genome_files[0]
|
||
|
||
# 查找基因ID文件（可选）
|
||
gene_id_files = list(in_dir.glob("*geneID*"))
|
||
gene_id_file = gene_id_files[0] if gene_id_files else None
|
||
|
||
script = Path("apps/cli/run_synteny.py")
|
||
if not script.exists():
|
||
return "❌ Synteny script not found: apps/cli/run_synteny.py", manifest
|
||
|
||
if synteny_scope_path:
|
||
try:
|
||
synteny_scope_data_for_split = read_scope(synteny_scope_path)
|
||
except Exception as exc:
|
||
return f"❌ Synteny scope 文件无法读取: {scope_file}; {exc}", manifest
|
||
synteny_targets = [str(x) for x in synteny_scope_data_for_split.get("target_species", []) if str(x)]
|
||
if len(synteny_targets) > 1:
|
||
scope_dir = ws / "gene_family_scopes"
|
||
scope_dir.mkdir(parents=True, exist_ok=True)
|
||
runs = []
|
||
success_count = 0
|
||
for target in synteny_targets:
|
||
child_scope = dict(synteny_scope_data_for_split)
|
||
child_scope["target_species"] = [target]
|
||
child_scope["selected_species"] = [target]
|
||
family = str(child_scope.get("family") or (prefix.split(".", 1)[1] if "." in prefix else prefix))
|
||
refs = [str(x) for x in child_scope.get("reference_species", []) if str(x)]
|
||
comps = [str(x) for x in child_scope.get("comparative_species", []) if str(x)]
|
||
scope_parts = [f"{family}.target-{target}"]
|
||
if refs:
|
||
scope_parts.append("ref-" + "-".join(refs))
|
||
if comps:
|
||
scope_parts.append("comp-" + "-".join(comps))
|
||
scope_parts.append(str(child_scope.get("scope_mode") or "target_only"))
|
||
child_scope["scope_id"] = ".".join(scope_parts)
|
||
child_scope_path = scope_dir / f"{prefix}.{sanitize_token(child_scope['scope_id'])}.scope.json"
|
||
child_scope_path.write_text(json.dumps(child_scope, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
child_log = log_file.with_name(f"{log_file.stem}_{sanitize_token(target)}{log_file.suffix}")
|
||
child_cmd = [
|
||
sys.executable, str(script),
|
||
"--input-dir", str(in_dir),
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix,
|
||
"--cpu", str(cpu),
|
||
"--log-file", str(child_log),
|
||
"--scope-file", str(child_scope_path),
|
||
]
|
||
print(f"[CMD] {' '.join(child_cmd)}")
|
||
child_code = run_cmd_logged(child_cmd, child_log, cwd=Path("."))
|
||
child_manifest = ws / "final" / f"{prefix}.{sanitize_token(child_scope['scope_id'])}.synteny.manifest.json"
|
||
if child_code == 0:
|
||
success_count += 1
|
||
runs.append({
|
||
"target_species": target,
|
||
"scope_file": str(child_scope_path),
|
||
"scope_id": child_scope["scope_id"],
|
||
"returncode": child_code,
|
||
"status": "success" if child_code == 0 else "failed",
|
||
"log": str(child_log),
|
||
"manifest": str(child_manifest) if child_manifest.exists() else "",
|
||
})
|
||
original_scope_id = str(synteny_scope_data_for_split.get("scope_id") or "")
|
||
aggregate_manifest = ws / "final" / f"{prefix}.{sanitize_token(original_scope_id)}.synteny.manifest.json" if original_scope_id else ws / "final" / f"{prefix}.synteny.manifest.json"
|
||
aggregate_status = "success" if success_count == len(runs) else ("partial_success" if success_count else "failed")
|
||
aggregate_data = {
|
||
"manifest_version": "1.0",
|
||
"stage": "synteny",
|
||
"status": aggregate_status,
|
||
"prefix": prefix,
|
||
"job_id": job.job_id,
|
||
"workspace": str(ws),
|
||
"workspace_root": str(ws),
|
||
"created_at": datetime.now().isoformat(),
|
||
"scope_file": str(synteny_scope_path),
|
||
"scope_id": original_scope_id,
|
||
"scope_mode": synteny_scope_data_for_split.get("scope_mode", ""),
|
||
"target_species": synteny_scope_data_for_split.get("target_species", []),
|
||
"reference_species": synteny_scope_data_for_split.get("reference_species", []),
|
||
"comparative_species": synteny_scope_data_for_split.get("comparative_species", []),
|
||
"selected_species": synteny_scope_data_for_split.get("selected_species", []),
|
||
"family": synteny_scope_data_for_split.get("family", ""),
|
||
"parameters": {
|
||
"prefix": prefix,
|
||
"scope_file": str(synteny_scope_path),
|
||
"split_multi_target": True,
|
||
},
|
||
"metrics": {
|
||
"species_runs": len(runs),
|
||
"success_count": success_count,
|
||
"failure_count": len(runs) - success_count,
|
||
},
|
||
"runs": runs,
|
||
"artifacts": {
|
||
f"{run['target_species']}_manifest": {
|
||
"artifact": "synteny_manifest",
|
||
"species": run["target_species"],
|
||
"path": str(Path(run["manifest"]).relative_to(ws)) if run["manifest"] and Path(run["manifest"]).exists() else "",
|
||
"full_path": run["manifest"],
|
||
"type": "json",
|
||
}
|
||
for run in runs
|
||
if run.get("manifest")
|
||
},
|
||
}
|
||
aggregate_manifest.write_text(json.dumps(aggregate_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
if success_count == len(runs):
|
||
return f"✅ Synteny analysis complete for {success_count}/{len(runs)} target species. Log: {log_file}", aggregate_manifest
|
||
if success_count:
|
||
return f"⚠️ Synteny partial success: {success_count}/{len(runs)} target species. See child logs.", aggregate_manifest
|
||
return f"❌ Synteny failed for all {len(runs)} target species. See child logs.", aggregate_manifest
|
||
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--input-dir", str(in_dir),
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix,
|
||
"--cpu", str(cpu),
|
||
"--log-file", str(log_file),
|
||
]
|
||
if synteny_scope_path:
|
||
cmd.extend(["--scope-file", str(synteny_scope_path)])
|
||
else:
|
||
cmd.extend(["--gff3", str(gff3_file), "--genome", str(genome_file)])
|
||
|
||
if gene_id_file:
|
||
cmd.extend(["--gene-id-file", str(gene_id_file)])
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
# 检查输出结果
|
||
synteny_manifest = ws / "final" / f"{prefix}.synteny.manifest.json"
|
||
if synteny_scope_path:
|
||
try:
|
||
synteny_scope_data = read_scope(synteny_scope_path)
|
||
synteny_scope_id = str(synteny_scope_data.get("scope_id") or "")
|
||
if synteny_scope_id:
|
||
scoped_synteny_manifest = ws / "final" / f"{prefix}.{sanitize_token(synteny_scope_id)}.synteny.manifest.json"
|
||
if scoped_synteny_manifest.exists():
|
||
synteny_manifest = scoped_synteny_manifest
|
||
except Exception:
|
||
pass
|
||
|
||
if code == 0 and synteny_manifest.exists():
|
||
if synteny_scope_path:
|
||
try:
|
||
synteny_scope_data_for_annotate = read_scope(synteny_scope_path)
|
||
annotate_stage_manifest_with_scope(
|
||
synteny_manifest, prefix, "synteny", synteny_scope_path, synteny_scope_data_for_annotate,
|
||
{"analysis_route": analysis_route, "member_set_id": member_set_id} if (analysis_route or member_set_id) else None,
|
||
)
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to annotate synteny manifest with scope identity: {exc}")
|
||
return f"✅ Synteny analysis complete. Log: {log_file}", synteny_manifest
|
||
elif code == 0:
|
||
return f"⚠️ Synteny finished but manifest not found. Log: {log_file}", manifest
|
||
else:
|
||
return f"❌ Synteny failed (exit code: {code}). Log: {log_file}", synteny_manifest if synteny_manifest.exists() else manifest
|
||
|
||
# ========== Karyotype 阶段 ==========
|
||
if stage == "karyotype":
|
||
karyotype_scope_path = None
|
||
karyotype_scope_data = {}
|
||
if scope_file:
|
||
karyotype_scope_path = Path(scope_file).expanduser()
|
||
if not karyotype_scope_path.exists():
|
||
for candidate in (in_dir / karyotype_scope_path, ws / karyotype_scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
karyotype_scope_path = candidate
|
||
break
|
||
if not karyotype_scope_path.exists():
|
||
return f"❌ Karyotype scope 文件不存在: {scope_file}", manifest
|
||
try:
|
||
karyotype_scope_data = read_scope(karyotype_scope_path)
|
||
except Exception as exc:
|
||
return f"❌ Karyotype scope 文件无法读取: {scope_file}; {exc}", manifest
|
||
else:
|
||
raw_targets = target_species or kwargs.get("targetSpecies") or kwargs.get("target_species") or []
|
||
if isinstance(raw_targets, str):
|
||
targets = [x.strip() for x in raw_targets.replace("，", ",").split(",") if x.strip()]
|
||
elif isinstance(raw_targets, (list, tuple, set)):
|
||
targets = [str(x).strip() for x in raw_targets if str(x).strip()]
|
||
else:
|
||
targets = []
|
||
if targets:
|
||
karyotype_scope_data = {
|
||
"family": kwargs.get("geneFamily") or kwargs.get("gene_family") or (prefix.split(".")[-1] if "." in prefix else prefix),
|
||
"scope_mode": kwargs.get("scopeMode") or kwargs.get("scope_mode") or "target_only",
|
||
"target_species": targets,
|
||
}
|
||
|
||
identify_manifest_file = ws / "final" / f"{prefix}.identify.manifest.json"
|
||
gene_id_file = resolve_single_target_gene_id_file(ws, in_dir, final_dir, prefix.split(".", 1)[0], prefix)
|
||
if not identify_manifest_file.exists():
|
||
target_species_for_scope = [str(x).strip() for x in (karyotype_scope_data.get("target_species") or []) if str(x).strip()]
|
||
if len(target_species_for_scope) == 1:
|
||
target = target_species_for_scope[0]
|
||
gene_id_file = resolve_single_target_gene_id_file(ws, in_dir, final_dir, target, prefix)
|
||
family = str(karyotype_scope_data.get("family") or (prefix.split(".")[-1] if "." in prefix else prefix))
|
||
matching_manifest = find_identify_manifest_for_target(final_dir, prefix, target, family)
|
||
if matching_manifest:
|
||
identify_manifest_file = matching_manifest
|
||
if not identify_manifest_file.exists():
|
||
scope_hint = ""
|
||
if karyotype_scope_data.get("scope_id") or karyotype_scope_data.get("target_species"):
|
||
scope_hint = f" Scope: {karyotype_scope_data.get('scope_id') or karyotype_scope_data.get('target_species')}"
|
||
return f"Karyotype stage requires identify results for {prefix}.{scope_hint} Please run identify first with the same target scope.", manifest
|
||
if not gene_id_file:
|
||
return f"Karyotype stage requires identify gene ID file for {prefix}. Please run identify first with the same target scope.", manifest
|
||
|
||
script = Path("apps/cli/run_karyotype.py")
|
||
if not script.exists():
|
||
return "Karyotype script not found: apps/cli/run_karyotype.py", manifest
|
||
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--input-dir", str(in_dir),
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix,
|
||
"--cpu", str(cpu),
|
||
"--log-file", str(log_file),
|
||
]
|
||
if karyotype_scope_path:
|
||
cmd.extend(["--scope-file", str(karyotype_scope_path)])
|
||
if analysis_route:
|
||
cmd.extend(["--analysis-route", analysis_route])
|
||
if member_set_id:
|
||
cmd.extend(["--member-set-id", member_set_id])
|
||
if gene_id_file:
|
||
cmd.extend(["--gene-id-file", str(gene_id_file)])
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
karyotype_manifest = ws / "final" / f"{prefix}.karyotype.manifest.json"
|
||
if karyotype_scope_path:
|
||
try:
|
||
karyotype_scope_data = read_scope(karyotype_scope_path)
|
||
karyotype_scope_id = str(karyotype_scope_data.get("scope_id") or "")
|
||
if karyotype_scope_id:
|
||
scoped_karyotype_manifest = ws / "final" / f"{prefix}.{sanitize_token(karyotype_scope_id)}.karyotype.manifest.json"
|
||
if scoped_karyotype_manifest.exists():
|
||
karyotype_manifest = scoped_karyotype_manifest
|
||
except Exception:
|
||
pass
|
||
|
||
if code == 0 and karyotype_manifest.exists():
|
||
if karyotype_scope_path:
|
||
try:
|
||
karyotype_scope_data_for_annotate = read_scope(karyotype_scope_path)
|
||
annotate_stage_manifest_with_scope(
|
||
karyotype_manifest,
|
||
prefix,
|
||
"karyotype",
|
||
karyotype_scope_path,
|
||
karyotype_scope_data_for_annotate,
|
||
{"analysis_route": analysis_route, "member_set_id": member_set_id} if (analysis_route or member_set_id) else None,
|
||
)
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to annotate karyotype manifest with scope identity: {exc}")
|
||
return f"Karyotype analysis complete. Log: {log_file}", karyotype_manifest
|
||
elif code == 0:
|
||
return f"Karyotype finished but manifest not found. Log: {log_file}", manifest
|
||
elif karyotype_manifest.exists():
|
||
return f"Karyotype failed (exit code: {code}). Log: {log_file}", karyotype_manifest
|
||
else:
|
||
return f"Karyotype failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
# ========== Interspecies Synteny 阶段 ==========
|
||
if stage == "interspecies_synteny":
|
||
interspecies_scope_path = None
|
||
if scope_file:
|
||
interspecies_scope_path = Path(scope_file).expanduser()
|
||
if not interspecies_scope_path.exists():
|
||
for candidate in (in_dir / interspecies_scope_path, ws / interspecies_scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
interspecies_scope_path = candidate
|
||
break
|
||
if not interspecies_scope_path.exists():
|
||
return f"❌ Interspecies Synteny scope 文件不存在: {scope_file}", manifest
|
||
else:
|
||
# Interspecies Synteny legacy mode requires protein FASTA files in input/final.
|
||
pep_files = list(in_dir.glob("*.pep.fasta")) + list(in_dir.glob("*pep*.fa")) + list(ws.glob("final/*.final_members.pep.fa"))
|
||
if not pep_files:
|
||
return "❌ Interspecies Synteny stage requires protein FASTA files. Please run identify first or upload protein sequences.", manifest
|
||
|
||
script = Path("apps/cli/run_interspecies_synteny.py")
|
||
if not script.exists():
|
||
return "❌ Interspecies Synteny script not found: apps/cli/run_interspecies_synteny.py", manifest
|
||
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--input-dir", str(in_dir),
|
||
"--workspace", str(ws),
|
||
"--prefix", prefix,
|
||
"--cpu", str(cpu),
|
||
"--log-file", str(log_file),
|
||
]
|
||
selected_targets = _species_arg_list(target_species)
|
||
selected_refs = _species_arg_list(interspecies_ref_species)
|
||
if selected_targets:
|
||
cmd += ["--target-species", ",".join(selected_targets)]
|
||
if selected_refs:
|
||
cmd += ["--ref-species", ",".join(selected_refs)]
|
||
cmd += ["--blast-engine", interspecies_blast_engine]
|
||
cmd += ["--plot-mode", interspecies_plot_mode]
|
||
if (ws / "final" / f"{prefix}.interspecies_tree.nwk").exists():
|
||
cmd += ["--reuse-tree"]
|
||
if interspecies_scope_path:
|
||
cmd += ["--scope-file", str(interspecies_scope_path)]
|
||
if analysis_route:
|
||
cmd += ["--analysis-route", analysis_route]
|
||
if member_set_id:
|
||
cmd += ["--member-set-id", member_set_id]
|
||
cmd += [
|
||
"--tree-subfamily-arc-enabled", "true" if tree_subfamily_arc_enabled else "false",
|
||
"--tree-subfamily-arc-gap", str(tree_subfamily_arc_gap),
|
||
"--tree-subfamily-arc-label-gap", str(tree_subfamily_arc_label_gap),
|
||
"--tree-subfamily-arc-label-size", str(tree_subfamily_arc_label_size or 0.0),
|
||
]
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
# 检查输出结果
|
||
interspecies_manifest = ws / "final" / f"{prefix}.interspecies_synteny.manifest.json"
|
||
if interspecies_scope_path:
|
||
try:
|
||
interspecies_scope_data = read_scope(interspecies_scope_path)
|
||
interspecies_scope_id = str(interspecies_scope_data.get("scope_id") or "")
|
||
if interspecies_scope_id:
|
||
scoped_interspecies_manifest = ws / "final" / f"{prefix}.{sanitize_token(interspecies_scope_id)}.interspecies_synteny.manifest.json"
|
||
if scoped_interspecies_manifest.exists():
|
||
interspecies_manifest = scoped_interspecies_manifest
|
||
except Exception:
|
||
pass
|
||
|
||
if code == 0 and interspecies_manifest.exists():
|
||
try:
|
||
interspecies_manifest_data = json.loads(interspecies_manifest.read_text(encoding="utf-8"))
|
||
except Exception as exc:
|
||
return f"Interspecies Synteny finished but manifest is invalid: {exc}. Log: {log_file}", interspecies_manifest
|
||
interspecies_status = str(interspecies_manifest_data.get("status") or "")
|
||
if interspecies_status not in {"success", "partial_success"}:
|
||
detail = interspecies_manifest_data.get("error") or (interspecies_manifest_data.get("parameters") or {}).get("error") or interspecies_status
|
||
return f"Interspecies Synteny failed: manifest status={interspecies_status}. {detail}. Log: {log_file}", interspecies_manifest
|
||
if not (interspecies_manifest_data.get("artifacts") or {}):
|
||
return f"Interspecies Synteny failed: manifest has no artifacts. Log: {log_file}", interspecies_manifest
|
||
if interspecies_scope_path:
|
||
try:
|
||
interspecies_scope_data_for_annotate = read_scope(interspecies_scope_path)
|
||
annotate_stage_manifest_with_scope(
|
||
interspecies_manifest,
|
||
prefix,
|
||
"interspecies_synteny",
|
||
interspecies_scope_path,
|
||
interspecies_scope_data_for_annotate,
|
||
{"analysis_route": analysis_route, "member_set_id": member_set_id} if (analysis_route or member_set_id) else None,
|
||
)
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to annotate interspecies_synteny manifest with scope identity: {exc}")
|
||
return f"Interspecies Synteny analysis complete. Log: {log_file}", interspecies_manifest
|
||
elif code == 0:
|
||
return f"Interspecies Synteny finished but manifest not found. Log: {log_file}", manifest
|
||
elif interspecies_manifest.exists():
|
||
return f"Interspecies Synteny failed (exit code: {code}). Log: {log_file}", interspecies_manifest
|
||
else:
|
||
return f"Interspecies Synteny failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
# ========== Expression 阶段 (差异表达分析) ==========
|
||
if stage == "expression":
|
||
# Expression 阶段需要：表达量矩阵、基因ID列表、样本信息
|
||
# 如果提供了geo_id，优先从GEO下载
|
||
expression_file = None
|
||
sample_file = None
|
||
expression_scope_path = None
|
||
expression_scope_data = None
|
||
expression_target_species = ""
|
||
if scope_file:
|
||
expression_scope_path = resolve_scope_path(scope_file, in_dir, ws)
|
||
if not expression_scope_path.exists():
|
||
return f"❌ Expression scope 文件不存在: {scope_file}", manifest
|
||
try:
|
||
expression_scope_data = read_scope(expression_scope_path)
|
||
except Exception as exc:
|
||
return f"❌ Expression scope 文件无法读取: {scope_file}; {exc}", manifest
|
||
target_species_for_expression = expression_scope_data.get("target_species", [])
|
||
if len(target_species_for_expression) == 0:
|
||
print("[Expression] Empty scope target_species; falling back to non-scoped expression inputs")
|
||
expression_scope_path = None
|
||
expression_scope_data = None
|
||
elif len(target_species_for_expression) != 1:
|
||
return (
|
||
f"❌ Expression 阶段一次只能分析一个目标物种；当前 scope target_species={target_species_for_expression}。"
|
||
"请在弹窗中只选择一个 target，或分别运行每个目标物种。",
|
||
manifest,
|
||
)
|
||
else:
|
||
expression_target_species = str(target_species_for_expression[0])
|
||
|
||
if geo_id:
|
||
print(f"[GEO] 开始下载GEO数据: {geo_id}")
|
||
|
||
# 导入下载模块
|
||
from apps.cli.run_fetch_expression import download_geo_expression
|
||
|
||
# 创建下载目录
|
||
geo_download_dir = in_dir / "geo_download"
|
||
geo_download_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# 下载GEO数据
|
||
expr_file, samp_file, msg = download_geo_expression(geo_id, geo_download_dir)
|
||
|
||
if not expr_file:
|
||
return f"❌ GEO数据下载失败: {msg}", manifest
|
||
|
||
# 移动文件到input目录
|
||
target_expr = in_dir / expr_file.name
|
||
shutil.copy(expr_file, target_expr)
|
||
expression_file = target_expr
|
||
|
||
if samp_file and samp_file.exists():
|
||
target_sample = in_dir / "sample.txt"
|
||
shutil.copy(samp_file, target_sample)
|
||
sample_file = target_sample
|
||
|
||
print(f"[GEO] 下载完成: {expression_file.name}")
|
||
|
||
# 检查输入文件。前端“Expression(使用FPKM)”可能显式传入 rnaseqExpressionMatrix；
|
||
# 先尊重显式选择，再自动搜索，自动搜索时优先使用 rnaseq_quant 生成的 FPKM。
|
||
if not expression_file and rnaseq_expression_matrix:
|
||
explicit_candidates = [Path(rnaseq_expression_matrix)]
|
||
if not Path(rnaseq_expression_matrix).is_absolute():
|
||
explicit_candidates.extend([
|
||
in_dir / rnaseq_expression_matrix,
|
||
final_dir / rnaseq_expression_matrix,
|
||
ws / rnaseq_expression_matrix,
|
||
])
|
||
explicit_candidates = [p for p in explicit_candidates if p.exists()]
|
||
if explicit_candidates:
|
||
expression_file = explicit_candidates[0]
|
||
|
||
if not expression_file:
|
||
expression_files = list(in_dir.glob("*.fpkm_matrix.txt")) + list(in_dir.glob("*_fpkm_matrix.txt")) \
|
||
+ list(in_dir.glob("*.fpkm.txt")) + list(in_dir.glob("*_fpkm.txt")) \
|
||
+ list(in_dir.glob("*.salmon.tpm_matrix.txt")) + list(in_dir.glob("*.kallisto.tpm_matrix.txt")) \
|
||
+ list(in_dir.glob("*.rsem.tpm_matrix.txt")) + list(in_dir.glob("*.tpm_matrix.txt")) \
|
||
+ list(in_dir.glob("*_tpm_matrix.txt")) + list(in_dir.glob("*.tpm.txt")) \
|
||
+ list(in_dir.glob("*_tpm.txt"))
|
||
|
||
if not expression_files:
|
||
expression_files = list((ws / "final").glob("*.fpkm_matrix.txt")) + list((ws / "final").glob("*_fpkm_matrix.txt")) \
|
||
+ list((ws / "final").glob("*.fpkm.txt")) + list((ws / "final").glob("*_fpkm.txt")) \
|
||
+ list((ws / "final").glob("*.salmon.tpm_matrix.txt")) + list((ws / "final").glob("*.kallisto.tpm_matrix.txt")) \
|
||
+ list((ws / "final").glob("*.rsem.tpm_matrix.txt")) + list((ws / "final").glob("*.tpm_matrix.txt")) \
|
||
+ list((ws / "final").glob("*_tpm_matrix.txt")) + list((ws / "final").glob("*.tpm.txt")) \
|
||
+ list((ws / "final").glob("*_tpm.txt"))
|
||
|
||
if not expression_files:
|
||
return "❌ Expression stage requires an expression matrix. Prefer *.fpkm_matrix.txt for this FPKM expression plot stage; TPM is accepted only as fallback.", manifest
|
||
|
||
expression_file = expression_files[0]
|
||
|
||
# 查找基因ID文件。scope模式必须使用单个target物种的成员ID，避免多物种合并ID误入表达矩阵。
|
||
if expression_scope_data:
|
||
gene_id_file = resolve_single_target_gene_id_file(ws, in_dir, final_dir, expression_target_species, prefix)
|
||
if not gene_id_file:
|
||
return (
|
||
f"❌ Expression stage requires {expression_target_species} gene family member IDs. "
|
||
f"Expected identify/{expression_target_species}/final/{expression_target_species}.final_gfmembers.id "
|
||
"or an equivalent uploaded file.",
|
||
manifest,
|
||
)
|
||
else:
|
||
gene_id_files = (
|
||
list(in_dir.glob("*geneID*"))
|
||
+ list(in_dir.glob("*.final_members.id"))
|
||
+ list((ws / "final").glob("*geneID*"))
|
||
+ list((ws / "final").glob("*.final_members.id"))
|
||
)
|
||
if not gene_id_files:
|
||
return "❌ Expression stage requires gene ID file (*geneID* or *.final_members.id). Please upload or run Identify first.", manifest
|
||
gene_id_file = gene_id_files[0]
|
||
|
||
# 查找样本信息文件
|
||
if not sample_file:
|
||
sample_files = list(in_dir.glob("sample.txt")) + list((ws / "final").glob("sample.txt"))
|
||
if not sample_files:
|
||
return "❌ Expression stage requires sample info file (sample.txt). Please upload.", manifest
|
||
sample_file = sample_files[0]
|
||
# 查找R脚本
|
||
script = Path("apps/cli/run_expression.py")
|
||
if not script.exists():
|
||
return "❌ Expression script not found: apps/cli/run_expression.py", manifest
|
||
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--prefix", prefix,
|
||
"--workspace", str(ws),
|
||
"--expression-file", str(expression_file),
|
||
"--gene-id-file", str(gene_id_file),
|
||
"--sample-file", str(sample_file),
|
||
"--id-transform", id_transform,
|
||
"--log-file", str(log_file),
|
||
]
|
||
if expression_scope_path:
|
||
cmd += ["--scope-file", str(expression_scope_path)]
|
||
if analysis_route:
|
||
cmd += ["--analysis-route", analysis_route]
|
||
if member_set_id:
|
||
cmd += ["--member-set-id", member_set_id]
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
# 检查输出结果
|
||
expression_manifest = ws / "final" / f"{prefix}.expression.manifest.json"
|
||
|
||
if code == 0 and expression_manifest.exists():
|
||
if expression_scope_path and expression_scope_data:
|
||
try:
|
||
annotate_stage_manifest_with_scope(
|
||
expression_manifest,
|
||
prefix,
|
||
"expression",
|
||
expression_scope_path,
|
||
expression_scope_data,
|
||
{"analysis_route": analysis_route, "member_set_id": member_set_id} if (analysis_route or member_set_id) else None,
|
||
)
|
||
scoped_expression_manifest = final_dir / f"{prefix}.{sanitize_token(str(expression_scope_data.get('scope_id') or ''))}.expression.manifest.json"
|
||
if scoped_expression_manifest.exists():
|
||
expression_manifest = scoped_expression_manifest
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to annotate expression manifest with scope identity: {exc}")
|
||
return f"✅ Expression analysis complete. Log: {log_file}", expression_manifest
|
||
elif code == 0:
|
||
return f"⚠️ Expression finished but manifest not found. Log: {log_file}", manifest
|
||
else:
|
||
return f"❌ Expression failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
# ========== RNA-seq 各步骤单独运行 ==========
|
||
if stage == "rnaseq_qc":
|
||
# Step 1: QC 质控
|
||
sample_config = _resolve_rnaseq_sample_config(in_dir, ws, rnaseq_sample_config)
|
||
if not sample_config:
|
||
return "❌ RNA-seq QC requires sample configuration file (samples.yaml). Please upload.", manifest
|
||
|
||
qc_dir = ws / "rnaseq_qc"
|
||
qc_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
import yaml
|
||
with open(sample_config, "r", encoding="utf-8") as handle:
|
||
sample_count = len((yaml.safe_load(handle) or {}).get("samples", {}) or {})
|
||
except Exception:
|
||
sample_count = 1
|
||
try:
|
||
from gfa_core.mas.resource_governor import recommend_rnaseq_parallelism
|
||
|
||
thread_plan = recommend_rnaseq_parallelism(
|
||
total_cpu=rnaseq_project_max_cpu or cpu,
|
||
sample_parallel_jobs=rnaseq_sample_parallel_jobs or None,
|
||
threads_per_sample=rnaseq_threads_per_sample or None,
|
||
sample_count=sample_count,
|
||
stage="rnaseq_qc",
|
||
)
|
||
except Exception as e:
|
||
print(f"[WARN] Failed to resolve RNA-seq QC thread plan, using serial fallback: {e}")
|
||
thread_plan = {
|
||
"total_cpu": cpu,
|
||
"project_cpu_limit": cpu,
|
||
"sample_parallel_jobs": 1,
|
||
"threads_per_sample": max(1, int(cpu or 1)),
|
||
}
|
||
print(
|
||
"[RNA-seq QC] thread plan: "
|
||
f"project_cpu_limit={thread_plan.get('project_cpu_limit')}, "
|
||
f"sample_parallel_jobs={thread_plan['sample_parallel_jobs']}, "
|
||
f"threads_per_sample={thread_plan['threads_per_sample']}, "
|
||
f"effective_cpu={thread_plan['total_cpu']}"
|
||
)
|
||
cmd = [
|
||
"/home/xzq/miniconda3/envs/rnaseq/bin/python", "apps/cli/run_rnaseq_qc.py",
|
||
"-i", str(in_dir),
|
||
"-c", str(sample_config),
|
||
"-o", str(qc_dir),
|
||
"-j", job.job_id,
|
||
"-t", str(thread_plan["total_cpu"]),
|
||
"-p", prefix,
|
||
"--sample-parallel-jobs", str(thread_plan["sample_parallel_jobs"]),
|
||
"--threads-per-sample", str(thread_plan["threads_per_sample"]),
|
||
"--resume-existing",
|
||
]
|
||
qc_tool = str(rnaseq_qc_tool or "fastp").strip().lower()
|
||
if qc_tool not in {"cutadapt", "fastp"}:
|
||
return f"❌ RNA-seq QC tool not supported: {rnaseq_qc_tool}. Choose cutadapt or fastp.", manifest
|
||
cmd.extend(["--qc-tool", qc_tool])
|
||
if qc_tool == "fastp":
|
||
fastp_bin = str(rnaseq_fastp_bin or "").strip()
|
||
if fastp_bin:
|
||
cmd.extend(["--fastp-bin", fastp_bin])
|
||
cmd.extend([
|
||
"--cut-window-size", str(int(rnaseq_fastp_cut_window_size or 4)),
|
||
"--cut-mean-quality", str(int(rnaseq_fastp_cut_mean_quality or 20)),
|
||
"--qualified-quality-phred", str(int(rnaseq_fastp_qualified_quality_phred or 20)),
|
||
"--unqualified-percent-limit", str(int(rnaseq_fastp_unqualified_percent_limit or 30)),
|
||
"--n-base-limit", str(int(rnaseq_fastp_n_base_limit or 5)),
|
||
"--length-required", str(int(rnaseq_fastp_length_required or 30)),
|
||
])
|
||
cmd.append("--detect-adapter-for-pe" if _bool_with_default(rnaseq_fastp_detect_adapter_for_pe, True) else "--no-detect-adapter-for-pe")
|
||
cmd.append("--cut-front" if _bool_with_default(rnaseq_fastp_cut_front, True) else "--no-cut-front")
|
||
cmd.append("--cut-tail" if _bool_with_default(rnaseq_fastp_cut_tail, True) else "--no-cut-tail")
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
if code == 0:
|
||
# 脚本已直接输出到 final/ 目录，验证结果
|
||
if not final_dir.exists():
|
||
return f"❌ RNA-seq QC failed: final directory not found", manifest
|
||
|
||
# 统计输出文件（只统计 rnaseq_qc 相关文件）
|
||
fastqc_html = list(final_dir.glob("*_fastqc.html"))
|
||
clean_fastq = list(final_dir.glob("*.clean.fastq.gz"))
|
||
cutadapt_logs = list(final_dir.glob("*_cutadapt.log"))
|
||
fastp_outputs = (
|
||
list(final_dir.glob("*_fastp.html"))
|
||
+ list(final_dir.glob("*_fastp.json"))
|
||
+ list(final_dir.glob("*_fastp.log"))
|
||
)
|
||
rnaseq_files = fastqc_html + clean_fastq + cutadapt_logs + fastp_outputs
|
||
file_count = len(rnaseq_files)
|
||
|
||
# 读取manifest验证样本数
|
||
rnaseq_manifest = final_dir / f"{prefix}.rnaseq_qc.manifest.json"
|
||
expected_samples = 0
|
||
actual_samples = 0
|
||
|
||
if rnaseq_manifest.exists():
|
||
try:
|
||
with open(rnaseq_manifest, 'r') as f:
|
||
mdata = json.load(f)
|
||
expected_samples = mdata.get("metrics", {}).get("total_samples", 0)
|
||
actual_samples = len(mdata.get("metrics", {}).get("samples", {}))
|
||
except Exception as e:
|
||
print(f"[WARN] Failed to parse manifest: {e}")
|
||
|
||
# 严格验证：必须有输出文件且非空
|
||
if file_count == 0:
|
||
return f"❌ RNA-seq QC failed: no output files generated", manifest
|
||
|
||
# 验证每个文件非空
|
||
empty_files = [f for f in rnaseq_files if f.stat().st_size == 0]
|
||
if empty_files:
|
||
empty_names = ", ".join([f.name for f in empty_files[:3]])
|
||
return f"❌ RNA-seq QC failed: {len(empty_files)} empty output file(s): {empty_names}...", manifest
|
||
|
||
# 验证样本数匹配（如果manifest可用）
|
||
if expected_samples > 0 and actual_samples != expected_samples:
|
||
return f"❌ RNA-seq QC failed: expected {expected_samples} samples but processed {actual_samples}", manifest
|
||
|
||
return f"✅ RNA-seq QC completed. {file_count} files in final/ ({actual_samples} samples)", rnaseq_manifest
|
||
else:
|
||
return f"❌ RNA-seq QC failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
if stage == "rnaseq_align":
|
||
# Step 2: 比对
|
||
# 输入：clean reads (来自 rnaseq_qc), HISAT2索引
|
||
# 输出：排序后的BAM文件
|
||
|
||
# 查找 clean reads
|
||
clean_reads = list(final_dir.glob("*.clean.fastq.gz"))
|
||
if not clean_reads:
|
||
return "❌ RNA-seq Align requires clean reads from QC stage. Please run RNA-seq QC first.", manifest
|
||
|
||
# 查找样本配置，并通过统一 RNA-seq context 锁定物种和参考资源。
|
||
sample_config = _resolve_rnaseq_sample_config(in_dir, ws, rnaseq_sample_config)
|
||
rnaseq_context = resolve_rnaseq_project_context(
|
||
input_dir=in_dir,
|
||
final_dir=final_dir,
|
||
requested_prefix=prefix,
|
||
sample_config=sample_config,
|
||
)
|
||
resolved_rnaseq_prefix = rnaseq_context.get("output_prefix", "")
|
||
if resolved_rnaseq_prefix and resolved_rnaseq_prefix != prefix:
|
||
print(
|
||
f"[INFO] Using RNA-seq output prefix {resolved_rnaseq_prefix} for rnaseq_align "
|
||
f"instead of {prefix or '<empty>'}"
|
||
)
|
||
prefix = resolved_rnaseq_prefix
|
||
rnaseq_species = rnaseq_context.get("species_code") or rnaseq_context.get("output_prefix", "").split(".", 1)[0]
|
||
rnaseq_reference_id = rnaseq_context.get("reference_id", "")
|
||
if not rnaseq_species or rnaseq_species.lower() in {"unknown", "genome", "genome_prepare", "rnaseq"}:
|
||
return (
|
||
"❌ RNA-seq Align requires a real species code. Add species.code/speciesCode "
|
||
"and rnaseq.outputPrefix/rnaseqPrefix to samples.yaml, e.g. Os.",
|
||
manifest,
|
||
)
|
||
|
||
hisat2_idx = find_rnaseq_hisat2_index(in_dir, rnaseq_species, rnaseq_reference_id)
|
||
if hisat2_idx:
|
||
print(f"[INFO] Using locked genome HISAT2 index from context: {hisat2_idx}")
|
||
else:
|
||
genome_file = find_rnaseq_reference_file(in_dir, "genome", rnaseq_species, rnaseq_reference_id)
|
||
if not genome_file:
|
||
return (
|
||
f"❌ RNA-seq Align requires {rnaseq_species}.genome.fasta/fa/fna or "
|
||
f"{rnaseq_species}.genome.*.ht2. CDS/PEP indexes are not valid for RNA-seq alignment.",
|
||
manifest,
|
||
)
|
||
hisat2_idx = genome_file.parent / f"{rnaseq_species}.genome"
|
||
print(f"[INFO] Locked genome FASTA for HISAT2 build in CLI: {genome_file}")
|
||
annotation_file = find_rnaseq_reference_file(in_dir, "annotation", rnaseq_species, rnaseq_reference_id)
|
||
|
||
align_dir = ws / "rnaseq_align"
|
||
align_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
sample_count = max(1, len({p.name.rsplit("_", 1)[0] for p in clean_reads}))
|
||
try:
|
||
from gfa_core.mas.resource_governor import recommend_rnaseq_parallelism
|
||
|
||
thread_plan = recommend_rnaseq_parallelism(
|
||
total_cpu=rnaseq_project_max_cpu or cpu,
|
||
sample_parallel_jobs=rnaseq_sample_parallel_jobs or None,
|
||
threads_per_sample=rnaseq_threads_per_sample or None,
|
||
sample_count=sample_count,
|
||
stage="rnaseq_align",
|
||
)
|
||
except Exception as e:
|
||
print(f"[WARN] Failed to resolve RNA-seq thread plan, using serial fallback: {e}")
|
||
thread_plan = {
|
||
"total_cpu": cpu,
|
||
"project_cpu_limit": cpu,
|
||
"sample_parallel_jobs": 1,
|
||
"threads_per_sample": max(1, int(cpu or 1)),
|
||
}
|
||
|
||
if rnaseq_resource_decision:
|
||
print(f"[RNA-seq Align] resource decision: {rnaseq_resource_decision}")
|
||
print(
|
||
"[RNA-seq Align] thread plan: "
|
||
f"project_cpu_limit={thread_plan.get('project_cpu_limit')}, "
|
||
f"sample_parallel_jobs={thread_plan['sample_parallel_jobs']}, "
|
||
f"threads_per_sample={thread_plan['threads_per_sample']}, "
|
||
f"effective_cpu={thread_plan['total_cpu']}"
|
||
)
|
||
|
||
cmd = [
|
||
"/home/xzq/miniconda3/envs/rnaseq/bin/python", "apps/cli/run_rnaseq_align.py",
|
||
"-i", str(final_dir), # 从 final/ 读取 clean reads
|
||
"-x", str(hisat2_idx),
|
||
"-o", str(align_dir),
|
||
"-j", job.job_id,
|
||
"-t", str(thread_plan["total_cpu"]),
|
||
"-p", prefix,
|
||
"--sample-parallel-jobs", str(thread_plan["sample_parallel_jobs"]),
|
||
"--threads-per-sample", str(thread_plan["threads_per_sample"]),
|
||
]
|
||
if sample_config:
|
||
cmd.extend(["--sample-config", str(sample_config)])
|
||
splice_source_val = str(rnaseq_splice_site_source or "auto").strip().lower()
|
||
if splice_source_val not in {"auto", "off", "manual", "annotation"}:
|
||
splice_source_val = "auto"
|
||
known_splice_text = str(rnaseq_known_splice_site or "").strip()
|
||
known_splice_path = Path(known_splice_text) if known_splice_text else None
|
||
if known_splice_path and not known_splice_path.is_absolute():
|
||
for base in (in_dir, final_dir, ws, Path(".")):
|
||
candidate = base / known_splice_path
|
||
if candidate.exists():
|
||
known_splice_path = candidate
|
||
break
|
||
if splice_source_val in {"auto", "annotation"} and not known_splice_path and annotation_file:
|
||
known_splice_path = annotation_file
|
||
cmd.extend(["--splice-site-source", splice_source_val])
|
||
if known_splice_path:
|
||
cmd.extend(["--known-splice-site", str(known_splice_path)])
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
if code == 0:
|
||
# 验证结果
|
||
bam_files = list(final_dir.glob("*.sorted.bam"))
|
||
file_count = len(bam_files)
|
||
|
||
# 返回新生成的 manifest
|
||
align_manifest = final_dir / f"{prefix}.rnaseq_align.manifest.json"
|
||
return f"✅ RNA-seq Align completed. {file_count} BAM files in final/", align_manifest
|
||
else:
|
||
return f"❌ RNA-seq Align failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
if stage == "rnaseq_quant":
|
||
# Step 3: 定量
|
||
# 输入：BAM文件 (来自rnaseq_align), GTF注释
|
||
bam_files = list(final_dir.glob("*.sorted.bam"))
|
||
sample_config_for_context = _resolve_rnaseq_sample_config(in_dir, ws, rnaseq_sample_config)
|
||
rnaseq_context = resolve_rnaseq_project_context(
|
||
input_dir=in_dir,
|
||
final_dir=final_dir,
|
||
requested_prefix=prefix,
|
||
sample_config=sample_config_for_context,
|
||
)
|
||
resolved_rnaseq_prefix = rnaseq_context.get("output_prefix", "")
|
||
if resolved_rnaseq_prefix and resolved_rnaseq_prefix != prefix:
|
||
print(
|
||
f"[INFO] Using RNA-seq output prefix {resolved_rnaseq_prefix} for rnaseq_quant "
|
||
f"instead of {prefix or '<empty>'}"
|
||
)
|
||
prefix = resolved_rnaseq_prefix
|
||
rnaseq_species = rnaseq_context.get("species_code") or rnaseq_context.get("output_prefix", "").split(".", 1)[0]
|
||
rnaseq_reference_id = rnaseq_context.get("reference_id", "")
|
||
gtf_file = find_rnaseq_reference_file(in_dir, "annotation", rnaseq_species, rnaseq_reference_id)
|
||
transcriptome_file = find_rnaseq_reference_file(in_dir, "transcriptome", rnaseq_species, rnaseq_reference_id)
|
||
transcriptome_candidates = [transcriptome_file] if transcriptome_file else []
|
||
|
||
if not bam_files:
|
||
return "❌ RNA-seq Quant requires sorted BAM files (*.sorted.bam). Please run RNA-seq Align first.", manifest
|
||
|
||
if not gtf_file:
|
||
return (
|
||
f"❌ RNA-seq Quant requires an annotation file for {rnaseq_species or 'the RNA-seq species'} "
|
||
"(*.gff3/*.gff/*.gtf). Please upload or run genome_prepare first.",
|
||
manifest,
|
||
)
|
||
|
||
quant_dir = ws / "rnaseq_quant"
|
||
quant_dir.mkdir(parents=True, exist_ok=True)
|
||
final_dir.mkdir(exist_ok=True)
|
||
|
||
print(f"[RNA-seq Quant] BAM files: {len(bam_files)}")
|
||
print(f"[RNA-seq Quant] GTF: {gtf_file.name}")
|
||
|
||
quant_mode_val = str(rnaseq_quant_mode or "dual").strip().lower()
|
||
if quant_mode_val not in {"dual", "count_only"}:
|
||
quant_mode_val = "dual"
|
||
abundance_engine_val = str(rnaseq_abundance_engine or "auto").strip().lower()
|
||
if abundance_engine_val not in {"auto", "salmon", "kallisto", "none"}:
|
||
abundance_engine_val = "auto"
|
||
tximport_counts_val = str(rnaseq_tximport_counts_from_abundance or "no").strip()
|
||
if tximport_counts_val not in {"no", "scaledTPM", "lengthScaledTPM", "dtuScaledTPM"}:
|
||
tximport_counts_val = "no"
|
||
salmon_libtype_val = str(rnaseq_salmon_libtype or "A").strip() or "A"
|
||
|
||
def resolve_optional_rnaseq_path(value: str) -> Path | None:
|
||
text = str(value or "").strip()
|
||
if not text:
|
||
return None
|
||
raw = Path(text)
|
||
if raw.is_absolute():
|
||
return raw
|
||
for base in (in_dir, final_dir, ws, Path(".")):
|
||
candidate = base / raw
|
||
if candidate.exists():
|
||
return candidate
|
||
return raw
|
||
|
||
transcriptome_override = resolve_optional_rnaseq_path(rnaseq_transcriptome_fasta)
|
||
tx2gene_override = resolve_optional_rnaseq_path(rnaseq_tx2gene)
|
||
sample_count = max(1, len(bam_files))
|
||
try:
|
||
from gfa_core.mas.resource_governor import recommend_rnaseq_parallelism
|
||
|
||
thread_plan = recommend_rnaseq_parallelism(
|
||
total_cpu=rnaseq_project_max_cpu or cpu,
|
||
sample_parallel_jobs=rnaseq_sample_parallel_jobs or None,
|
||
threads_per_sample=rnaseq_threads_per_sample or None,
|
||
sample_count=sample_count,
|
||
stage="rnaseq_quant",
|
||
)
|
||
except Exception as e:
|
||
print(f"[WARN] Failed to resolve RNA-seq Quant thread plan, using serial fallback: {e}")
|
||
thread_plan = {
|
||
"total_cpu": cpu,
|
||
"project_cpu_limit": cpu,
|
||
"sample_parallel_jobs": 1,
|
||
"threads_per_sample": max(1, int(cpu or 1)),
|
||
}
|
||
print(
|
||
"[RNA-seq Quant] thread plan: "
|
||
f"project_cpu_limit={thread_plan.get('project_cpu_limit')}, "
|
||
f"sample_parallel_jobs={thread_plan['sample_parallel_jobs']}, "
|
||
f"threads_per_sample={thread_plan['threads_per_sample']}, "
|
||
f"effective_cpu={thread_plan['total_cpu']}"
|
||
)
|
||
|
||
quant_cmd = [
|
||
"/home/xzq/miniconda3/envs/rnaseq/bin/python", "apps/cli/run_rnaseq_quant.py",
|
||
"-i", str(final_dir),
|
||
"-g", str(gtf_file),
|
||
"-o", str(quant_dir),
|
||
"-p", prefix,
|
||
"-j", job.job_id,
|
||
"-t", str(thread_plan["total_cpu"]),
|
||
"--sample-parallel-jobs", str(thread_plan["sample_parallel_jobs"]),
|
||
"--threads-per-sample", str(thread_plan["threads_per_sample"]),
|
||
"--quant-mode", quant_mode_val,
|
||
"--abundance-engine", abundance_engine_val,
|
||
"--salmon-libtype", salmon_libtype_val,
|
||
"--tximport-counts-from-abundance", tximport_counts_val,
|
||
]
|
||
if sample_config_for_context:
|
||
quant_cmd.extend(["--sample-config", str(sample_config_for_context)])
|
||
if transcriptome_override:
|
||
quant_cmd.extend(["--transcriptome-fasta", str(transcriptome_override)])
|
||
elif transcriptome_candidates:
|
||
quant_cmd.extend(["--transcriptome-fasta", str(transcriptome_candidates[0])])
|
||
if tx2gene_override:
|
||
quant_cmd.extend(["--tx2gene", str(tx2gene_override)])
|
||
if rnaseq_kallisto_bias:
|
||
quant_cmd.append("--kallisto-bias")
|
||
|
||
print(f"[CMD] {' '.join(quant_cmd)}")
|
||
code = run_cmd_logged(quant_cmd, log_file, cwd=Path("."))
|
||
|
||
if code == 0:
|
||
# 统计输出文件
|
||
count_files = list(final_dir.glob("*.count_matrix.txt"))
|
||
file_count = len(count_files)
|
||
|
||
quant_manifest = final_dir / f"{prefix}.rnaseq_quant.manifest.json"
|
||
if not quant_manifest.exists():
|
||
candidates = sorted(final_dir.glob("*.rnaseq_quant.manifest.json"))
|
||
if candidates:
|
||
quant_manifest = candidates[0]
|
||
return f"✅ RNA-seq Quant completed. {file_count} count matrices in final/", quant_manifest
|
||
else:
|
||
return f"❌ RNA-seq Quant failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
if stage == "rnaseq_de":
|
||
# Step 4: 差异表达
|
||
# 输入：count_matrix (来自rnaseq_quant), 分组配置文件
|
||
count_matrix_files = list(in_dir.glob("*.count_matrix.txt")) + list((ws / "final").glob("*.count_matrix.txt"))
|
||
config_file = _resolve_rnaseq_sample_config(in_dir, ws, rnaseq_sample_config)
|
||
|
||
if not count_matrix_files:
|
||
return "❌ RNA-seq DE requires count matrix file (*.count_matrix.txt). Please run RNA-seq Quant first or upload.", manifest
|
||
|
||
if not config_file:
|
||
return "❌ RNA-seq DE requires sample configuration file (samples.yaml). Please upload.", manifest
|
||
|
||
count_matrix = count_matrix_files[0]
|
||
|
||
de_dir = ws / "rnaseq_de"
|
||
de_dir.mkdir(parents=True, exist_ok=True)
|
||
final_dir.mkdir(exist_ok=True)
|
||
|
||
print(f"[RNA-seq DE] Count matrix: {count_matrix.name}")
|
||
print(f"[RNA-seq DE] Config: {config_file.name}")
|
||
|
||
try:
|
||
de_padj_val = float(rnaseq_de_padj)
|
||
except (TypeError, ValueError):
|
||
de_padj_val = 0.05
|
||
if not (0 <= de_padj_val <= 1):
|
||
de_padj_val = 0.05
|
||
try:
|
||
de_log2fc_val = float(rnaseq_de_log2fc)
|
||
except (TypeError, ValueError):
|
||
de_log2fc_val = 1.0
|
||
if de_log2fc_val < 0:
|
||
de_log2fc_val = 1.0
|
||
|
||
de_cmd = [
|
||
"/home/xzq/miniconda3/envs/rnaseq/bin/python", "apps/cli/run_rnaseq_de.py",
|
||
"-i", str(count_matrix.parent),
|
||
"-c", str(config_file),
|
||
"-o", str(de_dir),
|
||
"--final-dir", str(final_dir), # 指定主final目录
|
||
"-p", prefix,
|
||
"-j", job.job_id,
|
||
"-t", str(cpu),
|
||
"--padj", str(de_padj_val),
|
||
"--log2fc", str(de_log2fc_val),
|
||
]
|
||
|
||
if rnaseq_de_module:
|
||
de_cmd.extend(["--module", rnaseq_de_module])
|
||
|
||
print(f"[DEBUG] final_dir = {final_dir}")
|
||
print(f"[DEBUG] de_cmd = {' '.join(de_cmd)}")
|
||
|
||
foreground_mode_val = (rnaseq_foreground_mode or "auto").strip().lower()
|
||
if foreground_mode_val not in {"auto", "global", "family"}:
|
||
foreground_mode_val = "auto"
|
||
family_id_files = list(in_dir.glob("*.final_members.id")) + list(final_dir.glob("*.final_members.id"))
|
||
if foreground_mode_val == "auto":
|
||
foreground_mode_val = "family" if family_id_files else "global"
|
||
print(f"[RNA-seq DE] Foreground mode: {foreground_mode_val}")
|
||
|
||
# 仅在用户选择基因家族 foreground 时才生成 family-specific DE 文件。
|
||
# 全基因组模式仍会使用相同 count matrix 做完整 DE，但不按 final_members.id 过滤。
|
||
if foreground_mode_val == "family" and family_id_files:
|
||
family_id_file = family_id_files[0]
|
||
family_name = family_id_file.stem.replace(".final_members", "").replace(".final_members.id", "")
|
||
de_cmd.extend(["-f", str(family_id_file), "-n", family_name])
|
||
print(f"[RNA-seq DE] Family IDs: {family_id_file.name}")
|
||
elif foreground_mode_val == "family":
|
||
return (
|
||
"❌ RNA-seq DE family foreground requires gene family member file (*.final_members.id). "
|
||
"Please run Identify first, upload the member ID file, or choose global foreground mode.",
|
||
manifest,
|
||
)
|
||
|
||
print(f"[CMD] {' '.join(de_cmd)}")
|
||
code = run_cmd_logged(de_cmd, log_file, cwd=Path("."))
|
||
|
||
if code == 0:
|
||
# 统计输出文件
|
||
de_files = list(final_dir.glob("*.DE_results.txt")) + list(final_dir.glob("*.DE_genes.txt"))
|
||
file_count = len(de_files)
|
||
|
||
de_manifest = final_dir / f"{prefix}.rnaseq_de.manifest.json"
|
||
legacy_manifest = final_dir / "rnaseq_de.manifest.json"
|
||
if not de_manifest.exists() and legacy_manifest.exists():
|
||
try:
|
||
with open(legacy_manifest, "r", encoding="utf-8") as handle:
|
||
legacy_data = json.load(handle)
|
||
legacy_data["prefix"] = prefix or legacy_data.get("prefix")
|
||
legacy_data["stage"] = "rnaseq_de"
|
||
legacy_data["status"] = legacy_data.get("status") or "success"
|
||
with open(de_manifest, "w", encoding="utf-8") as handle:
|
||
json.dump(legacy_data, handle, ensure_ascii=False, indent=2)
|
||
print(f"[RNA-seq DE] Migrated legacy manifest to standard name: {de_manifest.name}")
|
||
except Exception as e:
|
||
print(f"[WARN] Failed to migrate legacy rnaseq_de manifest: {e}")
|
||
# RNA-seq-derived expression plots are an automatic post-processing
|
||
# product of rnaseq_de. The standalone expression stage remains
|
||
# available for user-supplied FPKM/TPM matrices.
|
||
expression_note = ""
|
||
fpkm_candidates = list(final_dir.glob(f"{prefix}.fpkm_matrix.txt")) + list(final_dir.glob("*.fpkm_matrix.txt"))
|
||
tpm_candidates = list(final_dir.glob(f"{prefix}.tpm_matrix.txt")) + list(final_dir.glob("*.tpm_matrix.txt"))
|
||
vst_candidates = list(final_dir.glob(f"{prefix}.rnaseq_sample_qc.vst_matrix.tsv")) + list(final_dir.glob("*.rnaseq_sample_qc.vst_matrix.tsv"))
|
||
sample_table_candidates = [ws / "rnaseq_sample_qc" / "sample_table.tsv"] + list(final_dir.glob("sample_table.tsv"))
|
||
count_candidates = list(final_dir.glob(f"{prefix}.count_matrix.txt")) + list(final_dir.glob("*.count_matrix.txt"))
|
||
fpkm_candidates = [p for p in fpkm_candidates if p.exists() and p.stat().st_size > 0]
|
||
tpm_candidates = [p for p in tpm_candidates if p.exists() and p.stat().st_size > 0]
|
||
vst_candidates = [p for p in vst_candidates if p.exists() and p.stat().st_size > 0]
|
||
sample_table_candidates = [p for p in sample_table_candidates if p.exists() and p.stat().st_size > 0]
|
||
count_candidates = [p for p in count_candidates if p.exists() and p.stat().st_size > 0]
|
||
|
||
if fpkm_candidates and tpm_candidates and vst_candidates and sample_table_candidates:
|
||
expression_cmd = [
|
||
"/home/xzq/miniconda3/envs/rnaseq/bin/python",
|
||
"apps/cli/run_expression.py",
|
||
"--mode", "rnaseq_auto",
|
||
"--prefix", prefix,
|
||
"--workspace", str(ws),
|
||
"--expression-file", str(fpkm_candidates[0]),
|
||
"--tpm-file", str(tpm_candidates[0]),
|
||
"--vst-file", str(vst_candidates[0]),
|
||
"--sample-file", str(sample_table_candidates[0]),
|
||
"--de-dir", str(final_dir),
|
||
"--triggered-by", "rnaseq_de",
|
||
]
|
||
if count_candidates:
|
||
expression_cmd.extend(["--count-matrix", str(count_candidates[0])])
|
||
family_id_files = list(in_dir.glob("*.final_members.id")) + list(final_dir.glob("*.final_members.id"))
|
||
family_id_files = [p for p in family_id_files if p.exists() and p.stat().st_size > 0]
|
||
if family_id_files:
|
||
family_id_file = family_id_files[0]
|
||
family_label = family_id_file.stem.replace(".final_members", "")
|
||
expression_cmd.extend(["--gene-id-file", str(family_id_file), "--family-label", family_label])
|
||
print(f"[RNA-seq DE] Auto expression plotting: {' '.join(expression_cmd)}")
|
||
expression_code = run_cmd_logged(expression_cmd, log_file, cwd=Path("."))
|
||
expression_manifest = final_dir / f"{prefix}.expression.manifest.json"
|
||
if not expression_manifest.exists():
|
||
alt_expression_manifest = final_dir / f"{prefix}.rnaseq_expression.manifest.json"
|
||
if alt_expression_manifest.exists():
|
||
expression_manifest = alt_expression_manifest
|
||
if expression_code == 0 and expression_manifest.exists():
|
||
try:
|
||
if de_manifest.exists():
|
||
de_data = json.loads(de_manifest.read_text(encoding="utf-8"))
|
||
de_data["triggered_expression"] = True
|
||
de_data["triggered_expression_manifest"] = f"final/{expression_manifest.name}"
|
||
params = de_data.setdefault("parameters", {})
|
||
if isinstance(params, dict):
|
||
params["triggered_expression"] = True
|
||
params["triggered_expression_manifest"] = f"final/{expression_manifest.name}"
|
||
de_manifest.write_text(json.dumps(de_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
except Exception as exc:
|
||
print(f"[WARN] Failed to annotate rnaseq_de manifest with expression trigger: {exc}")
|
||
expression_note = f" Auto expression plots generated: {expression_manifest.name}."
|
||
else:
|
||
expression_note = f" Auto expression plotting failed or manifest missing; see log: {log_file}."
|
||
else:
|
||
missing_expr_inputs = []
|
||
if not fpkm_candidates:
|
||
missing_expr_inputs.append("FPKM")
|
||
if not tpm_candidates:
|
||
missing_expr_inputs.append("TPM")
|
||
if not vst_candidates:
|
||
missing_expr_inputs.append("VST")
|
||
if not sample_table_candidates:
|
||
missing_expr_inputs.append("sample_table")
|
||
expression_note = f" Auto expression plotting skipped: missing {', '.join(missing_expr_inputs)}."
|
||
|
||
return f"✅ RNA-seq DE completed. {file_count} files in final/.{expression_note}", de_manifest
|
||
else:
|
||
return f"❌ RNA-seq DE failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
if stage == "rnaseq_sample_qc":
|
||
count_matrix_candidates = []
|
||
if rnaseq_count_matrix:
|
||
count_matrix_candidates.append(Path(rnaseq_count_matrix))
|
||
count_matrix_candidates.extend(list(in_dir.glob("*.count_matrix.txt")))
|
||
count_matrix_candidates.extend(list(final_dir.glob("*.count_matrix.txt")))
|
||
count_matrix_candidates = [p for p in count_matrix_candidates if p.exists()]
|
||
if not count_matrix_candidates:
|
||
return "❌ RNA-seq Sample QC requires count matrix (*.count_matrix.txt). Please run RNA-seq Quant first or upload.", manifest
|
||
|
||
sample_config = _resolve_rnaseq_sample_config(in_dir, ws, rnaseq_sample_config)
|
||
|
||
sample_qc_dir = ws / "rnaseq_sample_qc"
|
||
sample_qc_dir.mkdir(parents=True, exist_ok=True)
|
||
sample_qc_cmd = [
|
||
"/home/xzq/miniconda3/envs/rnaseq/bin/python",
|
||
"apps/cli/run_rnaseq_sample_qc.py",
|
||
"--workspace", str(ws),
|
||
"-p", prefix,
|
||
"-j", job.job_id,
|
||
"--count-matrix", str(count_matrix_candidates[0]),
|
||
]
|
||
if sample_config:
|
||
sample_qc_cmd.extend(["--config", str(sample_config)])
|
||
print(f"[CMD] {' '.join(sample_qc_cmd)}")
|
||
code = run_cmd_logged(sample_qc_cmd, log_file, cwd=Path("."))
|
||
sample_qc_manifest = final_dir / f"{prefix}.rnaseq_sample_qc.manifest.json"
|
||
if code == 0:
|
||
return "✅ RNA-seq Sample QC completed.", sample_qc_manifest
|
||
return f"❌ RNA-seq Sample QC failed (exit code: {code}). Log: {log_file}", sample_qc_manifest if sample_qc_manifest.exists() else manifest
|
||
|
||
if stage == "rnaseq_fgsea":
|
||
if not list(final_dir.glob(f"{prefix}.rnaseq_de.*.DE_results.txt")):
|
||
return f"❌ RNA-seq fgsea requires full DESeq2 results ({prefix}.rnaseq_de.*.DE_results.txt). Please run RNA-seq DE first.", manifest
|
||
gene2ko_candidates = (
|
||
list(in_dir.glob("*gene2ko*.txt")) + list(in_dir.glob("*gene2ko*.tsv")) +
|
||
list(final_dir.glob("*gene2ko*.txt")) + list(final_dir.glob("*gene2ko*.tsv"))
|
||
)
|
||
gene2ko_candidates = [p for p in gene2ko_candidates if _has_non_comment_data(p) and ".test." not in p.name]
|
||
if not gene2ko_candidates:
|
||
return "❌ RNA-seq fgsea requires gene2ko background file. Please run annotation or upload gene2ko.txt/gene2ko.tsv.", manifest
|
||
|
||
fgsea_cmd = [
|
||
"/home/xzq/miniconda3/envs/rnaseq/bin/python",
|
||
"apps/cli/run_rnaseq_fgsea.py",
|
||
"--workspace", str(ws),
|
||
"-p", prefix,
|
||
"-j", job.job_id,
|
||
"--gene2ko", str(gene2ko_candidates[0]),
|
||
"--species-name", species_name or "",
|
||
"--kegg-organism", kegg_organism or "",
|
||
"--kegg-mode", "auto",
|
||
"--kegg-filter-scope", "plant",
|
||
"--min-size", str(fgsea_min_size),
|
||
"--max-size", str(fgsea_max_size),
|
||
"--nperm", str(fgsea_nperm),
|
||
"--seed", str(fgsea_seed),
|
||
]
|
||
if fgsea_comparison:
|
||
fgsea_cmd.extend(["--comparison", fgsea_comparison])
|
||
print(f"[CMD] {' '.join(fgsea_cmd)}")
|
||
code = run_cmd_logged(fgsea_cmd, log_file, cwd=Path("."))
|
||
fgsea_manifest = final_dir / f"{prefix}.rnaseq_fgsea.manifest.json"
|
||
if code == 0:
|
||
return "✅ RNA-seq fgsea completed.", fgsea_manifest
|
||
return f"❌ RNA-seq fgsea failed (exit code: {code}). Log: {log_file}", fgsea_manifest if fgsea_manifest.exists() else manifest
|
||
|
||
if stage == "rnaseq_wgcna":
|
||
matrix_candidates = []
|
||
matrix_type = "expression"
|
||
if rnaseq_expression_matrix:
|
||
matrix_candidates.append(Path(rnaseq_expression_matrix))
|
||
matrix_candidates.append(final_dir / f"{prefix}.rnaseq_sample_qc.vst_matrix.tsv")
|
||
matrix_candidates = [p for p in matrix_candidates if p.exists()]
|
||
if not matrix_candidates:
|
||
matrix_type = "counts"
|
||
if rnaseq_count_matrix:
|
||
matrix_candidates.append(Path(rnaseq_count_matrix))
|
||
matrix_candidates.extend(list(final_dir.glob("*.count_matrix.txt")))
|
||
matrix_candidates.extend(list(in_dir.glob("*.count_matrix.txt")))
|
||
matrix_candidates = [p for p in matrix_candidates if p.exists()]
|
||
if not matrix_candidates:
|
||
return "❌ RNA-seq WGCNA requires a VST/expression matrix or count matrix. Please run Sample QC or RNA-seq Quant first.", manifest
|
||
|
||
sample_table_candidates = []
|
||
if rnaseq_sample_table:
|
||
sample_table_candidates.append(Path(rnaseq_sample_table))
|
||
sample_table_candidates.append(ws / "rnaseq_sample_qc" / "sample_table.tsv")
|
||
sample_table_candidates = [p for p in sample_table_candidates if p.exists()]
|
||
sample_config = _resolve_rnaseq_sample_config(in_dir, ws, rnaseq_sample_config)
|
||
|
||
wgcna_cmd = [
|
||
"/home/xzq/miniconda3/envs/rnaseq/bin/python",
|
||
"apps/cli/run_rnaseq_wgcna.py",
|
||
"--workspace", str(ws),
|
||
"-p", prefix,
|
||
"-j", job.job_id,
|
||
"--min-mean", str(wgcna_min_mean),
|
||
"--max-genes", str(wgcna_max_genes),
|
||
"--min-module-size", str(wgcna_min_module_size),
|
||
"--merge-cut-height", str(wgcna_merge_cut_height),
|
||
"--powers", str(wgcna_powers),
|
||
]
|
||
if matrix_type == "expression":
|
||
wgcna_cmd.extend(["--expression-matrix", str(matrix_candidates[0])])
|
||
else:
|
||
wgcna_cmd.extend(["--count-matrix", str(matrix_candidates[0])])
|
||
if sample_table_candidates:
|
||
wgcna_cmd.extend(["--sample-table", str(sample_table_candidates[0])])
|
||
elif sample_config:
|
||
wgcna_cmd.extend(["--config", str(sample_config)])
|
||
print(f"[CMD] {' '.join(wgcna_cmd)}")
|
||
code = run_cmd_logged(wgcna_cmd, log_file, cwd=Path("."))
|
||
wgcna_manifest = final_dir / f"{prefix}.rnaseq_wgcna.manifest.json"
|
||
if code == 0:
|
||
return "✅ RNA-seq WGCNA completed.", wgcna_manifest
|
||
return f"❌ RNA-seq WGCNA failed (exit code: {code}). Log: {log_file}", wgcna_manifest if wgcna_manifest.exists() else manifest
|
||
|
||
if stage == "rnaseq_candidate_family":
|
||
_SPECIES_RESOURCE_ALIASES = {
|
||
"os": ["Os", "Oryza", "Oryza_sativa", "rice"],
|
||
"ft": ["Ft", "Fagopyrum", "Fagopyrum_tataricum"],
|
||
"at": ["At", "Arabidopsis", "Arabidopsis_thaliana"],
|
||
}
|
||
|
||
def _species_aliases(species: str) -> list[str]:
|
||
species = str(species or "").strip()
|
||
aliases = [species] if species else []
|
||
aliases.extend(_SPECIES_RESOURCE_ALIASES.get(species.lower(), []))
|
||
out: list[str] = []
|
||
seen: set[str] = set()
|
||
for alias in aliases:
|
||
key = alias.lower()
|
||
if alias and key not in seen:
|
||
seen.add(key)
|
||
out.append(alias)
|
||
return out
|
||
|
||
def _is_species_resource(path: Path, aliases: list[str]) -> bool:
|
||
if not aliases:
|
||
return True
|
||
name = path.name.lower()
|
||
return any(alias.lower() in name for alias in aliases)
|
||
|
||
def _historical_annotation_roots() -> list[Path]:
|
||
bases = []
|
||
for root in (PROJECT_ROOT, in_dir):
|
||
try:
|
||
bases.append(Path(root).absolute().parent)
|
||
bases.append(Path(root).resolve().parent)
|
||
except OSError:
|
||
bases.append(Path(root).absolute().parent)
|
||
try:
|
||
bases.append(INPUT_ROOT.resolve().parent.parent)
|
||
except OSError:
|
||
pass
|
||
roots = []
|
||
seen_bases = set()
|
||
for base in bases:
|
||
if not base.exists() or base in seen_bases:
|
||
continue
|
||
seen_bases.add(base)
|
||
for project_dir in sorted(base.glob("*.bak")) + sorted(base.glob("*backup*")):
|
||
workspace_root = project_dir / "workspace"
|
||
if not workspace_root.exists():
|
||
continue
|
||
roots.extend(sorted(workspace_root.glob("job_*/annotation/analysis")))
|
||
job_id_for_priority = in_dir.name
|
||
return sorted(
|
||
[root.absolute() for root in roots if root.exists()],
|
||
key=lambda root: (0 if job_id_for_priority and job_id_for_priority in root.parts else 1, str(root)),
|
||
)
|
||
|
||
def _shared_resource_roots() -> list[Path]:
|
||
roots = [
|
||
in_dir,
|
||
ws / "annotation" / "analysis",
|
||
INPUT_ROOT,
|
||
PROJECT_ROOT / "data",
|
||
PROJECT_ROOT / "resources",
|
||
PROJECT_ROOT / "reference",
|
||
PROJECT_ROOT / "references",
|
||
]
|
||
roots.extend(_historical_annotation_roots())
|
||
out: list[Path] = []
|
||
seen: set[Path] = set()
|
||
for root in roots:
|
||
root = Path(root).absolute()
|
||
if root.exists() and root not in seen:
|
||
seen.add(root)
|
||
out.append(root)
|
||
return out
|
||
|
||
def _is_family_member_interproscan(path: Path) -> bool:
|
||
name = path.name.lower()
|
||
return any(
|
||
marker in name
|
||
for marker in (
|
||
"final_members.interproscan",
|
||
"curated_members.interproscan",
|
||
"candidate_members.interproscan",
|
||
)
|
||
)
|
||
|
||
def _prefix_species_family(value: str) -> tuple[str, str]:
|
||
parts = [part for part in str(value or "").split(".") if part]
|
||
species = parts[0] if parts else ""
|
||
family = parts[1] if len(parts) > 1 else ""
|
||
return species, family
|
||
|
||
def _is_family_subset_protein(path: Path, family: str = "") -> bool:
|
||
name = path.name.lower()
|
||
family = str(family or "").lower()
|
||
if "final_members" in name or "curated_members" in name or "candidate_members" in name:
|
||
return True
|
||
if name.startswith("ref_combined"):
|
||
return True
|
||
if family and f".{family}." in name:
|
||
return True
|
||
return False
|
||
|
||
de_files = list(final_dir.glob(f"{prefix}.rnaseq_de.*.DE_genes.txt"))
|
||
de_files = [p for p in de_files if "family_DE_genes" not in p.name]
|
||
if not de_files:
|
||
de_files = [p for p in final_dir.glob("*.DE_genes.txt") if "family_DE_genes" not in p.name]
|
||
if not de_files:
|
||
return (
|
||
"❌ RNA-seq candidate family requires global DE genes (*.DE_genes.txt). "
|
||
"Please run RNA-seq DE first.",
|
||
manifest,
|
||
)
|
||
|
||
species_code, family_name = _prefix_species_family(prefix)
|
||
species_aliases = _species_aliases(species_code)
|
||
preferred_proteins = []
|
||
if species_code:
|
||
preferred_proteins.extend(
|
||
[
|
||
in_dir / f"{species_code}.pep.fasta",
|
||
in_dir / f"{species_code}.pep.fa",
|
||
in_dir / species_code / f"{species_code}.pep.fasta",
|
||
in_dir / species_code / f"{species_code}.pep.fa",
|
||
in_dir / f"{species_code}.protein.fasta",
|
||
in_dir / f"{species_code}.protein.fa",
|
||
in_dir / species_code / f"{species_code}.protein.fasta",
|
||
in_dir / species_code / f"{species_code}.protein.fa",
|
||
in_dir / f"{species_code}.faa",
|
||
in_dir / species_code / f"{species_code}.faa",
|
||
]
|
||
)
|
||
protein_roots = [in_dir, ws / "annotation" / "analysis", *_shared_resource_roots()]
|
||
if in_dir.exists():
|
||
protein_roots.extend([p for p in sorted(in_dir.iterdir()) if p.is_dir()])
|
||
deduped_protein_roots = []
|
||
seen_protein_roots = set()
|
||
for root in protein_roots:
|
||
root = Path(root).absolute()
|
||
if root.exists() and root not in seen_protein_roots:
|
||
seen_protein_roots.add(root)
|
||
deduped_protein_roots.append(root)
|
||
protein_roots = deduped_protein_roots
|
||
protein_candidates = list(preferred_proteins)
|
||
local_protein_roots = {in_dir.absolute(), (ws / "annotation" / "analysis").absolute()}
|
||
shared_protein_roots = set(_shared_resource_roots())
|
||
for root in protein_roots:
|
||
globber = root.rglob if root in shared_protein_roots and root not in local_protein_roots else root.glob
|
||
protein_candidates.extend(
|
||
list(globber("*.pep.fasta"))
|
||
+ list(globber("*.pep.fa"))
|
||
+ list(globber("*.protein.fasta"))
|
||
+ list(globber("*.protein.fa"))
|
||
+ list(globber("*.faa"))
|
||
+ list(globber("*annotation_safe.pep.fasta"))
|
||
)
|
||
protein_candidates = [
|
||
p
|
||
for p in protein_candidates
|
||
if p.exists()
|
||
and p.stat().st_size > 0
|
||
and not _is_family_subset_protein(p, family_name)
|
||
and _is_species_resource(p, species_aliases)
|
||
]
|
||
interpro_candidates = (
|
||
list((ws / "annotation" / "analysis").glob(f"{prefix}.interproscan.tsv"))
|
||
+ list((ws / "annotation" / "analysis").glob("*interproscan.tsv"))
|
||
+ list((ws / "rnaseq_candidate_family" / "analysis").glob(f"{prefix}.rnaseq_candidate_family.interproscan.tsv"))
|
||
+ list(final_dir.glob("*interproscan*.tsv"))
|
||
)
|
||
for root in _shared_resource_roots():
|
||
interpro_candidates.extend(list(root.rglob("*interproscan*.tsv")))
|
||
interpro_candidates = [
|
||
p
|
||
for p in interpro_candidates
|
||
if p.exists()
|
||
and p.stat().st_size > 0
|
||
and not _is_family_member_interproscan(p)
|
||
and _is_species_resource(p, species_aliases)
|
||
]
|
||
if not protein_candidates and not interpro_candidates:
|
||
checked_roots = ", ".join(str(root) for root in _shared_resource_roots())
|
||
expected_roots = [
|
||
in_dir.absolute(),
|
||
(ws / "annotation" / "analysis").absolute(),
|
||
INPUT_ROOT,
|
||
PROJECT_ROOT / "data",
|
||
PROJECT_ROOT / "resources",
|
||
PROJECT_ROOT / "reference",
|
||
PROJECT_ROOT / "references",
|
||
]
|
||
missing_roots = ", ".join(str(root) for root in expected_roots if not Path(root).exists())
|
||
return (
|
||
"❌ RNA-seq candidate family requires a whole target proteome FASTA or a genome-wide InterProScan TSV. "
|
||
"Family-member InterProScan files such as *.final_members.interproscan.tsv are not valid for genome-wide RNA-seq candidate family ranking. "
|
||
f"Requested species={species_code or prefix}; accepted aliases={', '.join(species_aliases) or 'any'}. "
|
||
f"Checked roots: {checked_roots}. "
|
||
f"Missing expected roots: {missing_roots or 'none'}. "
|
||
f"Next action: place {species_code or 'target'}.pep.fasta/{species_code or 'target'}.faa or {species_code or 'target'}.interproscan.tsv under the job input directory or a project shared resource directory.",
|
||
manifest,
|
||
)
|
||
|
||
candidate_cmd = [
|
||
"/home/xzq/miniconda3/envs/test4py311/bin/python",
|
||
"apps/cli/run_rnaseq_candidate_family.py",
|
||
"--workspace", str(ws),
|
||
"--input-dir", str(in_dir),
|
||
"-p", prefix,
|
||
"-j", job.job_id,
|
||
"-t", str(cpu),
|
||
"--applications", annotation_applications_arg,
|
||
"--run-interproscan", str(candidate_run_interproscan or "auto"),
|
||
"--max-deg-proteins", str(candidate_max_deg_proteins or 500),
|
||
]
|
||
if protein_candidates:
|
||
candidate_cmd.extend(["--protein-fasta", str(protein_candidates[0])])
|
||
if interpro_candidates:
|
||
candidate_cmd.extend(["--interproscan-tsv", str(interpro_candidates[0])])
|
||
print(f"[CMD] {' '.join(candidate_cmd)}")
|
||
code = run_cmd_logged(candidate_cmd, log_file, cwd=Path("."))
|
||
candidate_manifest = final_dir / f"{prefix}.rnaseq_candidate_family.manifest.json"
|
||
if code == 0:
|
||
score_note, _score_manifest = _run_rnaseq_family_score_if_ready(
|
||
ws,
|
||
in_dir,
|
||
final_dir,
|
||
prefix,
|
||
job.job_id,
|
||
cpu,
|
||
log_file,
|
||
trigger="rnaseq_candidate_family",
|
||
)
|
||
return f"\u2705 RNA-seq candidate family completed.{score_note}", candidate_manifest
|
||
return f"❌ RNA-seq candidate family failed (exit code: {code}). Log: {log_file}", candidate_manifest if candidate_manifest.exists() else manifest
|
||
|
||
if stage == "rnaseq_family_score":
|
||
score_note, score_manifest = _run_rnaseq_family_score_if_ready(
|
||
ws,
|
||
in_dir,
|
||
final_dir,
|
||
prefix,
|
||
job.job_id,
|
||
cpu,
|
||
log_file,
|
||
trigger="manual_stage",
|
||
)
|
||
if score_manifest.exists():
|
||
try:
|
||
status = (json.loads(score_manifest.read_text(encoding="utf-8")) or {}).get("status", "")
|
||
except Exception:
|
||
status = ""
|
||
if status == "success":
|
||
return f"✅ RNA-seq family score completed.{score_note}", score_manifest
|
||
if status == "skipped":
|
||
return f"⚠️ RNA-seq family score skipped.{score_note}", score_manifest
|
||
return f"⚠️ RNA-seq family score post-processing finished with notice.{score_note}", score_manifest
|
||
|
||
if stage in {"rnaseq_enrich", "rnaseq_go_enrich", "rnaseq_kegg_enrich"}:
|
||
# Step 5: 富集分析
|
||
# 输入：DE genes (来自rnaseq_de), 物种代码
|
||
# 从主final目录查找DE结果（所有阶段结果都应放在这里）
|
||
foreground_mode_val = (rnaseq_foreground_mode or "auto").strip().lower()
|
||
if foreground_mode_val not in {"auto", "global", "family"}:
|
||
foreground_mode_val = "auto"
|
||
all_de_files = list((ws / "final").glob("*DE_genes.txt"))
|
||
family_de_files = [p for p in all_de_files if "family_DE_genes" in p.name]
|
||
global_de_files = [p for p in all_de_files if "family_DE_genes" not in p.name]
|
||
if foreground_mode_val == "auto":
|
||
# 数据驱动：优先家族前景（若可用），否则回退到全基因组前景
|
||
if family_de_files:
|
||
foreground_mode_val = "family"
|
||
elif global_de_files:
|
||
foreground_mode_val = "global"
|
||
else:
|
||
foreground_mode_val = "family"
|
||
if foreground_mode_val == "family":
|
||
de_files = family_de_files
|
||
else:
|
||
de_files = global_de_files
|
||
|
||
if not de_files:
|
||
if foreground_mode_val == "family":
|
||
return (
|
||
"❌ 无法进行 RNA-seq 富集分析：当前选择的是基因家族 foreground，"
|
||
"但没有找到家族差异基因文件 *family_DE_genes.txt。\n\n"
|
||
"前景依赖缺失：需要 family_DE_genes。\n"
|
||
"背景依赖（gene2go/gene2ko 或内置注释）与此错误无关。\n\n"
|
||
"可选处理方式：\n"
|
||
"1. 重新运行 RNA-seq DE 并选择基因家族 foreground；\n"
|
||
"2. 在富集参数中改选全基因组 foreground（若已有全局 DE_genes）。",
|
||
None,
|
||
)
|
||
return (
|
||
"❌ RNA-seq Enrich global foreground requires global DE genes (*.DE_genes.txt). "
|
||
"Please run RNA-seq DE first or upload.",
|
||
None,
|
||
)
|
||
|
||
requested_enrich_stage = stage
|
||
if requested_enrich_stage == "rnaseq_go_enrich":
|
||
enrich_type = "go"
|
||
elif requested_enrich_stage == "rnaseq_kegg_enrich":
|
||
enrich_type = "kegg"
|
||
|
||
enrich_dir_name = {
|
||
"go": "rnaseq_go_enrich",
|
||
"kegg": "rnaseq_kegg_enrich",
|
||
"both": "rnaseq_enrich",
|
||
}.get((enrich_type or "go").strip().lower(), "rnaseq_go_enrich")
|
||
enrich_dir = ws / enrich_dir_name
|
||
enrich_dir.mkdir(parents=True, exist_ok=True)
|
||
final_dir.mkdir(exist_ok=True)
|
||
|
||
print(f"[RNA-seq Enrich] Foreground mode: {foreground_mode_val}")
|
||
print(f"[RNA-seq Enrich] DE files: {len(de_files)}")
|
||
|
||
# 查找自定义GO注释文件。全基因组eggNOG注释是长任务，应通过
|
||
# 单独的后台任务显式启动，不能在交互式富集阶段隐式运行。
|
||
gene2go_files = (
|
||
list(in_dir.glob("*gene2go*.txt")) + list(in_dir.glob("*gene2go*.tsv")) +
|
||
list(final_dir.glob("*gene2go*.txt")) + list(final_dir.glob("*gene2go*.tsv"))
|
||
)
|
||
gene2ko_files = (
|
||
list(in_dir.glob("*gene2ko*.txt")) + list(in_dir.glob("*gene2ko*.tsv")) +
|
||
list(final_dir.glob("*gene2ko*.txt")) + list(final_dir.glob("*gene2ko*.tsv"))
|
||
)
|
||
gene2go_files = [p for p in gene2go_files if _has_non_comment_data(p) and ".test." not in p.name]
|
||
gene2ko_files = [p for p in gene2ko_files if _has_non_comment_data(p) and ".test." not in p.name]
|
||
compatible_gene2go_files, gene2go_overlap_reports = _filter_gene2go_files_by_de_overlap(gene2go_files, final_dir)
|
||
gene2go_mismatch = bool(gene2go_files and gene2go_overlap_reports and not compatible_gene2go_files)
|
||
if compatible_gene2go_files:
|
||
gene2go_files = compatible_gene2go_files
|
||
elif gene2go_mismatch:
|
||
gene2go_files = []
|
||
enrich_type_val = (enrich_type or "go").strip().lower()
|
||
if enrich_type_val not in {"go", "kegg", "both"}:
|
||
enrich_type_val = "go"
|
||
supported_orgdb_species = {"ath", "hsa", "mmu", "rno", "dme", "sce"}
|
||
enrich_species_val = (enrich_species or species_name or "").strip().lower()
|
||
kegg_organism_val = (kegg_organism or "").strip().lower()
|
||
requested_background_source_val = (enrich_background_source or "auto").strip().lower()
|
||
background_source_val = requested_background_source_val
|
||
confirmed_auto_annotation = bool(kwargs.get("confirmedAutoAnnotation") or kwargs.get("confirmed_auto_annotation"))
|
||
if background_source_val in {"annotation", "eggnog", "auto_annotation"}:
|
||
background_source_val = "annotation"
|
||
if gene2go_mismatch and background_source_val in {"gene2go", "gene2ko"} and enrich_type_val in {"go", "both"}:
|
||
report = gene2go_overlap_reports[0] if gene2go_overlap_reports else {}
|
||
return (
|
||
"❌ 无法进行 RNA-seq GO 富集：当前 gene2go 背景与 DE 基因没有交集。\n\n"
|
||
f"gene2go：{report.get('path', '')}\n"
|
||
f"DE genes：{report.get('de_genes', 0)}；背景基因：{report.get('background_genes', 0)}；overlap=0\n"
|
||
f"DE示例：{report.get('de_examples', [])}\n"
|
||
f"背景示例：{report.get('background_examples', [])}\n\n"
|
||
"处理方式：上传/生成与当前 DE 基因 ID 体系一致的 gene2go，或选择本地自动注释重新生成背景。",
|
||
None,
|
||
)
|
||
if background_source_val == "auto":
|
||
need_go = enrich_type_val in {"go", "both"}
|
||
need_kegg = enrich_type_val in {"kegg", "both"}
|
||
has_go_background = bool(gene2go_files) or (need_go and enrich_species_val in supported_orgdb_species)
|
||
has_kegg_background = bool(gene2ko_files)
|
||
if confirmed_auto_annotation and ((need_go and not has_go_background) or (need_kegg and not has_kegg_background)):
|
||
background_source_val = "annotation"
|
||
if background_source_val == "annotation":
|
||
pep_candidates = (
|
||
list(in_dir.glob("*.pep.fasta"))
|
||
+ list(in_dir.glob("*.pep.fa"))
|
||
+ list(in_dir.glob("*.protein.fa"))
|
||
+ list(in_dir.glob("*protein*.fasta"))
|
||
+ list(final_dir.glob("*final_members.pep.fa"))
|
||
)
|
||
if not pep_candidates:
|
||
return (
|
||
"❌ 无法进行富集分析：选择了内置自动注释，但没有找到蛋白序列文件。\n\n"
|
||
"需要上传目标物种的全基因组蛋白序列文件，例如 *.pep.fasta、*.pep.fa 或 *.protein.fa。"
|
||
"上传后可重新选择“本地按需自动注释 + 富集”。",
|
||
None,
|
||
)
|
||
pep_file = pep_candidates[0]
|
||
annotation_start = time.time()
|
||
need_go = enrich_type_val in {"go", "both"}
|
||
need_kegg = enrich_type_val in {"kegg", "both"}
|
||
go_annotation_manifest = _annotation_state_paths(ws, final_dir, prefix, "go")["manifest"]
|
||
kegg_annotation_manifest = _annotation_state_paths(ws, final_dir, prefix, "kegg")["manifest"]
|
||
annotation_manifest = go_annotation_manifest if need_go and not need_kegg else kegg_annotation_manifest if need_kegg and not need_go else final_dir / f"{prefix}.annotation.manifest.json"
|
||
|
||
def refresh_background_files() -> tuple[list[Path], list[Path]]:
|
||
go_files = (
|
||
list(final_dir.glob("*gene2go*.txt")) + list(final_dir.glob("*gene2go*.tsv"))
|
||
+ list(in_dir.glob("*gene2go*.txt")) + list(in_dir.glob("*gene2go*.tsv"))
|
||
)
|
||
ko_files = (
|
||
list(final_dir.glob("*gene2ko*.txt")) + list(final_dir.glob("*gene2ko*.tsv"))
|
||
+ list(in_dir.glob("*gene2ko*.txt")) + list(in_dir.glob("*gene2ko*.tsv"))
|
||
)
|
||
go_files = [p for p in go_files if _has_non_comment_data(p) and ".test." not in p.name]
|
||
ko_files = [p for p in ko_files if _has_non_comment_data(p) and ".test." not in p.name]
|
||
compatible_go_files, _ = _filter_gene2go_files_by_de_overlap(go_files, final_dir)
|
||
return compatible_go_files if compatible_go_files else [], ko_files
|
||
|
||
def run_annotation_scope(scope: str) -> tuple[str, Path | None]:
|
||
state = _annotation_state_paths(ws, final_dir, prefix, scope)
|
||
scope_manifest = state["manifest"]
|
||
progress_file = state["progress"]
|
||
annotation_lock = state["lock"]
|
||
annotation_lock.parent.mkdir(parents=True, exist_ok=True)
|
||
acquired_lock = False
|
||
try:
|
||
if _annotation_progress_active(progress_file, ws, scope, prefix):
|
||
print(f"[RNA-seq Enrich] Existing {scope} background annotation is running; waiting before deciding what is still missing")
|
||
if not _wait_for_annotation_reuse(scope_manifest, progress_file, log_file, scope=scope, ws=ws, prefix=prefix):
|
||
return (
|
||
"❌ 无法进行富集分析：已有本地自动注释任务未能成功完成。\n\n"
|
||
f"scope：{scope}\n"
|
||
f"蛋白文件：{pep_file}\n"
|
||
f"日志：{log_file}\n"
|
||
"处理方式：等待当前自动注释结束，或上传已有背景文件后重试。",
|
||
scope_manifest,
|
||
)
|
||
return "", scope_manifest
|
||
try:
|
||
annotation_lock.mkdir()
|
||
acquired_lock = True
|
||
except FileExistsError:
|
||
print(f"[RNA-seq Enrich] {scope} background annotation lock exists; waiting before deciding what is still missing")
|
||
if _annotation_progress_active(progress_file, ws, scope, prefix):
|
||
should_wait = _wait_for_annotation_reuse(scope_manifest, progress_file, log_file, scope=scope, ws=ws, prefix=prefix)
|
||
else:
|
||
should_wait = False
|
||
if not should_wait:
|
||
shutil.rmtree(annotation_lock, ignore_errors=True)
|
||
try:
|
||
annotation_lock.mkdir()
|
||
acquired_lock = True
|
||
except FileExistsError:
|
||
return (
|
||
"❌ 无法进行富集分析：自动注释锁仍存在，且无法重新获取。\n\n"
|
||
f"scope：{scope}\n"
|
||
f"蛋白文件：{pep_file}\n"
|
||
f"日志：{log_file}\n"
|
||
"处理方式：确认没有同一 job 的自动注释任务正在运行后重试。",
|
||
scope_manifest,
|
||
)
|
||
else:
|
||
return "", scope_manifest
|
||
if not acquired_lock:
|
||
return (
|
||
"❌ 无法进行富集分析：已有本地自动注释任务未能成功完成。\n\n"
|
||
f"scope：{scope}\n"
|
||
f"蛋白文件：{pep_file}\n"
|
||
f"日志：{log_file}\n"
|
||
"处理方式：等待当前自动注释结束，或上传已有背景文件后重试。",
|
||
scope_manifest,
|
||
)
|
||
|
||
annotation_cmd = [
|
||
"/home/xzq/miniconda3/envs/test4py311/bin/python",
|
||
"apps/cli/run_annotation.py",
|
||
"-i", str(pep_file),
|
||
"-o", str(ws),
|
||
"-p", prefix,
|
||
"-j", job.job_id,
|
||
"-t", str(cpu),
|
||
"--applications", annotation_applications_arg,
|
||
"--species-name", species_name or enrich_species or prefix.split(".", 1)[0],
|
||
"--skip-existing",
|
||
"--skip-existing-kegg",
|
||
"--annotation-scope", scope,
|
||
]
|
||
if scope == "go":
|
||
annotation_cmd.append("--skip-kegg")
|
||
else:
|
||
annotation_cmd.extend(["--skip-go", "--require-kegg"])
|
||
label = "GO/gene2go" if scope == "go" else "KEGG/gene2ko"
|
||
print(f"[RNA-seq Enrich] Running local {label} background annotation first: {' '.join(annotation_cmd)}")
|
||
annotation_code = run_cmd_logged(annotation_cmd, log_file, cwd=Path("."))
|
||
if annotation_code != 0:
|
||
return (
|
||
f"❌ 无法进行富集分析：本地 {label} 自动背景注释失败。\n\n"
|
||
f"蛋白文件：{pep_file}\n"
|
||
f"日志：{log_file}\n"
|
||
"处理方式：检查 InterProScan/KofamScan 数据库，或上传已有背景文件后重试。",
|
||
scope_manifest,
|
||
)
|
||
return "", scope_manifest
|
||
finally:
|
||
if acquired_lock:
|
||
shutil.rmtree(annotation_lock, ignore_errors=True)
|
||
|
||
missing_go_background = need_go and not gene2go_files
|
||
missing_kegg_background = need_kegg and not gene2ko_files
|
||
|
||
if missing_go_background:
|
||
message, scope_manifest = run_annotation_scope("go")
|
||
annotation_manifest = scope_manifest or go_annotation_manifest
|
||
if message:
|
||
return message, annotation_manifest
|
||
gene2go_files, gene2ko_files = refresh_background_files()
|
||
missing_go_background = need_go and not gene2go_files
|
||
|
||
if missing_kegg_background:
|
||
message, scope_manifest = run_annotation_scope("kegg")
|
||
annotation_manifest = scope_manifest or kegg_annotation_manifest
|
||
if message:
|
||
return message, annotation_manifest
|
||
gene2go_files, gene2ko_files = refresh_background_files()
|
||
missing_kegg_background = need_kegg and not gene2ko_files
|
||
|
||
gene2go_files, gene2ko_files = refresh_background_files()
|
||
if need_go and not any(_has_non_comment_data(p) for p in gene2go_files):
|
||
return (
|
||
"❌ 自动背景注释已运行，但没有生成可用的 GO 背景映射。\n\n"
|
||
f"蛋白文件：{pep_file}\n"
|
||
f"注释 manifest：{go_annotation_manifest}\n"
|
||
"处理方式：上传目标物种已有的 gene2go.txt/gene2go.tsv 背景文件后重试。",
|
||
go_annotation_manifest,
|
||
)
|
||
if need_kegg and not any(_has_non_comment_data(p) for p in gene2ko_files):
|
||
return (
|
||
"❌ 自动背景注释已运行，但没有生成可用的 KEGG/KO 背景映射。\n\n"
|
||
f"蛋白文件：{pep_file}\n"
|
||
f"注释 manifest：{kegg_annotation_manifest}\n"
|
||
"处理方式：确认 KofamScan profiles 已完整解压，或上传 gene2ko.txt/gene2ko.tsv 后重试。",
|
||
kegg_annotation_manifest,
|
||
)
|
||
background_source_val = "gene2ko" if enrich_type_val == "kegg" else "gene2go"
|
||
print(f"[RNA-seq Enrich] Background annotation completed in {time.time() - annotation_start:.1f}s")
|
||
if background_source_val == "auto":
|
||
if enrich_type_val == "go" and gene2go_files:
|
||
background_source_val = "gene2go"
|
||
elif enrich_type_val == "go" and enrich_species_val in supported_orgdb_species:
|
||
background_source_val = "orgdb"
|
||
elif enrich_type_val == "kegg" and gene2ko_files:
|
||
background_source_val = "gene2ko"
|
||
elif enrich_type_val == "both" and gene2ko_files and gene2go_files:
|
||
background_source_val = "gene2go"
|
||
else:
|
||
return (
|
||
"❌ 无法进行 RNA-seq 富集分析：缺少背景注释文件。\n\n"
|
||
f"当前 job 没有找到 *gene2go*.txt 或 *gene2go*.tsv；物种 {enrich_species_val or species_name or '未指定'} 也不是内置 OrgDb 支持物种 "
|
||
"(ath, hsa, mmu, rno, dme, sce)，因此系统无法自动确定富集背景。\n\n"
|
||
"可选处理方式：\n"
|
||
"1. 上传目标物种已有的 gene2go.txt/gene2go.tsv 背景文件后重新执行。\n"
|
||
"2. 如果要做 KEGG，请同时上传 gene2ko.txt/gene2ko.tsv（gene 到 KO 映射）。\n"
|
||
"3. 在富集分析参数中选择“本地按需自动注释 + 富集”。GO 会补 gene2go，KEGG 会补 gene2ko；"
|
||
"小型/中型蛋白组通常需要数小时，几百 MB 级别数据可能需要半天左右。\n"
|
||
"4. 如果分析的是支持的模式物种，选择对应 OrgDb 物种代码。",
|
||
None,
|
||
)
|
||
if background_source_val == "orgdb" and not enrich_species_val:
|
||
return (
|
||
"❌ RNA-seq Enrich built-in OrgDb requires an explicit species code. "
|
||
"Supported species: ath, hsa, mmu, rno, dme, sce.",
|
||
None,
|
||
)
|
||
if background_source_val == "orgdb" and enrich_species_val not in supported_orgdb_species:
|
||
return (
|
||
f"❌ RNA-seq Enrich built-in OrgDb does not support species '{enrich_species_val}'. "
|
||
"Supported species: ath, hsa, mmu, rno, dme, sce. "
|
||
"Please upload a gene2go background file or run local InterProScan annotation as a separate background task.",
|
||
None,
|
||
)
|
||
if background_source_val in {"gene2go", "gene2ko"} and enrich_type_val in {"go", "both"} and not gene2go_files:
|
||
return (
|
||
"❌ 无法进行 RNA-seq 富集分析：你选择了 gene2go 背景文件，但当前 job 中没有找到 "
|
||
"*gene2go*.txt 或 *gene2go*.tsv。\n\n"
|
||
"请上传背景文件，或改选“内置自动注释 + 富集”。自动注释会作为后台长任务运行，"
|
||
"完成并生成 gene2go 后继续富集分析。",
|
||
None,
|
||
)
|
||
if enrich_type_val in {"kegg", "both"} and not gene2ko_files:
|
||
return (
|
||
"❌ 无法进行 KEGG 富集分析：缺少 gene2ko 背景文件（gene 到 KO 映射）。\n\n"
|
||
"请上传 *gene2ko*.txt/*.tsv，或选择“本地按需自动注释 + 富集”自动生成后再执行。",
|
||
None,
|
||
)
|
||
if enrich_type_val in {"kegg", "both"} and background_source_val == "orgdb":
|
||
return (
|
||
"❌ 当前 KEGG 富集不走 OrgDb keyType 背景。请使用 gene2ko 背景（上传或自动注释生成）。\n"
|
||
"说明：GO 依赖 OrgDb keyType；KEGG 依赖 KEGG organism code 与 KO/ID 映射，这是两套参数。",
|
||
None,
|
||
)
|
||
|
||
# 获取top_pathways参数（从前端传入，默认20）
|
||
top_pathways_val = top_pathways if top_pathways else 20
|
||
try:
|
||
pvalue_cutoff_val = float(enrich_pvalue_cutoff)
|
||
except (TypeError, ValueError):
|
||
pvalue_cutoff_val = 0.05
|
||
if not (0 <= pvalue_cutoff_val <= 1):
|
||
pvalue_cutoff_val = 0.05
|
||
try:
|
||
qvalue_cutoff_val = float(enrich_qvalue_cutoff)
|
||
except (TypeError, ValueError):
|
||
qvalue_cutoff_val = 0.05
|
||
if not (0 <= qvalue_cutoff_val <= 1):
|
||
qvalue_cutoff_val = 0.05
|
||
go_obo_val = str(go_obo or "data/go/go-basic.obo").strip() or "data/go/go-basic.obo"
|
||
kegg_level_val = str(kegg_level or "pathway").strip().lower()
|
||
if kegg_level_val not in {"pathway", "ko"}:
|
||
kegg_level_val = "pathway"
|
||
kegg_filter_scope_val = str(kegg_filter_scope or "plant").strip().lower()
|
||
if kegg_filter_scope_val not in {"plant", "all"}:
|
||
kegg_filter_scope_val = "plant"
|
||
kegg_highlight_mode_val = str(kegg_highlight_mode or "auto").strip().lower()
|
||
if kegg_highlight_mode_val not in {"auto", "ko", "organism"}:
|
||
kegg_highlight_mode_val = "auto"
|
||
enrich_plot_type_val = str(enrich_plot_type or "r_plot").strip().lower()
|
||
if enrich_plot_type_val not in {"svg", "r_plot"}:
|
||
enrich_plot_type_val = "r_plot"
|
||
|
||
enrich_cmd = [
|
||
"/home/xzq/miniconda3/envs/rnaseq/bin/python", "apps/cli/run_rnaseq_enrich.py",
|
||
"-i", str(ws / "final"),
|
||
"-s", enrich_species_val,
|
||
"-o", str(enrich_dir),
|
||
"-p", prefix,
|
||
"-j", job.job_id,
|
||
"-t", str(cpu),
|
||
"--top-pathways", str(top_pathways_val),
|
||
"--background-source", background_source_val,
|
||
"--key-type", enrich_key_type or "auto",
|
||
"--pvalue-cutoff", str(pvalue_cutoff_val),
|
||
"--qvalue-cutoff", str(qvalue_cutoff_val),
|
||
"--go-obo", go_obo_val,
|
||
"--foreground-mode", foreground_mode_val,
|
||
"--enrich-type", enrich_type_val,
|
||
"--kegg-organism", kegg_organism_val,
|
||
"--kegg-level", kegg_level_val,
|
||
"--kegg-filter-scope", kegg_filter_scope_val,
|
||
"--kegg-highlight-mode", kegg_highlight_mode_val,
|
||
"--stage", enrich_dir_name,
|
||
"--plot-type", enrich_plot_type_val,
|
||
]
|
||
if kegg_no_network:
|
||
enrich_cmd.append("--no-kegg-network")
|
||
if kegg_skip_pathview:
|
||
enrich_cmd.append("--skip-pathview")
|
||
if foreground_mode_val == "family":
|
||
family_name = (prefix)
|
||
enrich_cmd.extend(["--family-name", family_name])
|
||
|
||
if background_source_val in {"gene2go", "gene2ko"}:
|
||
if gene2go_files:
|
||
gene2go_file = gene2go_files[0]
|
||
enrich_cmd.extend(["-g", str(gene2go_file)])
|
||
print(f"[RNA-seq Enrich] Custom GO annotation: {gene2go_file.name}")
|
||
if enrich_type_val in {"kegg", "both"} and gene2ko_files:
|
||
gene2ko_file = gene2ko_files[0]
|
||
enrich_cmd.extend(["--gene2ko", str(gene2ko_file)])
|
||
print(f"[RNA-seq Enrich] Custom KEGG annotation: {gene2ko_file.name}")
|
||
else:
|
||
print(f"[RNA-seq Enrich] Built-in OrgDb species: {enrich_species_val}, keyType={enrich_key_type or 'auto'}")
|
||
|
||
print(f"[CMD] {' '.join(enrich_cmd)}")
|
||
code = run_cmd_logged(enrich_cmd, log_file, cwd=Path("."))
|
||
|
||
if code == 0:
|
||
# 统计输出文件
|
||
if enrich_dir_name == "rnaseq_go_enrich":
|
||
enrich_result_files = list(final_dir.glob(f"{prefix}.rnaseq_go_enrich.*.GO_enrich.txt"))
|
||
elif enrich_dir_name == "rnaseq_kegg_enrich":
|
||
enrich_result_files = list(final_dir.glob(f"{prefix}.rnaseq_kegg_enrich.*.KEGG_enrich.txt"))
|
||
else:
|
||
enrich_result_files = (
|
||
list(final_dir.glob(f"{prefix}.rnaseq_enrich.*.GO_enrich.txt"))
|
||
+ list(final_dir.glob(f"{prefix}.rnaseq_enrich.*.KEGG_enrich.txt"))
|
||
)
|
||
file_count = len(enrich_result_files)
|
||
|
||
enrich_manifest = final_dir / f"{prefix}.{enrich_dir_name}.manifest.json"
|
||
return f"✅ RNA-seq Enrich completed. {file_count} files in final/", enrich_manifest
|
||
else:
|
||
return f"❌ RNA-seq Enrich failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
# ========== RNA-seq 完整流程 ==========
|
||
if stage == "rnaseq":
|
||
# RNA-seq 阶段需要：原始FASTQ文件 + 样本配置 + 参考基因组/GTF
|
||
# 两种输入模式：用户上传 或 检索下载
|
||
|
||
# 直接使用函数参数，不再从kwargs获取（避免被默认值覆盖）
|
||
print(f"[DEBUG rnaseq] stage={stage}, rnaseq_mode={rnaseq_mode}, species_name={species_name}")
|
||
|
||
# FASTQ 的 R1/R2 角色由 samples.yaml 定义，不从文件名推断。
|
||
sample_config = _resolve_rnaseq_sample_config(in_dir, ws, rnaseq_sample_config)
|
||
|
||
# 查找参考基因组和GTF
|
||
ref_genome = None
|
||
gtf_file = None
|
||
|
||
ref_genome_files = list(in_dir.glob("*.fa")) + list(in_dir.glob("*.fasta")) \
|
||
+ list(in_dir.glob("*.fa.gz")) + list(in_dir.glob("*.fasta.gz"))
|
||
if ref_genome_files:
|
||
ref_genome = ref_genome_files[0]
|
||
|
||
gtf_files = list(in_dir.glob("*.gtf")) + list(in_dir.glob("*.gff")) \
|
||
+ list(in_dir.glob("*.gtf.gz")) + list(in_dir.glob("*.gff.gz"))
|
||
if gtf_files:
|
||
gtf_file = gtf_files[0]
|
||
|
||
# 检索下载模式 - 直接返回搜索结果，不需要本地文件
|
||
if rnaseq_mode == "download" and species_name:
|
||
print(f"[RNA-seq] 检索下载模式: {species_name}")
|
||
|
||
species_underscore = species_name.replace(" ", "_").lower()
|
||
|
||
# 构建Ensembl FTP链接
|
||
ftp_fasta = f"ftp://ftp.ensemblgenomes.ebi.ac.uk/pub/current/plants/fasta/{species_underscore}/dna/"
|
||
ftp_gtf = f"ftp://ftp.ensemblgenomes.ebi.ac.uk/pub/plants/current/gff3/{species_underscore}/"
|
||
|
||
return f"🔍 物种: **{species_name}**\n\n" \
|
||
f"**参考基因组FTP:**\n```\n{ftp_fasta}\n```\n\n" \
|
||
f"**GTF注释FTP:**\n```\n{ftp_gtf}\n```\n\n" \
|
||
f"请手动下载这些文件上传到input目录，或使用上传模式直接提供文件。\n\n" \
|
||
f"**下载命令示例:**\n```bash\n# 下载基因组\nwget -r -np -nH --cut-dirs=4 {ftp_fasta}\n# 下载GTF\nwget -r -np -nH --cut-dirs=4 {ftp_gtf}\n```\n\n" \
|
||
f"或者你可以使用上传模式，直接提供本地文件。", manifest
|
||
|
||
# 上传模式才需要检查本地文件
|
||
if not sample_config:
|
||
return "❌ RNA-seq stage requires sample configuration file (samples.yaml). Please upload.", manifest
|
||
|
||
if not ref_genome:
|
||
return "❌ RNA-seq stage requires reference genome file (*.fa or *.fasta). Please upload.", manifest
|
||
|
||
if not gtf_file:
|
||
return "❌ RNA-seq stage requires GTF/GFF annotation file. Please upload.", manifest
|
||
|
||
print(f"[RNA-seq] 样本配置: {sample_config.name}")
|
||
print(f"[RNA-seq] 参考基因组: {ref_genome.name}")
|
||
print(f"[RNA-seq] GTF文件: {gtf_file.name}")
|
||
|
||
# 创建输出目录
|
||
rnaseq_output_dir = ws / "rnaseq"
|
||
rnaseq_output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# Step 1: 质控 (QC)
|
||
print("[RNA-seq] Step 1: Quality Control...")
|
||
qc_dir = rnaseq_output_dir / "qc"
|
||
qc_dir.mkdir(exist_ok=True)
|
||
|
||
qc_cmd = [
|
||
"/home/xzq/miniconda3/envs/rnaseq/bin/python", "apps/cli/run_rnaseq_qc.py",
|
||
"-i", str(in_dir),
|
||
"-c", str(sample_config),
|
||
"-o", str(qc_dir),
|
||
"-j", job.job_id,
|
||
"-t", str(cpu),
|
||
"--resume-existing",
|
||
]
|
||
qc_tool = str(rnaseq_qc_tool or "fastp").strip().lower()
|
||
if qc_tool not in {"cutadapt", "fastp"}:
|
||
return f"❌ RNA-seq QC tool not supported: {rnaseq_qc_tool}. Choose cutadapt or fastp.", manifest
|
||
qc_cmd.extend(["--qc-tool", qc_tool])
|
||
if qc_tool == "fastp" and str(rnaseq_fastp_bin or "").strip():
|
||
qc_cmd.extend(["--fastp-bin", str(rnaseq_fastp_bin).strip()])
|
||
if qc_tool == "fastp":
|
||
qc_cmd.extend([
|
||
"--cut-window-size", str(int(rnaseq_fastp_cut_window_size or 4)),
|
||
"--cut-mean-quality", str(int(rnaseq_fastp_cut_mean_quality or 20)),
|
||
"--qualified-quality-phred", str(int(rnaseq_fastp_qualified_quality_phred or 20)),
|
||
"--unqualified-percent-limit", str(int(rnaseq_fastp_unqualified_percent_limit or 30)),
|
||
"--n-base-limit", str(int(rnaseq_fastp_n_base_limit or 5)),
|
||
"--length-required", str(int(rnaseq_fastp_length_required or 30)),
|
||
])
|
||
qc_cmd.append("--detect-adapter-for-pe" if _bool_with_default(rnaseq_fastp_detect_adapter_for_pe, True) else "--no-detect-adapter-for-pe")
|
||
qc_cmd.append("--cut-front" if _bool_with_default(rnaseq_fastp_cut_front, True) else "--no-cut-front")
|
||
qc_cmd.append("--cut-tail" if _bool_with_default(rnaseq_fastp_cut_tail, True) else "--no-cut-tail")
|
||
|
||
print(f"[CMD] {' '.join(qc_cmd)}")
|
||
code = run_cmd_logged(qc_cmd, log_file, cwd=Path("."))
|
||
|
||
if code != 0:
|
||
return f"❌ RNA-seq QC failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
# 返回进度信息
|
||
return f"✅ RNA-seq 流程正在开发中...\n\n" \
|
||
f"已完成: Step 1 - QC (质控)\n" \
|
||
f"待集成: Step 2 - Alignment (HISAT2)\n" \
|
||
f"待集成: Step 3 - Quantification (featureCounts)\n" \
|
||
f"待集成: Step 4 - Differential Expression\n" \
|
||
f"待集成: Step 5 - Enrichment Analysis\n\n" \
|
||
f"QC输出目录: {qc_dir}\n" \
|
||
f"日志: {log_file}", manifest
|
||
|
||
# ========== Annotation 阶段 (GO功能注释) ==========
|
||
if stage == "annotation":
|
||
pep_files = (
|
||
list(in_dir.glob("*.pep.fa"))
|
||
+ list(in_dir.glob("*.pep.fasta"))
|
||
+ list(in_dir.glob("*.protein.fa"))
|
||
+ list(in_dir.glob("*.protein.fasta"))
|
||
+ list(in_dir.glob("*protein*.fa"))
|
||
+ list(in_dir.glob("*protein*.fasta"))
|
||
+ list((ws / "final").glob("*final_members.pep.fa"))
|
||
+ list((ws / "final").glob("*final_members.pep.fasta"))
|
||
)
|
||
|
||
if not pep_files:
|
||
# 也搜索input目录
|
||
pep_files = (
|
||
list(in_dir.glob("*/*.pep.fa"))
|
||
+ list(in_dir.glob("*/*.pep.fasta"))
|
||
+ list(in_dir.glob("*/*.protein.fa"))
|
||
+ list(in_dir.glob("*/*.protein.fasta"))
|
||
)
|
||
|
||
if not pep_files:
|
||
return "❌ Annotation stage requires protein sequence file (*.pep.fa or *.protein.fa). Please upload.", manifest
|
||
|
||
pep_file = pep_files[0]
|
||
|
||
annotation_manifest = ws / "final" / f"{prefix}.annotation.manifest.json"
|
||
annotation_cmd = [
|
||
"/home/xzq/miniconda3/envs/test4py311/bin/python",
|
||
"apps/cli/run_annotation.py",
|
||
"-i",
|
||
str(pep_file),
|
||
"-o",
|
||
str(ws),
|
||
"-p",
|
||
prefix,
|
||
"-j",
|
||
job.job_id,
|
||
"-t",
|
||
str(cpu),
|
||
"--applications",
|
||
annotation_applications_arg,
|
||
"--species-name",
|
||
species_name or enrich_species or prefix.split(".", 1)[0],
|
||
"--annotation-scope",
|
||
"both",
|
||
]
|
||
print(f"[Annotation] 使用蛋白文件: {pep_file.name}")
|
||
print(f"[CMD] {' '.join(annotation_cmd)}")
|
||
code = run_cmd_logged(annotation_cmd, log_file, cwd=Path("."))
|
||
if code != 0:
|
||
return f"❌ Annotation failed (exit code: {code}). Log: {log_file}", annotation_manifest
|
||
|
||
try:
|
||
manifest_data = json.loads(annotation_manifest.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
manifest_data = {}
|
||
metrics = manifest_data.get("metrics", {})
|
||
gene2go = ws / "final" / f"{prefix}.gene2go.txt"
|
||
if not _has_non_comment_data(gene2go):
|
||
return (
|
||
"❌ Annotation 已运行，但没有生成可用于富集分析的 gene2go 背景映射。\n\n"
|
||
f"蛋白文件：{pep_file}\n"
|
||
f"输出文件：{gene2go}\n\n"
|
||
"可能原因：InterProScan 没有为当前蛋白集匹配到 GO term，或原始 TSV 解析失败。",
|
||
annotation_manifest,
|
||
)
|
||
|
||
return (
|
||
"✅ Annotation complete! "
|
||
f"{metrics.get('canonical_genes_with_go', 0)} genes, "
|
||
f"{metrics.get('go_term_count', 0)} GO terms. Method: interproscan",
|
||
annotation_manifest,
|
||
)
|
||
|
||
|
||
if stage == "combo":
|
||
combo_manifest = ws / "final" / f"{prefix}.combo.manifest.json"
|
||
combo_scope_path = None
|
||
combo_scope_data = None
|
||
if scope_file:
|
||
combo_scope_path = Path(scope_file).expanduser()
|
||
if not combo_scope_path.is_absolute():
|
||
for candidate in (in_dir / combo_scope_path, ws / combo_scope_path, Path(scope_file)):
|
||
if candidate.exists():
|
||
combo_scope_path = candidate
|
||
break
|
||
if not combo_scope_path.exists():
|
||
return f"❌ Combo scope 文件不存在: {scope_file}", combo_manifest
|
||
try:
|
||
combo_scope_data = read_scope(combo_scope_path)
|
||
combo_scope_id = str((combo_scope_data or {}).get("scope_id") or "").strip()
|
||
if combo_scope_id and analysis_route:
|
||
expected_member_set = member_set_id_for(prefix, combo_scope_data, analysis_route)
|
||
if expected_member_set and member_set_id != expected_member_set:
|
||
print(
|
||
f"[Combo] member_set_id synchronized with scope_file: "
|
||
f"{member_set_id or '<empty>'} -> {expected_member_set}"
|
||
)
|
||
member_set_id = expected_member_set
|
||
combo_manifest = ws / "final" / f"{stage_output_prefix('combo', prefix, combo_scope_data, analysis_route, member_set_id)}.combo.manifest.json"
|
||
except Exception as exc:
|
||
return f"❌ Combo scope 文件无法读取: {scope_file}; {exc}", combo_manifest
|
||
|
||
raw_combo_panels = combo_panels or kwargs.get("comboPanels") or kwargs.get("combo_panels")
|
||
if raw_combo_panels:
|
||
inferred_combo_panels = raw_combo_panels
|
||
else:
|
||
inferred_combo_panels = infer_completed_combo_panels(
|
||
ws / "final",
|
||
prefix,
|
||
combo_scope_data or {},
|
||
analysis_route,
|
||
member_set_id,
|
||
)
|
||
try:
|
||
selected_combo_panels = normalize_combo_panels(inferred_combo_panels)
|
||
except ValueError as exc:
|
||
return f"❌ Combo panel 参数错误：{exc}", combo_manifest
|
||
|
||
# Combo 阶段需要先检查依赖：phylogeny + 已选择的 motif/structure/domain 面板
|
||
deps = {
|
||
dep_stage: first_existing_stage_manifest(
|
||
ws / "final",
|
||
dep_stage,
|
||
prefix,
|
||
combo_scope_data or {},
|
||
analysis_route,
|
||
member_set_id,
|
||
)
|
||
for dep_stage in ["phylogeny", *selected_combo_panels]
|
||
}
|
||
missing = [k for k, v in deps.items() if not v]
|
||
if missing:
|
||
return f"❌ Combo 阶段缺少依赖：{', '.join(missing)}。请先运行这些阶段。", combo_manifest
|
||
|
||
# 查找 subfamily 文件，优先使用当前 combo scope 对应的确认结果。
|
||
combo_scope_id = str(combo_scope_data.get("scope_id") or "") if combo_scope_data else ""
|
||
subfamily_file = None
|
||
disable_combo_subfamily_auto = should_disable_cli_subfamily_auto(analysis_route, subfamily_label_source)
|
||
if not disable_combo_subfamily_auto:
|
||
subfamily_file = resolve_subfamily_for_plotting(ws / "final", in_dir, prefix, combo_scope_id, subfamily_label_source)
|
||
|
||
cmd = [
|
||
sys.executable, "apps/cli/run_combo.py",
|
||
"--prefix", prefix,
|
||
"--workspace", str(ws),
|
||
"--panels", ",".join(selected_combo_panels),
|
||
]
|
||
if combo_output_label:
|
||
cmd.extend(["--output-label", combo_output_label])
|
||
if disable_combo_subfamily_auto:
|
||
cmd.extend(["--subfamily", "NONE"])
|
||
elif subfamily_file and subfamily_file.exists():
|
||
cmd.extend(["--subfamily", str(subfamily_file)])
|
||
if combo_scope_path:
|
||
cmd.extend(["--scope-file", str(combo_scope_path)])
|
||
if analysis_route:
|
||
cmd.extend(["--analysis-route", analysis_route])
|
||
if member_set_id:
|
||
cmd.extend(["--member-set-id", member_set_id])
|
||
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
combo_manifest = first_existing_stage_manifest(
|
||
ws / "final", "combo", prefix, combo_scope_data or {}, analysis_route, member_set_id
|
||
) or combo_manifest
|
||
if combo_scope_data and analysis_route == "subfamily_evidence":
|
||
materialize_compact_combo_outputs(ws / "final", prefix, combo_manifest)
|
||
if code != 0:
|
||
return f"❌ Combo 失败，日志：{log_file}", combo_manifest
|
||
if not combo_manifest.exists():
|
||
return f"⚠️ Combo 完成但未找到manifest。日志：{log_file}", combo_manifest
|
||
return f"✅ Combo 完成。日志：{log_file}", combo_manifest
|
||
|
||
if stage == "all":
|
||
msgs = []
|
||
|
||
if not (ws / "final" / f"{prefix}.identify.manifest.json").exists():
|
||
m1, manifest = run_stage("identify", job, prefix, cpu=cpu,
|
||
required_pfams=required_pfams,
|
||
ref_species=ref_species,
|
||
retention_days=retention_days)
|
||
msgs.append(m1)
|
||
else:
|
||
msgs.append("✅ Identify 已完成，跳过")
|
||
|
||
m2, manifest = run_stage("phylogeny", job, prefix, cpu=cpu)
|
||
msgs.append(m2)
|
||
|
||
m3, manifest = run_stage("structure", job, prefix, cpu=cpu)
|
||
msgs.append(m3)
|
||
|
||
m4, manifest = run_stage("domain", job, prefix, cpu=cpu)
|
||
msgs.append(m4)
|
||
|
||
m5, manifest = run_stage("motif", job, prefix, cpu=cpu)
|
||
msgs.append(m5)
|
||
|
||
m6, manifest = run_stage("promoter", job, prefix, cpu=cpu)
|
||
msgs.append(m6)
|
||
|
||
m7, manifest = run_stage("combo", job, prefix, cpu=cpu)
|
||
msgs.append(m7)
|
||
|
||
# ppi
|
||
m8, manifest = run_stage("ppi", job, prefix, cpu=cpu, ppi_species=ppi_species, ppi_score=ppi_score)
|
||
msgs.append(m8)
|
||
|
||
# summary
|
||
m9, manifest = run_stage("summary", job, prefix, cpu=cpu)
|
||
msgs.append(m9)
|
||
|
||
|
||
# ========== Ortholog 阶段 ==========
|
||
if stage == "ortholog":
|
||
script = Path("apps/cli/run_ortholog.py")
|
||
if not script.exists():
|
||
return "❌ Ortholog script not found: apps/cli/run_ortholog.py", manifest
|
||
|
||
ortholog_dir = ws / "ortholog"
|
||
ortholog_dir.mkdir(parents=True, exist_ok=True)
|
||
final_dir.mkdir(exist_ok=True)
|
||
|
||
ortholog_scope_path = None
|
||
ortholog_scope_data = None
|
||
family = prefix.split(".")[-1] if "." in prefix else prefix
|
||
run_specs = []
|
||
reference_species = ""
|
||
ppi_owner_storage_key = str(ppi_reference_storage_key or "").strip()
|
||
ppi_owner_taxon_id = str(ppi_reference_taxon_id or "").strip()
|
||
ppi_owned_producer = bool(
|
||
str(dependency_owner or "").strip().lower() == "ppi"
|
||
and ppi_reference_user_confirmed
|
||
and ppi_owner_storage_key
|
||
)
|
||
if scope_file:
|
||
ortholog_scope_path = resolve_scope_path(scope_file, in_dir, ws)
|
||
if not ortholog_scope_path.exists():
|
||
return f"❌ Ortholog scope 文件不存在: {scope_file}", manifest
|
||
try:
|
||
ortholog_scope_data = read_scope(ortholog_scope_path)
|
||
except Exception as exc:
|
||
return f"❌ Ortholog scope 文件无法读取: {scope_file}; {exc}", manifest
|
||
family = str(ortholog_scope_data.get("family") or family)
|
||
targets = canonicalize_species_codes_for_job(in_dir, [str(x) for x in ortholog_scope_data.get("target_species", []) if str(x)])
|
||
references = (
|
||
[ppi_owner_storage_key]
|
||
if ppi_owned_producer
|
||
else canonicalize_species_codes_for_job(
|
||
in_dir,
|
||
[str(x) for x in ortholog_scope_data.get("reference_species", []) if str(x)],
|
||
)
|
||
)
|
||
if not targets:
|
||
return "❌ Ortholog scope 中没有 target_species，无法确定 query 物种。", manifest
|
||
if not references:
|
||
return "❌ Ortholog scope 中没有 reference_species，无法确定 subject/reference 物种。", manifest
|
||
reference_species = references[0]
|
||
subject_fasta = first_existing_scope_file(ortholog_scope_data, reference_species, ["family_pep", "pep"])
|
||
if not subject_fasta:
|
||
subject_fasta = resolve_species_family_pep(ws, final_dir, in_dir, reference_species, family)
|
||
if not subject_fasta:
|
||
subject_fasta = resolve_species_sequence_file(
|
||
ws,
|
||
final_dir,
|
||
in_dir,
|
||
ortholog_scope_data,
|
||
reference_species,
|
||
family,
|
||
"pep",
|
||
family_only=False,
|
||
)
|
||
if not subject_fasta:
|
||
return (
|
||
f"❌ Ortholog stage requires reference protein FASTA for {reference_species}. "
|
||
f"Expected {reference_species}.{family}.pep.fa or {reference_species}.pep.fasta.",
|
||
manifest,
|
||
)
|
||
for species in targets:
|
||
query_fasta = resolve_species_family_pep(ws, final_dir, in_dir, species, family)
|
||
if not query_fasta:
|
||
return (
|
||
f"❌ Ortholog stage requires target family protein FASTA for {species}. "
|
||
f"Expected identify/{species}/final/{species}.final_gfmembers.pep.fa or {species}.{family}.pep.fa.",
|
||
manifest,
|
||
)
|
||
run_specs.append((species, reference_species, query_fasta, subject_fasta, f"{prefix}.{species}_to_{reference_species}"))
|
||
else:
|
||
query_fasta = final_dir / f"{prefix}.final_members.pep.fa"
|
||
if not query_fasta.exists():
|
||
query_fasta = final_dir / f"{prefix}.final_members.pep.fasta"
|
||
if not query_fasta.exists():
|
||
return "❌ Ortholog stage requires identify protein output. Please run identify first.", manifest
|
||
|
||
ref_candidates = [
|
||
in_dir / "Ath.pep.fasta",
|
||
in_dir / "Ath.pep.fa",
|
||
in_dir / "At.pep.fasta",
|
||
in_dir / "At.pep.fa",
|
||
in_dir / f"At.{family}.pep.fa",
|
||
in_dir / f"At.{family}.pep.fasta",
|
||
in_dir / "SPL_Ath.fasta",
|
||
]
|
||
subject_fasta = next((p for p in ref_candidates if p.exists()), None)
|
||
if subject_fasta is None:
|
||
return "❌ Ortholog stage requires reference protein FASTA, e.g. At.SPL.pep.fa, Ath.pep.fasta or SPL_Ath.fasta.", manifest
|
||
reference_species = subject_fasta.stem.split(".")[0]
|
||
run_specs.append(("", reference_species, query_fasta, subject_fasta, prefix))
|
||
|
||
from gfa_core.ortholog_materialize import (
|
||
OrthologRunRecord,
|
||
build_ortholog_run_identity,
|
||
build_run_record_from_spec,
|
||
materialize_ortholog_final_outputs,
|
||
ortholog_run_identity_matches,
|
||
ortholog_pairs_has_data,
|
||
write_ortholog_run_sidecar,
|
||
)
|
||
|
||
scope_fields: dict[str, Any] = {}
|
||
if ortholog_scope_data:
|
||
source_scope_mode = str(ortholog_scope_data.get("scope_mode") or "")
|
||
preserve_target_only_scope = ppi_owned_producer and source_scope_mode == "target_only"
|
||
effective_scope_id = str(ortholog_scope_data.get("scope_id") or "")
|
||
if ppi_owned_producer:
|
||
effective_scope_id = _effective_ppi_owner_scope_id(
|
||
family=family,
|
||
selected_targets=[record[0] for record in run_specs],
|
||
scope_data=ortholog_scope_data,
|
||
reference_storage_key=reference_species,
|
||
)
|
||
scope_fields = {
|
||
"scope_file": str(ortholog_scope_path),
|
||
"scope_id": effective_scope_id,
|
||
"scope_mode": ortholog_scope_data.get("scope_mode", ""),
|
||
"target_species": [record[0] for record in run_specs],
|
||
"reference_species": (
|
||
ortholog_scope_data.get("reference_species", [])
|
||
if preserve_target_only_scope
|
||
else [reference_species] if reference_species else []
|
||
),
|
||
"comparative_species": ortholog_scope_data.get("comparative_species", []),
|
||
"selected_species": (
|
||
ortholog_scope_data.get("selected_species", [record[0] for record in run_specs])
|
||
if preserve_target_only_scope
|
||
else [record[0] for record in run_specs] + ([reference_species] if reference_species else [])
|
||
),
|
||
"family": family,
|
||
}
|
||
|
||
run_records: list[OrthologRunRecord] = []
|
||
run_identities: list[dict[str, Any]] = []
|
||
for query_species, subject_species, query_fasta, subject_fasta, run_prefix in run_specs:
|
||
pairs_path = ortholog_dir / f"{run_prefix}_pairs.txt"
|
||
expected_identity = build_ortholog_run_identity(
|
||
query_species=query_species,
|
||
reference_taxon_id=ppi_owner_taxon_id if ppi_owned_producer else "",
|
||
reference_storage_key=reference_species,
|
||
query_fasta=query_fasta,
|
||
reference_pep=subject_fasta,
|
||
scope_id=str(scope_fields.get("scope_id") or ""),
|
||
member_set_id="" if ppi_owned_producer else member_set_id,
|
||
mapping_path=pairs_path,
|
||
)
|
||
existing = None
|
||
if ortholog_run_identity_matches(pairs_path, expected_identity):
|
||
existing = build_run_record_from_spec(
|
||
ortholog_dir,
|
||
query_species=query_species,
|
||
subject_species=subject_species,
|
||
query_fasta=query_fasta,
|
||
subject_fasta=subject_fasta,
|
||
run_prefix=run_prefix,
|
||
)
|
||
if existing is not None:
|
||
print(f"[INFO] Reusing identity-matched ortholog pairs for {query_species or 'query'} -> {subject_species}: {existing.pairs_src}")
|
||
run_records.append(existing)
|
||
run_identities.append(expected_identity)
|
||
continue
|
||
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--query", str(query_fasta),
|
||
"--subject", str(subject_fasta),
|
||
"--output", str(ortholog_dir),
|
||
"--prefix", run_prefix,
|
||
"--evalue", str(blast_evalue),
|
||
]
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
if code != 0:
|
||
return f"❌ Ortholog analysis failed for {query_species or 'query'} -> {subject_species} (exit code: {code}). Log: {log_file}", manifest
|
||
record = build_run_record_from_spec(
|
||
ortholog_dir,
|
||
query_species=query_species,
|
||
subject_species=subject_species,
|
||
query_fasta=query_fasta,
|
||
subject_fasta=subject_fasta,
|
||
run_prefix=run_prefix,
|
||
)
|
||
if record is None:
|
||
pairs_src = ortholog_dir / f"{run_prefix}_pairs.txt"
|
||
detail = "empty pairs file" if pairs_src.exists() else "pairs file not found"
|
||
return (
|
||
f"⚠️ Ortholog finished but valid pairs not found for {query_species or 'query'} -> {subject_species} ({detail}). Log: {log_file}",
|
||
manifest,
|
||
)
|
||
completed_identity = build_ortholog_run_identity(
|
||
query_species=query_species,
|
||
reference_taxon_id=ppi_owner_taxon_id if ppi_owned_producer else "",
|
||
reference_storage_key=reference_species,
|
||
query_fasta=query_fasta,
|
||
reference_pep=subject_fasta,
|
||
scope_id=str(scope_fields.get("scope_id") or ""),
|
||
member_set_id="" if ppi_owned_producer else member_set_id,
|
||
mapping_path=record.pairs_src,
|
||
)
|
||
write_ortholog_run_sidecar(record.pairs_src, completed_identity)
|
||
run_records.append(record)
|
||
run_identities.append(completed_identity)
|
||
|
||
try:
|
||
pair_count, ortholog_manifest = materialize_ortholog_final_outputs(
|
||
final_dir=final_dir,
|
||
prefix=prefix,
|
||
run_records=run_records,
|
||
job_id=job.job_id,
|
||
workspace=ws,
|
||
reference_species=reference_species,
|
||
reference_taxon_id=ppi_owner_taxon_id if ppi_owned_producer else "",
|
||
reference_storage_key=reference_species,
|
||
ortholog_identity=run_identities[0] if run_identities else None,
|
||
analysis_route=analysis_route,
|
||
member_set_id=member_set_id,
|
||
scope_fields=scope_fields,
|
||
blast_evalue=blast_evalue,
|
||
cpu=cpu,
|
||
)
|
||
except Exception as exc:
|
||
traceback.print_exc()
|
||
intermediate = ", ".join(str(record.pairs_src.name) for record in run_records)
|
||
return (
|
||
f"❌ Ortholog post-processing failed after BLAST/intermediate output ({intermediate}): {exc}. Log: {log_file}",
|
||
manifest,
|
||
)
|
||
|
||
return f"✅ Ortholog analysis complete. Found {pair_count} pairs. Log: {log_file}", ortholog_manifest
|
||
|
||
# 生成可复现性文件（如果管理器已初始化）
|
||
if stage == "ppi":
|
||
# PPI蛋白互作分析 (技能化版本)
|
||
# 输入：基因家族ID列表（来自identify阶段）
|
||
ppi_scope_path = None
|
||
ppi_scope_data = None
|
||
ppi_reference_species = ""
|
||
ppi_selected_targets = []
|
||
requested_ppi_species = str(ppi_species or "").strip().lower()
|
||
explicit_ppi_species = bool(requested_ppi_species and requested_ppi_species != "auto")
|
||
ppi_owner_intent = str(dependency_owner or "").strip().lower() == "ppi"
|
||
ppi_owner_taxon_id = str(ppi_reference_taxon_id or "").strip()
|
||
ppi_owner_auto_fallback = False
|
||
mapping_file = None
|
||
|
||
def fail_ppi_execution(message: str, diagnostic: Optional[Dict[str, Any]] = None) -> Tuple[str, Path]:
|
||
current_targets = list(ppi_selected_targets)
|
||
if not current_targets:
|
||
requested_targets = target_species if isinstance(target_species, list) else [target_species]
|
||
current_targets = canonicalize_species_codes_for_job(
|
||
in_dir, [str(value) for value in requested_targets if str(value or "").strip()]
|
||
)
|
||
if not current_targets:
|
||
current_targets = [prefix.split(".", 1)[0]] if prefix else []
|
||
current_target = current_targets[0] if len(current_targets) == 1 else ""
|
||
current_identity = resolve_ppi_target_identity(in_dir, current_target)
|
||
try:
|
||
current_resolution = resolve_ppi_string_species(
|
||
load_string_snapshot(STRING_SPECIES_SNAPSHOT_ROOT),
|
||
current_target,
|
||
target_scientific_name=str(current_identity.get("scientific_name") or ""),
|
||
ppi_target_tree_taxon_id=str(current_identity.get("tree_taxon_id") or ""),
|
||
)
|
||
except StringSnapshotError as exc:
|
||
current_resolution = {
|
||
"selection_mode": "resource_error",
|
||
"string_version": "",
|
||
"snapshot_id": "",
|
||
"issue_code": exc.code,
|
||
"requires_user_confirmation": False,
|
||
}
|
||
current_ppi_species = str(ppi_species or requested_ppi_species or "")
|
||
current_taxon_id = (
|
||
current_ppi_species
|
||
if current_ppi_species.isdigit()
|
||
else string_taxon_id(current_ppi_species, "")
|
||
)
|
||
current_mode = str(
|
||
ppi_reference_selection_mode
|
||
or current_resolution.get("selection_mode")
|
||
or "unresolved"
|
||
)
|
||
current_normalization = str(normalization_species_label or "").strip()
|
||
if not current_normalization:
|
||
current_normalization = current_target or current_ppi_species
|
||
provenance = build_ppi_provenance(
|
||
raw_target_species=current_targets,
|
||
target_identity=current_identity,
|
||
species_resolution=current_resolution,
|
||
selected_string_taxon_id=current_taxon_id,
|
||
selection_mode=current_mode,
|
||
user_confirmed=ppi_reference_user_confirmed,
|
||
mapping_reference_species=ppi_reference_species or ppi_reference_storage_key,
|
||
mapping_path=str(mapping_file) if mapping_file else "",
|
||
normalization_species_label=current_normalization,
|
||
snapshot_root=STRING_SPECIES_SNAPSHOT_ROOT,
|
||
status="failed",
|
||
)
|
||
failure = dict(diagnostic or {})
|
||
failure.update({
|
||
"manifest_version": str(failure.get("manifest_version") or "1.0"),
|
||
"stage": "ppi",
|
||
"status": "failed",
|
||
"job_id": job.job_id,
|
||
"workspace": str(ws),
|
||
"workspace_root": str(ws),
|
||
"error": message,
|
||
"artifacts": dict(failure.get("artifacts") or {}),
|
||
})
|
||
failed_manifest = write_ppi_failed_manifest(final_dir, prefix, failure)
|
||
apply_ppi_provenance(failed_manifest, provenance, status="failed", error=message)
|
||
return message, failed_manifest
|
||
|
||
try:
|
||
if explicit_ppi_species:
|
||
ppi_species = resolve_ppi_execution_species(requested_ppi_species)
|
||
elif ppi_owner_intent:
|
||
if ppi_owner_taxon_id.isdigit():
|
||
ppi_species = ppi_owner_taxon_id
|
||
if not ppi_reference_user_confirmed:
|
||
return fail_ppi_execution(
|
||
"❌ PPI owner STRING taxon must be confirmed before execution."
|
||
)
|
||
if not ppi_owner_taxon_id.isdigit():
|
||
return fail_ppi_execution(
|
||
"❌ PPI owner STRING taxon must be a numeric taxonomy ID before execution."
|
||
)
|
||
ppi_owner_auto_fallback = True
|
||
else:
|
||
ppi_species = ""
|
||
except (StringSnapshotError, ValueError) as exc:
|
||
return fail_ppi_execution(f"❌ PPI explicit unsupported STRING species: {requested_ppi_species}; {exc}")
|
||
if scope_file:
|
||
ppi_scope_path = resolve_scope_path(scope_file, in_dir, ws)
|
||
if not ppi_scope_path.exists():
|
||
return fail_ppi_execution(f"❌ PPI scope 文件不存在: {scope_file}")
|
||
try:
|
||
ppi_scope_data = read_scope(ppi_scope_path)
|
||
except Exception as exc:
|
||
return fail_ppi_execution(f"❌ PPI scope 文件无法读取: {scope_file}; {exc}")
|
||
ppi_selected_targets = canonicalize_species_codes_for_job(
|
||
in_dir,
|
||
[str(x) for x in ppi_scope_data.get("target_species", []) if str(x)],
|
||
)
|
||
references = canonicalize_species_codes_for_job(
|
||
in_dir,
|
||
[str(x) for x in ppi_scope_data.get("reference_species", []) if str(x)],
|
||
)
|
||
ppi_reference_species = references[0] if references else ""
|
||
target_string_code = species_to_string_code(ppi_selected_targets[0], "") if len(ppi_selected_targets) == 1 else ""
|
||
target_string_taxon_id = string_taxon_id(target_string_code, "") if target_string_code else ""
|
||
if len(ppi_selected_targets) == 1:
|
||
target_identity = resolve_ppi_target_identity(in_dir, ppi_selected_targets[0])
|
||
if target_identity.get("status") == "resolved":
|
||
try:
|
||
target_resolution = resolve_ppi_string_species(
|
||
load_string_snapshot(STRING_SPECIES_SNAPSHOT_ROOT),
|
||
ppi_selected_targets[0],
|
||
target_scientific_name=str(target_identity.get("scientific_name") or ""),
|
||
ppi_target_tree_taxon_id=str(target_identity.get("tree_taxon_id") or ""),
|
||
)
|
||
if target_resolution.get("selection_mode") == "exact_string":
|
||
target_string_taxon_id = str(target_resolution.get("selected_string_taxon_id") or "")
|
||
except StringSnapshotError:
|
||
target_string_taxon_id = ""
|
||
if not explicit_ppi_species and not ppi_owner_auto_fallback:
|
||
if target_string_code:
|
||
ppi_species = target_string_code
|
||
elif ppi_reference_species:
|
||
ppi_species = species_to_string_code(ppi_reference_species, ppi_species)
|
||
|
||
scope_id = str(ppi_scope_data.get("scope_id") or "")
|
||
selected_execution_taxon_id = string_taxon_id(ppi_species, "") if ppi_species else ""
|
||
if str(ppi_species or "").isdigit():
|
||
selected_execution_taxon_id = str(ppi_species)
|
||
use_target_string_directly = bool(
|
||
len(ppi_selected_targets) == 1
|
||
and target_string_taxon_id
|
||
and selected_execution_taxon_id == target_string_taxon_id
|
||
)
|
||
mapping_candidates = []
|
||
if not use_target_string_directly:
|
||
if scope_id:
|
||
mapping_candidates.extend([
|
||
final_dir / f"{prefix}.{sanitize_token(scope_id)}.orthologs.txt",
|
||
final_dir / f"{prefix}.{sanitize_token(scope_id)}.ortholog_pairs.txt",
|
||
])
|
||
mapping_candidates.extend([
|
||
final_dir / f"{prefix}.orthologs.txt",
|
||
final_dir / f"{prefix}.ortholog_pairs.txt",
|
||
])
|
||
mapping_file = next((p for p in mapping_candidates if p.exists() and p.stat().st_size > 0), None)
|
||
if mapping_file:
|
||
gene_id_file = ws / "ppi" / "analysis" / f"{prefix}.{sanitize_token(scope_id) if scope_id else 'scope'}.ppi_source_ids.txt"
|
||
mapped_source_count = build_ppi_gene_ids_from_ortholog_mapping(mapping_file, gene_id_file, ppi_selected_targets)
|
||
if mapped_source_count == 0:
|
||
return fail_ppi_execution(
|
||
f"❌ PPI scope mapping file has no source IDs for target species {ppi_selected_targets}: {mapping_file}"
|
||
)
|
||
if not ppi_species:
|
||
ppi_species = infer_string_species_from_ortholog_mapping(mapping_file, "")
|
||
if not ppi_species:
|
||
diagnostic = build_ppi_species_diagnostic(
|
||
requested_ppi_species=requested_ppi_species or "auto",
|
||
target_species=",".join(ppi_selected_targets),
|
||
target_string_supported=False,
|
||
reference_species=ppi_reference_species,
|
||
reference_data_source="",
|
||
ortholog_mapping_used=True,
|
||
fallback_level="invalid_mapping",
|
||
fallback_reason="ortholog_mapping_target_species_not_supported_by_string",
|
||
prefix=prefix,
|
||
)
|
||
return fail_ppi_execution(
|
||
"❌ PPI 无法从 Ortholog 映射推断 STRING 参考物种；"
|
||
"请检查映射文件目标物种列，或显式选择受支持的参考物种后重跑 Ortholog。",
|
||
diagnostic,
|
||
)
|
||
elif len(ppi_selected_targets) == 1:
|
||
gene_id_file = resolve_single_target_gene_id_file(ws, in_dir, final_dir, ppi_selected_targets[0], prefix)
|
||
if not gene_id_file:
|
||
return fail_ppi_execution(
|
||
f"❌ PPI stage requires {ppi_selected_targets[0]} member IDs or ortholog mapping. Please run Identify/Ortholog first."
|
||
)
|
||
if not use_target_string_directly and not target_string_code and not ppi_reference_species:
|
||
family = str(ppi_scope_data.get("family") or (prefix.split(".", 1)[1] if "." in prefix else prefix))
|
||
reference_data_source = ""
|
||
reference_species = references[0] if references else ""
|
||
if reference_species:
|
||
ref_pep = resolve_species_sequence_file(
|
||
ws,
|
||
final_dir,
|
||
in_dir,
|
||
ppi_scope_data,
|
||
reference_species,
|
||
family,
|
||
"pep",
|
||
family_only=False,
|
||
)
|
||
reference_data_source = str(ref_pep) if ref_pep else ""
|
||
diagnostic = build_ppi_species_diagnostic(
|
||
requested_ppi_species=requested_ppi_species or "auto",
|
||
target_species=ppi_selected_targets[0],
|
||
target_string_supported=False,
|
||
reference_species=reference_species,
|
||
reference_data_source=reference_data_source,
|
||
ortholog_mapping_used=False,
|
||
fallback_level="near_reference" if reference_data_source else "reference_data_missing",
|
||
fallback_reason="target_species_not_supported_by_string",
|
||
prefix=prefix,
|
||
)
|
||
return fail_ppi_execution(
|
||
f"❌ PPI requires Ortholog before STRING fallback for target species {ppi_selected_targets[0]}. "
|
||
"See the failed PPI manifest for details.",
|
||
diagnostic,
|
||
)
|
||
if ppi_species and not use_target_string_directly and target_string_code != ppi_species:
|
||
family = str(ppi_scope_data.get("family") or (prefix.split(".", 1)[1] if "." in prefix else prefix))
|
||
ref_pep = resolve_species_sequence_file(
|
||
ws,
|
||
final_dir,
|
||
in_dir,
|
||
ppi_scope_data,
|
||
ppi_reference_species or ppi_species,
|
||
family,
|
||
"pep",
|
||
family_only=False,
|
||
)
|
||
diagnostic = build_ppi_species_diagnostic(
|
||
requested_ppi_species=requested_ppi_species or ppi_species or "auto",
|
||
target_species=ppi_selected_targets[0],
|
||
target_string_supported=bool(target_string_code),
|
||
reference_species=ppi_reference_species or ppi_species,
|
||
reference_data_source=str(ref_pep) if ref_pep else "",
|
||
ortholog_mapping_used=False,
|
||
fallback_level="explicit_reference",
|
||
fallback_reason="ortholog_mapping_required_for_reference_string_species",
|
||
prefix=prefix,
|
||
)
|
||
return fail_ppi_execution(
|
||
f"❌ PPI 使用 {ppi_species} STRING 库前需要先运行 Ortholog，将 {ppi_selected_targets[0]} 成员 ID 映射为参考物种 ID；"
|
||
f"当前只有本地成员文件 {gene_id_file}，不能直接提交到 {ppi_species} STRING。",
|
||
diagnostic,
|
||
)
|
||
ppi_species = ppi_species or target_string_code
|
||
else:
|
||
return fail_ppi_execution("❌ PPI 多 target scope 需要先运行 Ortholog，生成 target→reference 映射后再查询 STRING。")
|
||
else:
|
||
try:
|
||
ppi_member_resolution = resolve_effective_members(
|
||
workspace=ws,
|
||
final_dir=final_dir,
|
||
input_dir=in_dir,
|
||
prefix=prefix,
|
||
phase="auto",
|
||
require="id",
|
||
)
|
||
gene_id_file = ppi_member_resolution.path_id
|
||
if ppi_member_resolution.warnings:
|
||
print(f"[WARN] {'; '.join(ppi_member_resolution.warnings)}")
|
||
print(
|
||
f"[INFO] PPI member source: {ppi_member_resolution.source_kind} "
|
||
f"({ppi_member_resolution.member_count} IDs)"
|
||
)
|
||
except MemberResolutionError as exc:
|
||
return fail_ppi_execution(f"❌ PPI stage requires an effective member ID file: {exc}")
|
||
if not gene_id_file or not gene_id_file.exists():
|
||
return fail_ppi_execution(
|
||
"❌ PPI stage requires an effective member ID file. Please run Identify or confirm subfamily first."
|
||
)
|
||
if not ppi_species:
|
||
prefix_species = prefix.split(".", 1)[0] if "." in prefix else ""
|
||
ppi_species = species_to_string_code(prefix_species, "")
|
||
if not ppi_species:
|
||
diagnostic = build_ppi_species_diagnostic(
|
||
requested_ppi_species=requested_ppi_species or "auto",
|
||
target_species=prefix_species,
|
||
target_string_supported=False,
|
||
reference_species="",
|
||
reference_data_source="",
|
||
ortholog_mapping_used=False,
|
||
fallback_level="reference_data_missing",
|
||
fallback_reason="target_species_not_supported_by_string",
|
||
prefix=prefix,
|
||
)
|
||
return fail_ppi_execution(
|
||
f"❌ PPI target species {prefix_species or '<unknown>'} is not directly supported by STRING; "
|
||
"provide reference protein data and run Ortholog before fallback.",
|
||
diagnostic,
|
||
)
|
||
elif explicit_ppi_species:
|
||
prefix_species = prefix.split(".", 1)[0] if "." in prefix else ""
|
||
target_string_code = species_to_string_code(prefix_species, "")
|
||
if target_string_code and target_string_code != ppi_species and not mapping_file:
|
||
diagnostic = build_ppi_species_diagnostic(
|
||
requested_ppi_species=requested_ppi_species or ppi_species,
|
||
target_species=prefix_species,
|
||
target_string_supported=True,
|
||
reference_species=ppi_species,
|
||
reference_data_source="",
|
||
ortholog_mapping_used=False,
|
||
fallback_level="explicit_reference",
|
||
fallback_reason="ortholog_mapping_required_for_reference_string_species",
|
||
prefix=prefix,
|
||
)
|
||
return fail_ppi_execution(
|
||
f"❌ PPI 使用 {ppi_species} STRING 库前需要 Ortholog 映射；"
|
||
f"不能把 {prefix_species} 成员 ID 直接提交到 {ppi_species} STRING。",
|
||
diagnostic,
|
||
)
|
||
if not ppi_species:
|
||
return fail_ppi_execution(
|
||
"❌ PPI 无法从 scope/prefix 推断 STRING 物种代码；"
|
||
"请显式设置 ppiSpecies，或先提供 Ortholog 映射到支持的参考物种。"
|
||
)
|
||
|
||
# 检查技能脚本
|
||
skill_script = Path("skills/ppi/run.py")
|
||
if not skill_script.exists():
|
||
return fail_ppi_execution(
|
||
"❌ Authoritative PPI skill not found: skills/ppi/run.py; "
|
||
"legacy fallback disabled"
|
||
)
|
||
script = skill_script
|
||
|
||
ppi_dir = ws / "ppi"
|
||
ppi_dir.mkdir(parents=True, exist_ok=True)
|
||
final_dir.mkdir(exist_ok=True)
|
||
|
||
print(f"[PPI] Gene ID file: {gene_id_file.name}")
|
||
print(f"[PPI] Species: {ppi_species}")
|
||
effective_normalization_label = str(normalization_species_label or "").strip()
|
||
if not effective_normalization_label:
|
||
if mapping_file:
|
||
effective_normalization_label = (
|
||
ppi_reference_storage_key
|
||
if ppi_owner_auto_fallback and ppi_reference_storage_key
|
||
else ppi_reference_species or ppi_species
|
||
)
|
||
elif len(ppi_selected_targets) == 1:
|
||
effective_normalization_label = target_string_code or ppi_species
|
||
else:
|
||
effective_normalization_label = ppi_species
|
||
print(f"[PPI] Normalization species label: {effective_normalization_label}")
|
||
print(f"[PPI] Score threshold: {ppi_score}")
|
||
print("[PPI] Using authoritative skill version")
|
||
|
||
raw_ppi_targets = ppi_selected_targets or ([prefix.split(".", 1)[0]] if prefix else [])
|
||
raw_ppi_target = raw_ppi_targets[0] if len(raw_ppi_targets) == 1 else ""
|
||
submitted_target_identity = resolve_ppi_target_identity(in_dir, raw_ppi_target)
|
||
try:
|
||
submitted_species_resolution = resolve_ppi_string_species(
|
||
load_string_snapshot(STRING_SPECIES_SNAPSHOT_ROOT),
|
||
raw_ppi_target,
|
||
target_scientific_name=str(submitted_target_identity.get("scientific_name") or ""),
|
||
ppi_target_tree_taxon_id=str(submitted_target_identity.get("tree_taxon_id") or ""),
|
||
)
|
||
except StringSnapshotError as exc:
|
||
submitted_species_resolution = {
|
||
"selection_mode": "resource_error",
|
||
"string_version": "",
|
||
"snapshot_id": "",
|
||
"issue_code": exc.code,
|
||
"requires_user_confirmation": False,
|
||
}
|
||
submitted_taxon_id = string_taxon_id(ppi_species, "") if not str(ppi_species).isdigit() else str(ppi_species)
|
||
submitted_selection_mode = str(
|
||
ppi_reference_selection_mode
|
||
or submitted_species_resolution.get("selection_mode")
|
||
or "unresolved"
|
||
)
|
||
ppi_provenance = build_ppi_provenance(
|
||
raw_target_species=raw_ppi_targets,
|
||
target_identity=submitted_target_identity,
|
||
species_resolution=submitted_species_resolution,
|
||
selected_string_taxon_id=submitted_taxon_id,
|
||
selection_mode=submitted_selection_mode,
|
||
user_confirmed=ppi_reference_user_confirmed,
|
||
mapping_reference_species=ppi_reference_species or ppi_reference_storage_key,
|
||
mapping_path=str(mapping_file) if mapping_file else "",
|
||
normalization_species_label=effective_normalization_label,
|
||
snapshot_root=STRING_SPECIES_SNAPSHOT_ROOT,
|
||
status="submitted",
|
||
)
|
||
ppi_provenance_path = ppi_dir / f"{prefix}.ppi_provenance.json"
|
||
ppi_provenance_path.write_text(
|
||
json.dumps(ppi_provenance, ensure_ascii=False, indent=2),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
# Do not opportunistically pick stale ortholog/KaKs/synteny files here.
|
||
# PPI fallback must use scope-derived or explicit mapping evidence.
|
||
if mapping_file:
|
||
print(f"[PPI] Found ortholog mapping file: {mapping_file}")
|
||
|
||
# STRING TaxID is the remote query identity; local reference files use
|
||
# the stable storage key selected by the PPI resolver.
|
||
ppi_storage_key = str(ppi_reference_storage_key or "").strip()
|
||
if not ppi_storage_key and str(ppi_reference_species).startswith("string_taxon_"):
|
||
ppi_storage_key = str(ppi_reference_species)
|
||
if not ppi_storage_key:
|
||
ppi_storage_key = str(ppi_species)
|
||
id_map_file = Path(job.input_dir) / ppi_storage_key / f"{ppi_storage_key}.pep_cds_idmap.tsv"
|
||
if mapping_file and not id_map_file.is_file():
|
||
return fail_ppi_execution(
|
||
f"❌ PPI projected analysis requires species-matched ID map: {id_map_file}"
|
||
)
|
||
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--gene-ids", str(gene_id_file),
|
||
"--species", ppi_species,
|
||
"--normalization-species-label", effective_normalization_label,
|
||
"--prefix", prefix,
|
||
"--output", str(ppi_dir),
|
||
"--score", str(ppi_score),
|
||
"--provenance-json", str(ppi_provenance_path),
|
||
]
|
||
if mapping_file:
|
||
cmd.extend(["--mapping", str(mapping_file)])
|
||
if id_map_file.is_file():
|
||
cmd.extend(["--id-map", str(id_map_file)])
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
# 技能化版本：复制产物到 final/
|
||
if code == 0:
|
||
print(f"[PPI] Copying artifacts to final/...")
|
||
|
||
# 技能实际生成的文件 -> final/目标文件名
|
||
ppi_file_mappings = {
|
||
f"{prefix}_network.tsv": f"{prefix}.ppi_network.tsv",
|
||
f"{prefix}_report.md": f"{prefix}.ppi_report.md",
|
||
f"{prefix}_url.txt": f"{prefix}.ppi_url.txt",
|
||
f"{prefix}_id_resolution.tsv": f"{prefix}.ppi_id_resolution.tsv",
|
||
f"{prefix}.manifest.json": f"{prefix}.ppi.manifest.json",
|
||
}
|
||
|
||
copied = 0
|
||
for src_name, dst_name in ppi_file_mappings.items():
|
||
src = ppi_dir / src_name
|
||
if src.exists():
|
||
dst = final_dir / dst_name
|
||
shutil.copy2(src, dst)
|
||
print(f"[PPI] Copied: {src.name} -> {dst.name}")
|
||
copied += 1
|
||
|
||
print(f"[PPI] Copied {copied} files to final/")
|
||
sync_ppi_manifest_final_artifacts(final_dir / f"{prefix}.ppi.manifest.json", final_dir, prefix)
|
||
ppi_manifest_after_copy = final_dir / f"{prefix}.ppi.manifest.json"
|
||
if ppi_manifest_after_copy.exists():
|
||
apply_ppi_provenance(ppi_manifest_after_copy, ppi_provenance, status="success")
|
||
|
||
if code == 0:
|
||
# 读取 STRING URL
|
||
url_file = final_dir / f"{prefix}.ppi_url.txt"
|
||
string_url = None
|
||
if url_file.exists():
|
||
with open(url_file) as f:
|
||
for line in f:
|
||
if line.startswith("View online:"):
|
||
string_url = line.replace("View online:", "").strip()
|
||
break
|
||
|
||
string_image_file = None
|
||
if string_url:
|
||
string_image_path = final_dir / f"{prefix}.ppi_network.string.png"
|
||
string_svg_path = string_image_path.with_suffix(".svg")
|
||
for stale_path in (string_image_path, string_svg_path):
|
||
if not stale_path.exists():
|
||
continue
|
||
try:
|
||
stale_path.unlink()
|
||
except Exception as exc:
|
||
print(f"[PPI] Could not remove stale STRING image: {exc}")
|
||
string_image_file = download_string_network_image(
|
||
string_url,
|
||
string_image_path,
|
||
)
|
||
if string_image_file:
|
||
print(f"[PPI] STRING network image saved: {string_image_file.name}")
|
||
|
||
alphafold2_links_file = create_alphafold2_links(
|
||
id_resolution_file=final_dir / f"{prefix}.ppi_id_resolution.tsv",
|
||
network_file=final_dir / f"{prefix}.ppi_network.tsv",
|
||
output_file=final_dir / f"{prefix}.ppi_alphafold2_links.tsv",
|
||
)
|
||
if alphafold2_links_file:
|
||
print(f"[PPI] AlphaFold DB/Fold2 link table saved: {alphafold2_links_file.name}")
|
||
|
||
# 统计输出文件
|
||
ppi_files = list(final_dir.glob("*.ppi_network.tsv"))
|
||
file_count = len(ppi_files)
|
||
|
||
ppi_manifest = final_dir / f"{prefix}.ppi.manifest.json"
|
||
if not ppi_manifest.exists():
|
||
ppi_artifacts = {}
|
||
network_file = final_dir / f"{prefix}.ppi_network.tsv"
|
||
if network_file.exists():
|
||
ppi_artifacts["ppi_network"] = {
|
||
"artifact": "ppi_network",
|
||
"path": f"final/{network_file.name}",
|
||
"full_path": str(network_file.absolute()),
|
||
"type": "table",
|
||
"format": "tsv",
|
||
"size": network_file.stat().st_size,
|
||
"sha256": _sha256_file(network_file),
|
||
}
|
||
if url_file.exists():
|
||
ppi_artifacts["string_url"] = {
|
||
"artifact": "string_url",
|
||
"path": f"final/{url_file.name}",
|
||
"full_path": str(url_file.absolute()),
|
||
"type": "link",
|
||
"format": "txt",
|
||
"size": url_file.stat().st_size,
|
||
"sha256": _sha256_file(url_file),
|
||
}
|
||
if alphafold2_links_file and alphafold2_links_file.exists():
|
||
ppi_artifacts["alphafold2_links"] = {
|
||
"artifact": "alphafold2_links",
|
||
"path": f"final/{alphafold2_links_file.name}",
|
||
"full_path": str(alphafold2_links_file.absolute()),
|
||
"type": "table",
|
||
"format": "tsv",
|
||
"size": alphafold2_links_file.stat().st_size,
|
||
"sha256": _sha256_file(alphafold2_links_file),
|
||
"source": "AlphaFold DB links derived from STRING PPI outputs",
|
||
"structure_type": "AlphaFold DB/Fold2 single protein",
|
||
}
|
||
|
||
if ppi_artifacts:
|
||
ppi_manifest_data = {
|
||
"version": "1.0",
|
||
"stage": "ppi",
|
||
"status": "success",
|
||
"prefix": prefix,
|
||
"job_id": job.job_id,
|
||
"workspace": str(ws),
|
||
"workspace_root": str(ws),
|
||
"created_at": datetime.now().isoformat(),
|
||
"inputs": [
|
||
{"type": "gene_id_file", "path": str(gene_id_file)},
|
||
{"type": "ortholog_mapping", "path": str(mapping_file) if mapping_file else ""},
|
||
],
|
||
"parameters": {
|
||
"prefix": prefix,
|
||
"species": ppi_species,
|
||
"string_taxon_id": string_taxon_id(ppi_species, "") if not str(ppi_species).isdigit() else ppi_species,
|
||
"normalization_species_label": effective_normalization_label,
|
||
"score": ppi_score,
|
||
},
|
||
"metrics": {
|
||
"ppi_network_files": file_count,
|
||
},
|
||
"artifacts": ppi_artifacts,
|
||
}
|
||
with ppi_manifest.open("w", encoding="utf-8") as handle:
|
||
json.dump(ppi_manifest_data, handle, ensure_ascii=False, indent=2)
|
||
|
||
if string_image_file and ppi_manifest.exists():
|
||
try:
|
||
with ppi_manifest.open("r", encoding="utf-8") as handle:
|
||
ppi_manifest_data = json.load(handle)
|
||
string_image_format = string_image_file.suffix.lower().lstrip(".") or "png"
|
||
ppi_manifest_data.setdefault("artifacts", {})["string_network_image"] = {
|
||
"artifact": "string_network_image",
|
||
"path": f"final/{string_image_file.name}",
|
||
"full_path": str(string_image_file.absolute()),
|
||
"type": string_image_format,
|
||
"format": string_image_format,
|
||
"size": string_image_file.stat().st_size,
|
||
"sha256": _sha256_file(string_image_file),
|
||
"source": "STRING network image API",
|
||
}
|
||
with ppi_manifest.open("w", encoding="utf-8") as handle:
|
||
json.dump(ppi_manifest_data, handle, ensure_ascii=False, indent=2)
|
||
except Exception as exc:
|
||
print(f"[PPI] Could not record STRING image in manifest: {exc}")
|
||
if alphafold2_links_file and ppi_manifest.exists():
|
||
try:
|
||
with ppi_manifest.open("r", encoding="utf-8") as handle:
|
||
ppi_manifest_data = json.load(handle)
|
||
ppi_manifest_data.setdefault("artifacts", {})["alphafold2_links"] = {
|
||
"artifact": "alphafold2_links",
|
||
"path": f"final/{alphafold2_links_file.name}",
|
||
"full_path": str(alphafold2_links_file.absolute()),
|
||
"type": "table",
|
||
"format": "tsv",
|
||
"size": alphafold2_links_file.stat().st_size,
|
||
"sha256": _sha256_file(alphafold2_links_file),
|
||
"source": "AlphaFold DB links derived from STRING PPI outputs",
|
||
"structure_type": "AlphaFold DB/Fold2 single protein",
|
||
}
|
||
with ppi_manifest.open("w", encoding="utf-8") as handle:
|
||
json.dump(ppi_manifest_data, handle, ensure_ascii=False, indent=2)
|
||
except Exception as exc:
|
||
print(f"[PPI] Could not record AlphaFold DB/Fold2 links in manifest: {exc}")
|
||
if ppi_scope_path and ppi_scope_data:
|
||
annotate_stage_manifest_with_scope(
|
||
ppi_manifest,
|
||
prefix,
|
||
"ppi",
|
||
ppi_scope_path,
|
||
ppi_scope_data,
|
||
extra_parameters={
|
||
"analysis_route": analysis_route,
|
||
"member_set_id": member_set_id,
|
||
"gene_id_file": str(gene_id_file),
|
||
"ortholog_mapping": str(mapping_file) if mapping_file else "",
|
||
"string_species": ppi_species,
|
||
"string_taxon_id": string_taxon_id(ppi_species, "") if not str(ppi_species).isdigit() else ppi_species,
|
||
"normalization_species_label": effective_normalization_label,
|
||
"reference_species": ppi_reference_species,
|
||
},
|
||
)
|
||
ppi_scope_id = str(ppi_scope_data.get("scope_id") or "")
|
||
if ppi_scope_id:
|
||
scoped_ppi_manifest = final_dir / f"{prefix}.{sanitize_token(ppi_scope_id)}.ppi.manifest.json"
|
||
if scoped_ppi_manifest.exists():
|
||
ppi_manifest = scoped_ppi_manifest
|
||
|
||
# 返回消息包含 STRING 链接
|
||
if string_url:
|
||
return f"✅ PPI analysis completed. {file_count} files in final/. <a href='{string_url}' target='_blank'>View STRING Network</a>", ppi_manifest
|
||
else:
|
||
return f"✅ PPI analysis completed. {file_count} files in final/", ppi_manifest
|
||
else:
|
||
ppi_manifest = final_dir / f"{prefix}.ppi.manifest.json"
|
||
if not ppi_manifest.exists():
|
||
write_ppi_failed_manifest(final_dir, prefix, {
|
||
"manifest_version": "1.0",
|
||
"stage": "ppi",
|
||
"status": "failed",
|
||
"job_id": job.job_id,
|
||
"workspace": str(ws),
|
||
"workspace_root": str(ws),
|
||
"parameters": {
|
||
"string_taxon_id": submitted_taxon_id,
|
||
"normalization_species_label": effective_normalization_label,
|
||
},
|
||
"artifacts": {},
|
||
})
|
||
apply_ppi_provenance(
|
||
ppi_manifest,
|
||
ppi_provenance,
|
||
status="failed",
|
||
error=f"PPI analysis failed (exit code: {code}). Log: {log_file}",
|
||
)
|
||
return f"❌ PPI analysis failed (exit code: {code}). Log: {log_file}", ppi_manifest
|
||
|
||
if stage == "kaks":
|
||
# Ka/Ks分析
|
||
# 输入：需要两个物种的CDS和蛋白序列，以及同源基因对列表
|
||
kaks_dir = ws / "kaks"
|
||
kaks_dir.mkdir(parents=True, exist_ok=True)
|
||
final_dir.mkdir(exist_ok=True)
|
||
|
||
if scope_file:
|
||
kaks_scope_path = resolve_scope_path(scope_file, in_dir, ws)
|
||
if not kaks_scope_path.exists():
|
||
return f"❌ KaKs scope 文件不存在: {scope_file}", manifest
|
||
try:
|
||
kaks_scope_data = read_scope(kaks_scope_path)
|
||
except Exception as exc:
|
||
return f"❌ KaKs scope 文件无法读取: {scope_file}; {exc}", manifest
|
||
family = str(kaks_scope_data.get("family") or (prefix.split(".", 1)[1] if "." in prefix else prefix))
|
||
target_species_list = [str(x) for x in kaks_scope_data.get("target_species", []) if str(x)]
|
||
reference_species_list = [str(x) for x in kaks_scope_data.get("reference_species", []) if str(x)]
|
||
if not target_species_list:
|
||
return "❌ KaKs scope 中没有 target_species，无法确定物种1。", manifest
|
||
if not reference_species_list:
|
||
return "❌ KaKs scope 中没有 reference_species，无法确定物种2/reference。", manifest
|
||
ref_species = reference_species_list[0]
|
||
scope_id = str(kaks_scope_data.get("scope_id") or "")
|
||
candidate_prefixes = [prefix]
|
||
for target_species_name in target_species_list:
|
||
if family:
|
||
candidate_prefixes.extend([
|
||
f"{target_species_name}.{family}",
|
||
f"Target.{target_species_name}.{family}",
|
||
])
|
||
candidate_prefixes = list(dict.fromkeys(str(x) for x in candidate_prefixes if str(x)))
|
||
mapping_candidates = []
|
||
if scope_id:
|
||
for candidate_prefix in candidate_prefixes:
|
||
mapping_candidates.extend([
|
||
final_dir / f"{candidate_prefix}.{sanitize_token(scope_id)}.orthologs.txt",
|
||
final_dir / f"{candidate_prefix}.{sanitize_token(scope_id)}.ortholog_pairs.txt",
|
||
])
|
||
for candidate_prefix in candidate_prefixes:
|
||
mapping_candidates.extend([
|
||
final_dir / f"{candidate_prefix}.orthologs.txt",
|
||
final_dir / f"{candidate_prefix}.ortholog_pairs.txt",
|
||
])
|
||
mapping_candidates.append(in_dir / "ortholog_pairs.txt")
|
||
scoped_pairs_source = next((p for p in mapping_candidates if p.exists() and p.stat().st_size > 0), None)
|
||
if not scoped_pairs_source:
|
||
return "❌ KaKs scope 模式需要 Ortholog 映射文件。请先运行 Ortholog 阶段。", manifest
|
||
|
||
ref_cds = resolve_species_sequence_file(ws, final_dir, in_dir, kaks_scope_data, ref_species, family, "cds")
|
||
ref_pep = resolve_species_sequence_file(ws, final_dir, in_dir, kaks_scope_data, ref_species, family, "pep", family_only=False)
|
||
if not ref_cds:
|
||
return f"❌ KaKs 缺少参考物种 {ref_species} CDS 文件，例如 {ref_species}.cds.fasta 或全基因组 CDS。", manifest
|
||
if not ref_pep:
|
||
return f"❌ KaKs 缺少参考物种 {ref_species} 全蛋白文件；Ka/Ks 要求 reference CDS 和 PEP 来自同一套注释。", manifest
|
||
|
||
run_records = []
|
||
for target_species_name in target_species_list:
|
||
target_cds = resolve_species_sequence_file(ws, final_dir, in_dir, kaks_scope_data, target_species_name, family, "cds")
|
||
target_pep = resolve_species_sequence_file(ws, final_dir, in_dir, kaks_scope_data, target_species_name, family, "pep", family_only=True)
|
||
target_gene_list = resolve_single_target_gene_id_file(ws, in_dir, final_dir, target_species_name, prefix)
|
||
if not target_cds:
|
||
return f"❌ KaKs 缺少目标物种 {target_species_name} CDS 文件，例如 {target_species_name}.cds.fasta。", manifest
|
||
if not target_pep:
|
||
return f"❌ KaKs 缺少目标物种 {target_species_name} 家族蛋白文件。请先运行 Identify。", manifest
|
||
pair_file = kaks_dir / "analysis" / f"{target_species_name}_to_{ref_species}.ortholog_pairs.tsv"
|
||
pair_count = write_species_filtered_pairs(scoped_pairs_source, pair_file, target_species_name, ref_species)
|
||
if pair_count == 0:
|
||
print(f"[KaKs] No pairs for {target_species_name}->{ref_species}, skipping.")
|
||
continue
|
||
run_prefix = f"{prefix}.{target_species_name}_to_{ref_species}"
|
||
kaks_cmd = [
|
||
sys.executable,
|
||
"apps/cli/run_kaks_v2.py",
|
||
"-o", str(kaks_dir),
|
||
"--prefix-name", run_prefix,
|
||
"--cds1", str(target_cds),
|
||
"--cds2", str(ref_cds),
|
||
"--pep1", str(target_pep),
|
||
"--pep2", str(ref_pep),
|
||
"--pairs", str(pair_file),
|
||
"--neutral-rate", str(kaks_neutral_rate),
|
||
]
|
||
if target_gene_list:
|
||
kaks_cmd.extend(["--gene-list1", str(target_gene_list)])
|
||
print(f"[CMD] {' '.join(kaks_cmd)}")
|
||
code = run_cmd_logged(kaks_cmd, log_file, cwd=Path("."))
|
||
if code != 0:
|
||
return f"❌ Ka/Ks analysis failed for {target_species_name}->{ref_species} (exit code: {code}). Log: {log_file}", manifest
|
||
run_records.append({
|
||
"target_species": target_species_name,
|
||
"reference_species": ref_species,
|
||
"prefix": run_prefix,
|
||
"pairs": str(pair_file.resolve()),
|
||
"pair_count": pair_count,
|
||
"cds1": str(Path(target_cds).resolve()),
|
||
"cds2": str(Path(ref_cds).resolve()),
|
||
"pep1": str(Path(target_pep).resolve()),
|
||
"pep2": str(Path(ref_pep).resolve()),
|
||
"manifest": str((final_dir / f"{run_prefix}.kaks.manifest.json").resolve()),
|
||
"results": str((final_dir / f"{run_prefix}.kaks_results.csv").resolve()),
|
||
})
|
||
if not run_records:
|
||
return f"❌ KaKs 没有可分析的 {','.join(target_species_list)}->{ref_species} ortholog pairs。", manifest
|
||
|
||
artifacts = {}
|
||
for record in run_records:
|
||
result_path = Path(record["results"])
|
||
manifest_path = Path(record["manifest"])
|
||
key_base = f"{record['target_species']}_to_{record['reference_species']}"
|
||
if result_path.exists():
|
||
result_artifact = {
|
||
"artifact": f"{key_base}_kaks_results",
|
||
"path": f"final/{result_path.name}",
|
||
"full_path": str(result_path.resolve()),
|
||
"type": "table",
|
||
"format": "csv",
|
||
"size": result_path.stat().st_size,
|
||
"sha256": _sha256_file(result_path),
|
||
}
|
||
artifacts[f"{key_base}_kaks_results"] = result_artifact
|
||
artifacts.setdefault("kaks_results", {**result_artifact, "artifact": "kaks_results"})
|
||
if manifest_path.exists():
|
||
artifacts[f"{key_base}_manifest"] = {
|
||
"artifact": f"{key_base}_manifest",
|
||
"path": f"final/{manifest_path.name}",
|
||
"full_path": str(manifest_path.resolve()),
|
||
"type": "json",
|
||
"format": "json",
|
||
"size": manifest_path.stat().st_size,
|
||
"sha256": _sha256_file(manifest_path),
|
||
}
|
||
scope_fields = {
|
||
"scope_file": str(kaks_scope_path),
|
||
"scope_id": kaks_scope_data.get("scope_id", ""),
|
||
"scope_mode": kaks_scope_data.get("scope_mode", ""),
|
||
"target_species": kaks_scope_data.get("target_species", []),
|
||
"reference_species": kaks_scope_data.get("reference_species", []),
|
||
"comparative_species": kaks_scope_data.get("comparative_species", []),
|
||
"selected_species": kaks_scope_data.get("selected_species", []),
|
||
"family": family,
|
||
}
|
||
aggregate_metrics = _aggregate_kaks_run_metrics(run_records)
|
||
aggregate_mean_ka_ks = _float_or_none(aggregate_metrics.get("mean_ka_ks"))
|
||
if aggregate_mean_ka_ks is None:
|
||
aggregate_selection_pressure = "Unknown"
|
||
elif aggregate_mean_ka_ks > 1:
|
||
aggregate_selection_pressure = "Positive"
|
||
elif aggregate_mean_ka_ks < 1:
|
||
aggregate_selection_pressure = "Purifying"
|
||
else:
|
||
aggregate_selection_pressure = "Neutral"
|
||
aggregate_manifest = final_dir / f"{prefix}.kaks.manifest.json"
|
||
manifest_data = {
|
||
"manifest_version": "1.0",
|
||
"stage": "kaks",
|
||
"status": "success",
|
||
"prefix": prefix,
|
||
"job_id": job.job_id,
|
||
"workspace": str(ws),
|
||
"workspace_root": str(ws),
|
||
"created_at": datetime.now().isoformat(),
|
||
"parameters": {
|
||
"prefix": prefix,
|
||
"analysis_route": analysis_route,
|
||
"member_set_id": member_set_id,
|
||
"neutral_rate": kaks_neutral_rate,
|
||
"ortholog_mapping": str(scoped_pairs_source),
|
||
"reference_cds": str(ref_cds),
|
||
"reference_pep": str(ref_pep),
|
||
**scope_fields,
|
||
},
|
||
"metrics": aggregate_metrics,
|
||
"method_summary": {
|
||
"scope_mode": "target_reference_aggregate",
|
||
"run_count": len(run_records),
|
||
"aggregation": "weighted_by_successful_pairs",
|
||
"note": "Per-comparison Ka/Ks metrics are preserved in metrics.run_summaries and runs.",
|
||
},
|
||
"comparative_summary": {
|
||
"total_pairs": aggregate_metrics.get("total_pairs", 0),
|
||
"successful_pairs": aggregate_metrics.get("successful_pairs", 0),
|
||
"positive_selection_count": aggregate_metrics.get("positive_selection_count", 0),
|
||
"purifying_selection_count": aggregate_metrics.get("purifying_selection_count", 0),
|
||
"neutral_selection_count": aggregate_metrics.get("neutral_selection_count", 0),
|
||
"mean_ka_ks": aggregate_metrics.get("mean_ka_ks"),
|
||
"mean_ks": aggregate_metrics.get("mean_ks"),
|
||
"selection_pressure": aggregate_selection_pressure,
|
||
},
|
||
"analysis_route": analysis_route,
|
||
"member_set_id": member_set_id,
|
||
"runs": run_records,
|
||
"artifacts": artifacts,
|
||
**scope_fields,
|
||
}
|
||
aggregate_manifest.write_text(json.dumps(manifest_data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
if scope_fields.get("scope_id"):
|
||
scoped_manifest = final_dir / f"{prefix}.{sanitize_token(str(scope_fields['scope_id']))}.kaks.manifest.json"
|
||
manifest_data["scope_manifest"] = str(scoped_manifest.absolute())
|
||
scoped_manifest.write_text(json.dumps(manifest_data, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
aggregate_manifest = scoped_manifest
|
||
return f"✅ Ka/Ks analysis completed. {len(run_records)} species comparison(s) in final/", aggregate_manifest
|
||
|
||
# 检查输入文件
|
||
cds1_file = in_dir / "cds1.fasta"
|
||
cds2_file = in_dir / "cds2.fasta"
|
||
pep1_file = in_dir / "pep1.fasta"
|
||
pep2_file = in_dir / "pep2.fasta"
|
||
pairs_file = in_dir / "ortholog_pairs.txt"
|
||
target_species = prefix.split('.')[0] if prefix else "Ft"
|
||
gene_family = prefix.split('.', 1)[1] if prefix and '.' in prefix else ""
|
||
requested_ref_species = ref_species
|
||
if isinstance(requested_ref_species, (list, tuple)):
|
||
requested_ref_species = requested_ref_species[0] if requested_ref_species else ""
|
||
ref_species = str(requested_ref_species or "").strip() or "Ath"
|
||
inter_manifest = final_dir / f"{prefix}.interspecies_synteny.manifest.json"
|
||
if inter_manifest.exists():
|
||
try:
|
||
inter_data = json.loads(inter_manifest.read_text(encoding="utf-8"))
|
||
params = inter_data.get("parameters") or {}
|
||
refs = params.get("selected_ref_species") or params.get("ref_species") or []
|
||
targets = params.get("selected_target_species") or params.get("target_species") or []
|
||
if targets:
|
||
target_species = targets[0]
|
||
if refs:
|
||
ref_species = refs[0]
|
||
except Exception as exc:
|
||
print(f"[KaKs] Warning: failed to read interspecies manifest: {exc}")
|
||
|
||
# 自动查找同源基因对文件（支持多种位置）
|
||
# 优先查找 .anchors 文件（种间共线性分析结果）
|
||
if not pairs_file.exists():
|
||
pairs_candidates = [
|
||
final_dir / f"{prefix}.orthologs.txt",
|
||
final_dir / f"{prefix}.ortholog_pairs.txt",
|
||
ws / "interspecies_synteny" / "analysis" / f"{target_species}.{ref_species}.anchors",
|
||
ws / "interspecies_synteny" / "analysis" / "Ft.Ath.anchors",
|
||
ws / "interspecies_synteny" / "analysis" / f"{prefix.split('.')[0] if prefix else 'Ft'}.Ath.anchors",
|
||
ws / "interspecies_synteny" / "analysis" / f"{target_species}.{ref_species}.anchors.simple",
|
||
in_dir / "Ft.Ath.anchors",
|
||
in_dir / "ortholog_pairs.txt",
|
||
ws / "kaks_analysis" / "ortholog_pairs.txt",
|
||
ws / "kaks_analysis" / "ortholog_pairs_fixed.txt",
|
||
in_dir / "Ft_Ath_orthologs.txt",
|
||
in_dir / "orthologs.txt",
|
||
]
|
||
for candidate in pairs_candidates:
|
||
if candidate.exists():
|
||
pairs_file = candidate
|
||
print(f"[KaKs] Found ortholog pairs file: {pairs_file}")
|
||
break
|
||
|
||
# 自动查找CDS和蛋白序列文件
|
||
if not cds1_file.exists():
|
||
cds1_candidates = [
|
||
in_dir / "Ft.cds.fasta",
|
||
in_dir / f"{target_species}.cds.fasta",
|
||
in_dir / target_species / f"{target_species}.cds.fasta",
|
||
]
|
||
for candidate in cds1_candidates:
|
||
if candidate.exists():
|
||
cds1_file = candidate
|
||
print(f"[KaKs] Found CDS1 file: {cds1_file}")
|
||
break
|
||
|
||
if not cds2_file.exists():
|
||
cds2_candidates = [
|
||
in_dir / f"{ref_species}.cds.fasta",
|
||
in_dir / ref_species / f"{ref_species}.cds.fasta",
|
||
in_dir / "Ath.cds.fasta",
|
||
in_dir / "ath.cds.fasta",
|
||
]
|
||
for candidate in cds2_candidates:
|
||
if candidate.exists():
|
||
cds2_file = candidate
|
||
print(f"[KaKs] Found CDS2 file: {cds2_file}")
|
||
break
|
||
|
||
if not pep1_file.exists():
|
||
pep1_candidates = [
|
||
in_dir / "Ft.pep.fasta",
|
||
in_dir / f"{target_species}.pep.fasta",
|
||
in_dir / target_species / f"{target_species}.pep.fasta",
|
||
final_dir / f"{prefix}.final_members.pep.fa",
|
||
]
|
||
for candidate in pep1_candidates:
|
||
if candidate.exists():
|
||
pep1_file = candidate
|
||
print(f"[KaKs] Found PEP1 file: {pep1_file}")
|
||
break
|
||
|
||
if not pep2_file.exists():
|
||
pep2_candidates = [
|
||
in_dir / f"{ref_species}.pep.fasta",
|
||
in_dir / ref_species / f"{ref_species}.pep.fasta",
|
||
in_dir / "Ath.pep.fasta",
|
||
in_dir / "ath.pep.fasta",
|
||
]
|
||
for candidate in pep2_candidates:
|
||
if candidate.exists():
|
||
pep2_file = candidate
|
||
print(f"[KaKs] Found PEP2 file: {pep2_file}")
|
||
break
|
||
|
||
# 验证必要文件是否存在
|
||
missing_files = []
|
||
if not pep1_file.exists():
|
||
missing_files.append("pep1.fasta (物种1蛋白序列)")
|
||
if not pep2_file.exists():
|
||
missing_files.append("pep2.fasta (物种2蛋白序列)")
|
||
if not pairs_file.exists():
|
||
missing_files.append("ortholog_pairs.txt (同源基因对)")
|
||
|
||
# 如果缺少CDS文件，尝试从GFF3+Genome提取
|
||
if not cds1_file.exists():
|
||
print("[KaKs] CDS1 file not found, attempting to extract from GFF3+Genome...")
|
||
gff1_candidates = [
|
||
in_dir / "Ft_final.gff3",
|
||
in_dir / f"{prefix.split('.')[0] if prefix else 'Ft'}_final.gff3",
|
||
in_dir / "Ft.gff3",
|
||
]
|
||
genome1_candidates = [
|
||
in_dir / "Ft.genome.fasta",
|
||
in_dir / f"{prefix.split('.')[0] if prefix else 'Ft'}.genome.fasta",
|
||
in_dir / "Ft.fa",
|
||
]
|
||
gff1_file = None
|
||
genome1_file = None
|
||
for candidate in gff1_candidates:
|
||
if candidate.exists():
|
||
gff1_file = candidate
|
||
break
|
||
for candidate in genome1_candidates:
|
||
if candidate.exists():
|
||
genome1_file = candidate
|
||
break
|
||
|
||
if gff1_file and genome1_file:
|
||
cds1_extracted = in_dir / "Ft.cds.fasta"
|
||
cmd = [
|
||
'/home/xzq/miniconda3/bin/conda', 'run', '-n', 'test4py311',
|
||
'gffread', '-x', str(cds1_extracted), '-g', str(genome1_file), str(gff1_file)
|
||
]
|
||
print(f"[KaKs] Extracting CDS1 with conda gffread...")
|
||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||
if result.returncode == 0 and cds1_extracted.exists() and cds1_extracted.stat().st_size > 0:
|
||
cds1_file = cds1_extracted
|
||
print(f"[KaKs] ✅ Successfully extracted CDS1: {cds1_file} ({cds1_extracted.stat().st_size} bytes)")
|
||
else:
|
||
print(f"[KaKs] ❌ Failed to extract CDS1: {result.stderr}")
|
||
missing_files.append("cds1.fasta (物种1 CDS序列，尝试从GFF3提取失败)")
|
||
else:
|
||
missing_files.append("cds1.fasta (物种1 CDS序列，缺少GFF3或Genome文件)")
|
||
|
||
if not cds2_file.exists():
|
||
print("[KaKs] CDS2 file not found, attempting to extract from GFF3+Genome...")
|
||
gff2_candidates = [
|
||
in_dir / "Ath_final.gff3",
|
||
in_dir / "ath_final.gff3",
|
||
in_dir / "Ath.gff3",
|
||
]
|
||
genome2_candidates = [
|
||
in_dir / "Ath.genome.fasta",
|
||
in_dir / "ath.genome.fasta",
|
||
]
|
||
gff2_file = None
|
||
genome2_file = None
|
||
for candidate in gff2_candidates:
|
||
if candidate.exists():
|
||
gff2_file = candidate
|
||
break
|
||
for candidate in genome2_candidates:
|
||
if candidate.exists():
|
||
genome2_file = candidate
|
||
break
|
||
|
||
if gff2_file and genome2_file:
|
||
cds2_extracted = in_dir / "Ath.cds.fasta"
|
||
cmd = [
|
||
'/home/xzq/miniconda3/bin/conda', 'run', '-n', 'test4py311',
|
||
'gffread', '-x', str(cds2_extracted), '-g', str(genome2_file), str(gff2_file)
|
||
]
|
||
print(f"[KaKs] Extracting CDS2 with conda gffread...")
|
||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||
if result.returncode == 0 and cds2_extracted.exists() and cds2_extracted.stat().st_size > 0:
|
||
cds2_file = cds2_extracted
|
||
print(f"[KaKs] ✅ Successfully extracted CDS2: {cds2_file} ({cds2_extracted.stat().st_size} bytes)")
|
||
else:
|
||
print(f"[KaKs] ❌ Failed to extract CDS2: {result.stderr}")
|
||
missing_files.append("cds2.fasta (物种2 CDS序列，尝试从GFF3提取失败)")
|
||
else:
|
||
# Ath的CDS可能不是必须的，如果用户提供了蛋白序列，我们可以尝试继续
|
||
print(f"[KaKs] ⚠️ Cannot extract CDS2: GFF2={gff2_file}, Genome2={genome2_file}")
|
||
missing_files.append("cds2.fasta (物种2 CDS序列)")
|
||
|
||
if missing_files:
|
||
return f"❌ KaKs stage missing required files: {', '.join(missing_files)}", manifest
|
||
|
||
print(f"[KaKs] Neutral rate: {kaks_neutral_rate}")
|
||
print(f"[KaKs] Output dir: {kaks_dir}")
|
||
print(f"[KaKs] CDS1: {cds1_file}")
|
||
print(f"[KaKs] CDS2: {cds2_file}")
|
||
print(f"[KaKs] PEP1: {pep1_file}")
|
||
print(f"[KaKs] PEP2: {pep2_file}")
|
||
print(f"[KaKs] Pairs: {pairs_file}")
|
||
|
||
# 查找基因列表文件（用于过滤pairs，只分析特定基因家族）
|
||
gene_list1_file = None
|
||
gene_list2_file = None
|
||
|
||
# 物种1的基因列表
|
||
if prefix:
|
||
gene_list1_candidates = [
|
||
in_dir / f"{prefix}.geneID.subfamily", # Multi.SPL.geneID.subfamily
|
||
in_dir / f"{target_species}.geneID.subfamily", # Ft.geneID.subfamily
|
||
final_dir / f"{prefix}.geneID.subfamily",
|
||
]
|
||
for candidate in gene_list1_candidates:
|
||
if candidate.exists():
|
||
gene_list1_file = candidate
|
||
print(f"[KaKs] Found gene list 1: {gene_list1_file}")
|
||
break
|
||
|
||
# 物种2的基因列表（Ath）
|
||
gene_list2_candidates = [
|
||
in_dir / f"{ref_species}.{gene_family}.geneID.subfamily",
|
||
in_dir / ref_species / f"{ref_species}.{gene_family}.geneID.subfamily",
|
||
final_dir / f"{ref_species}.{gene_family}.geneID.subfamily",
|
||
in_dir / "SPL_Ath.subfamily",
|
||
in_dir / "Ath.geneID.subfamily",
|
||
]
|
||
for candidate in gene_list2_candidates:
|
||
if candidate.exists():
|
||
gene_list2_file = candidate
|
||
print(f"[KaKs] Found gene list 2: {gene_list2_file}")
|
||
break
|
||
|
||
# 使用conda环境运行KaKs分析
|
||
kaks_cmd = [
|
||
"/home/xzq/miniconda3/bin/conda", "run", "-n", "test4py311",
|
||
"python", "apps/cli/run_kaks_v2.py",
|
||
"-o", str(kaks_dir),
|
||
"--prefix-name", prefix,
|
||
"--cds1", str(cds1_file),
|
||
"--cds2", str(cds2_file),
|
||
"--pep1", str(pep1_file),
|
||
"--pep2", str(pep2_file),
|
||
"--pairs", str(pairs_file),
|
||
"--neutral-rate", str(kaks_neutral_rate),
|
||
]
|
||
|
||
# 添加基因列表参数（如果找到）
|
||
if gene_list1_file:
|
||
kaks_cmd.extend(["--gene-list1", str(gene_list1_file)])
|
||
if gene_list2_file:
|
||
kaks_cmd.extend(["--gene-list2", str(gene_list2_file)])
|
||
|
||
print(f"[CMD] {' '.join(kaks_cmd)}")
|
||
code = run_cmd_logged(kaks_cmd, log_file, cwd=Path("."))
|
||
|
||
if code == 0:
|
||
# 复制结果到final目录
|
||
kaks_results = list(kaks_dir.glob("*.kaks_results.csv"))
|
||
for f in kaks_results:
|
||
shutil.copy2(f, final_dir)
|
||
|
||
file_count = len(kaks_results)
|
||
kaks_manifest = final_dir / f"{prefix}.kaks.manifest.json"
|
||
return f"✅ Ka/Ks analysis completed. {file_count} files in final/", kaks_manifest
|
||
else:
|
||
return f"❌ Ka/Ks analysis failed (exit code: {code}). Log: {log_file}", manifest
|
||
|
||
# ========== Genome Prepare stage ==========
|
||
if stage == "genome_prepare":
|
||
script = Path("apps/cli/run_genome_prepare.py")
|
||
if not script.exists():
|
||
return "Genome Prepare script not found: apps/cli/run_genome_prepare.py", manifest
|
||
|
||
# genome_prepare is a data-preparation stage, not a gene-family stage.
|
||
# Ignore stale UI/plan gene-family prefixes so
|
||
# prepared reference resources are not labeled with an invented family.
|
||
genome_prepare_prefix = "genome_prepare"
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--input", str(in_dir),
|
||
"--output", str(in_dir),
|
||
"--workspace", str(ws),
|
||
"--prefix", genome_prepare_prefix,
|
||
"--scripts-dir", str(Path("scripts").resolve()),
|
||
]
|
||
|
||
print(f"[CMD] {' '.join(cmd)}")
|
||
code = run_cmd_logged(cmd, log_file, cwd=Path("."))
|
||
|
||
input_final_dir = in_dir / "final"
|
||
workspace_final_dir = ws / "final"
|
||
genome_prepare_manifest = None
|
||
if workspace_final_dir.exists():
|
||
manifest_files = sorted(workspace_final_dir.glob("*.genome_prepare.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||
if manifest_files:
|
||
genome_prepare_manifest = manifest_files[0]
|
||
|
||
if not genome_prepare_manifest and input_final_dir.exists():
|
||
manifest_files = sorted(input_final_dir.glob("*.genome_prepare.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||
if manifest_files:
|
||
input_manifest = manifest_files[0]
|
||
workspace_final_dir.mkdir(parents=True, exist_ok=True)
|
||
workspace_manifest = workspace_final_dir / input_manifest.name
|
||
try:
|
||
shutil.copy2(input_manifest, workspace_manifest)
|
||
genome_prepare_manifest = workspace_manifest
|
||
except Exception as e:
|
||
print(f"[WARN] Failed to copy genome_prepare manifest to workspace: {e}")
|
||
genome_prepare_manifest = input_manifest
|
||
|
||
if not genome_prepare_manifest:
|
||
genome_prepare_manifest = workspace_final_dir / f"{genome_prepare_prefix or prefix or 'multi_species'}.genome_prepare.manifest.json"
|
||
|
||
def _genome_prepare_summary(path: Path) -> str:
|
||
if not path or not path.exists():
|
||
return ""
|
||
try:
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
except Exception as exc:
|
||
return f" Manifest read failed: {exc}"
|
||
metrics = data.get("metrics") or {}
|
||
preflight = metrics.get("config_preflight") or {}
|
||
parts = []
|
||
invalid = preflight.get("invalid_species") or []
|
||
if invalid:
|
||
parts.append("Skipped by config preflight: " + ", ".join(str(x) for x in invalid))
|
||
for item in (preflight.get("row_errors") or [])[:8]:
|
||
row = item.get("row", "?")
|
||
species = item.get("species", "?")
|
||
original = item.get("original") or "-"
|
||
reason = item.get("reason") or "unknown"
|
||
parts.append(f"{species} row {row} {original}: {reason}")
|
||
runtime_failures = []
|
||
for result in metrics.get("species_results") or []:
|
||
if result.get("success"):
|
||
continue
|
||
if result.get("error_type") == "config_preflight_failed":
|
||
continue
|
||
runtime_failures.append(f"{result.get('species', '?')}: {result.get('error', 'unknown')}")
|
||
if runtime_failures:
|
||
parts.append("Runtime failures: " + "; ".join(runtime_failures[:5]))
|
||
successes = [str(r.get("species")) for r in metrics.get("species_results") or [] if r.get("success")]
|
||
if successes:
|
||
parts.append("Executed successfully: " + ", ".join(successes))
|
||
return " | " + " | ".join(parts) if parts else ""
|
||
|
||
existing_files = [genome_prepare_manifest] if genome_prepare_manifest.exists() else []
|
||
summary = _genome_prepare_summary(genome_prepare_manifest)
|
||
if code == 0 and genome_prepare_manifest.exists():
|
||
return f"Genome Prepare complete. Generated {len(existing_files)} files. Log: {log_file}{summary}", genome_prepare_manifest
|
||
elif code == 0:
|
||
return f"Genome Prepare finished but manifest not found in workspace/final. Log: {log_file}", manifest
|
||
else:
|
||
return f"Genome Prepare failed (exit code: {code}). Log: {log_file}{summary}", genome_prepare_manifest if genome_prepare_manifest.exists() else manifest
|
||
|
||
if 'repro_manager' in locals() and repro_manager:
|
||
try:
|
||
repro_manager.finalize()
|
||
except Exception as e:
|
||
print(f"[WARN] Failed to finalize reproducibility: {e}")
|
||
|
||
return f"❌ 未知 stage: {stage}", manifest
|