#!/usr/bin/env python3 """ BotanicalBaseAgent - FastAPI Backend 支持Vue3前端的API接口 - Celery任务队列模式 """ import os import sys import json import csv import asyncio import hashlib import uuid import time import signal import shutil import re import subprocess import hashlib from io import BytesIO from pathlib import Path from datetime import datetime, timedelta from typing import Any, Optional, List, Dict, Literal from fastapi import FastAPI, UploadFile, File, Form, HTTPException, Request, BackgroundTasks, Query from fastapi.responses import JSONResponse, FileResponse, Response, StreamingResponse from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles from pydantic import BaseModel, Field from concurrent.futures import ThreadPoolExecutor import builtins as _builtins _ORIGINAL_PRINT = _builtins.print def _safe_print(*args, **kwargs): try: _ORIGINAL_PRINT(*args, **kwargs) except BrokenPipeError: pass _builtins.print = _safe_print # 添加项目路径 PROJECT_ROOT = Path(__file__).parent.parent.parent.parent sys.path.insert(0, str(PROJECT_ROOT)) from gfa_core.project_paths import get_conda_env_bin, get_conda_env_rscript, get_configured_path, get_wgs_bcftools, get_wgs_java, get_wgs_samtools ASAW_BIN = get_conda_env_bin("ASAW") ASAW_RSCRIPT = get_conda_env_rscript("ASAW") TASSEL_PIPELINE = get_configured_path("TASSEL_PIPELINE", "run_pipeline.pl") GWAS_EMMAX_BIN = get_configured_path("GWAS_EMMAX_BIN", "emmax") WGS_BEAGLE_JAR = get_configured_path("WGS_BEAGLE_JAR") POPULATION_STAGES = {"pop_pca", "pop_phylogeny", "pop_structure", "pop_ld_decay", "pop_snp_density"} GF_MAS_ROOT = Path(os.environ.get("GF_MAS_ROOT", "/home/xzq/gene-family-mas-v2")).expanduser() if GF_MAS_ROOT.exists() and str(GF_MAS_ROOT) not in sys.path: sys.path.insert(0, str(GF_MAS_ROOT)) PLANTAI_RAG_ROOT = Path(os.environ.get("PLANTAI_RAG_ROOT", "/home/xzq/PlantAI-rag")).expanduser() if PLANTAI_RAG_ROOT.exists() and str(PLANTAI_RAG_ROOT) not in sys.path: sys.path.insert(0, str(PLANTAI_RAG_ROOT)) try: from src.rag.unified.bbav5_methodology_config import build_bbav5_methodology_config METHODOLOGY_CONFIG_AVAILABLE = True except Exception as e: METHODOLOGY_CONFIG_AVAILABLE = False print(f"⚠️ 方法学配置模块加载失败: {e}") try: from gfa_core.mas.agents.interpretation_evidence import build_evidence_report EVIDENCE_INTERPRETATION_AVAILABLE = True except Exception as e: EVIDENCE_INTERPRETATION_AVAILABLE = False build_evidence_report = None print(f"⚠️ 证据驱动解读模块加载失败: {e}") from gfa_core.gene_family_scope import build_gene_family_scope, read_scope, scope_manifest_path, write_scope from gfa_core.gene_family_stage_context import build_stage_context, enrich_stage_params_with_context from gfa_core.rnaseq_de_design import ( build_de_design_review, normalize_group_role, parse_bool, plan_rnaseq_de_design, ) from gfa_core.rnaseq_context import read_rnaseq_sample_metadata, resolve_rnaseq_project_context from gfa_core.gwas_fmt_context import ( canonicalize_gwas_task_results as _canonicalize_gwas_task_results, gwas_fmt_result_artifacts as _gwas_fmt_result_artifacts, resolve_gwas_fmt_input, semantic_external_vcf_preflight, trusted_genotype_scope_id as _trusted_genotype_scope_id, ) from gfa_core.gwas_upstream_manifest import ( GWAS_RESULT_STAGES as _GWAS_RESULT_STAGES, GWAS_STAGE_SCOPE_FIELDS as _GWAS_STAGE_SCOPE_FIELDS, validated_result_artifacts as _validated_gwas_result_artifacts, ) from gfa_core.genotype_context import sha256_file from gfa_core.population_manifest import ( PopulationManifestError, resolve_population_manifest_identity, ) from gfa_core.mas.artifact_resolver import ArtifactResolver from gfa_core.mas.context_index import build_project_context from gfa_core.mas.stage_capabilities import get_stage_capability from gfa_core.mas.subfamily_status import ( confirmed_subfamily_file as resolve_confirmed_subfamily_file, prediction_subfamily_file as resolve_prediction_subfamily_file, subfamily_state_for_scope, subfamily_state_for_species as resolve_subfamily_state_for_species, target_member_files as resolve_target_member_files, ) from gfa_core.extension_stage import ExtensionStageAgent, ExtensionStageError, ExtensionStageManager from gfa_core.extension_stage.intent import resolve_extension_slots from gfa_core.methodology_bridge import ( bind_methodology_plan, is_methodology_execution_intent, load_bound_methodology_plan, methodology_agent_params, methodology_plan_readiness, ) # 全局状态存储（用于等待用户确认） pending_confirmations: Dict[str, Dict] = {} # 会话状态存储 session_states: Dict[str, Dict] = {} # 深度解读报告缓存：按 session_id 保存最近一次真实 report，供下载接口使用。 interpretation_report_cache: Dict[str, Dict[str, Any]] = {} extension_stage_manager = ExtensionStageManager( project_root=PROJECT_ROOT, store_root=PROJECT_ROOT / "extension_stages", workspace_root=PROJECT_ROOT / "workspace", input_root=PROJECT_ROOT / "input", ) extension_stage_agent = ExtensionStageAgent(extension_stage_manager) def _cache_interpretation_report(session_id: str, stage: str, report: dict, source: str = "") -> None: if not session_id or not isinstance(report, dict): return metadata = report.get("metadata") or {} interpretation_report_cache[session_id] = { "stage": stage or metadata.get("current_stage", ""), "report": report, "source": source, "cached_at": datetime.now().isoformat(), "identity": metadata.get("resolved_identity") or { "prefix": metadata.get("prefix", ""), "scope_id": metadata.get("scope_id", ""), "analysis_route": metadata.get("analysis_route", ""), "member_set_id": metadata.get("member_set_id", ""), "rnaseq_scope_id": metadata.get("rnaseq_scope_id", ""), }, } def _persist_interpretation_report(job_id: str, prefix: str, report: dict, *, scope_id: str = "", rnaseq_scope_id: str = "") -> bool: if not job_id or not prefix or not isinstance(report, dict): return False try: from gfa_core.mas.agents.result_parser import create_result_parser_agent agent = create_result_parser_agent(None) return bool(agent.save_interpretation( job_id, prefix, report, scope_id=scope_id, rnaseq_scope_id=rnaseq_scope_id, )) except Exception as exc: print(f"[INTERPRET] 保存scope-aware深度解读失败: {exc}") return False def _report_to_markdown(report: dict) -> str: if not isinstance(report, dict): return "" download_info = report.get("download_info") or {} if isinstance(download_info, dict) and download_info.get("markdown"): return str(download_info.get("markdown")) metadata = report.get("metadata") or {} lines = [ f"# {metadata.get('title') or '深度解读报告'}", "", f"Job: {metadata.get('job_id', '')}", f"Stage: {metadata.get('current_stage', '')}", f"Generated: {metadata.get('generated_at', '')}", f"Scope ID: {metadata.get('scope_id', '')}", f"RNA-seq Scope ID: {metadata.get('rnaseq_scope_id', '')}", "", ] stages = ((report.get("stage_flow") or {}).get("stages") or []) if stages: lines.append("## Stage Flow") for stage in stages: lines.append(f"- {stage.get('name') or stage.get('id')}: {stage.get('key_result', '')}") lines.append("") findings = report.get("key_findings") or [] if findings: lines.append("## Key Findings") for finding in findings: lines.extend([ "", f"### {finding.get('id', '')}. {finding.get('title', '')}", str(finding.get("description", "")), f"Evidence: {finding.get('evidence', '')}", ]) lines.append("") discoveries = report.get("scientific_discoveries") or [] if discoveries: lines.append("## Scientific Discoveries") for item in discoveries: lines.extend(["", f"### {item.get('id', '')}. {item.get('title', '')}", str(item.get("claim", "")), f"Evidence: {item.get('evidence', '')}"]) lines.append("") risks = report.get("risk_flags") or [] if risks: lines.append("## Risk Flags") for risk in risks: lines.append(f"- [{risk.get('severity', '')}] {risk.get('stage', '')}: {risk.get('message', '')}") lines.append("") if report.get("biological_story"): lines.extend(["## Biological Story", str(report.get("biological_story")), ""]) next_steps = report.get("next_steps") or [] if next_steps: lines.append("## Next Steps") for step in next_steps: lines.append(f"- [{step.get('priority', '')}] {step.get('stage', '')}: {step.get('description', '')} ({step.get('rationale', '')})") return "\n".join(lines).strip() + "\n" def _stage_manifest_aliases(stage: str) -> list[str]: aliases = { "ncbi_fetch_sequences": ["ncbi_fetch_sequences", "ncbi_fetch"], "sequence_extract_by_id": ["sequence_extract_by_id", "sequence_extract"], } return aliases.get(stage, [stage]) def _find_stage_manifest(final_dir: Path, stage: str, prefix: str = "") -> Optional[Path]: patterns: list[str] = [] for alias in _stage_manifest_aliases(stage): if prefix: patterns.append(f"{prefix}.{alias}.manifest.json") patterns.extend([f"*.{alias}.manifest.json", f"*{alias}*.manifest.json"]) for pattern in patterns: for path in sorted(final_dir.glob(pattern), key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True): try: data = json.loads(path.read_text(encoding="utf-8")) except Exception: continue manifest_stage = data.get("stage") if manifest_stage == stage or manifest_stage in _stage_manifest_aliases(stage) or any(alias in path.name for alias in _stage_manifest_aliases(stage)): return path return None COMBO_EVIDENCE_PANELS = ["motif", "structure", "domain"] def _normalize_combo_panel_list(raw_panels: Any, *, default: Optional[list[str]] = None) -> list[str]: aliases = { "motifs": "motif", "domains": "domain", "gene_structure": "structure", "gene-structure": "structure", "genestructure": "structure", "all": "all", } if isinstance(raw_panels, str): raw_items = re.split(r"[,;，；\s]+", raw_panels) elif isinstance(raw_panels, (list, tuple, set)): raw_items = list(raw_panels) else: raw_items = [] panels: list[str] = [] for item in raw_items: key = str(item or "").strip().lower().replace(" ", "_") if not key: continue key = aliases.get(key, key) if key == "all": return list(COMBO_EVIDENCE_PANELS) if key in COMBO_EVIDENCE_PANELS and key not in panels: panels.append(key) return panels or list(default or []) def _manifest_combo_output_label_value(manifest: dict) -> str: params = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {} return str(params.get("output_label") or manifest.get("output_label") or "").strip() def _manifest_standard_combo_panels(manifest_path: Path) -> list[str]: try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: return [] if _manifest_combo_output_label_value(manifest): return [] artifacts = manifest.get("artifacts") if isinstance(manifest.get("artifacts"), dict) else {} if any(str(key or "").startswith("combo_replot") for key in artifacts): return [] params = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {} return _normalize_combo_panel_list(params.get("panels") or manifest.get("panels"), default=list(COMBO_EVIDENCE_PANELS)) def _completed_combo_evidence_panels(job_id: str, prefix: str) -> list[str]: if not job_id: return [] final_dir = WORKSPACE_ROOT / job_id / "final" if not final_dir.exists(): return [] panels: list[str] = [] for panel in COMBO_EVIDENCE_PANELS: if _find_stage_manifest(final_dir, panel, prefix): panels.append(panel) return panels if len(panels) >= 2 else [] def _combo_panels_from_params(params: Dict[str, Any]) -> list[str]: return _normalize_combo_panel_list((params or {}).get("comboPanels") or (params or {}).get("combo_panels")) def _resolve_standard_combo_panels_for_plan( *, stage_ids: list[str], plan_params: Dict[str, Any], job_id: str, prefix: str, direct_combo: bool = False, ) -> list[str]: explicit = _combo_panels_from_params(plan_params or {}) if explicit: return explicit planned = [panel for panel in COMBO_EVIDENCE_PANELS if panel in stage_ids] if len(planned) >= 2: return planned if direct_combo: completed = _completed_combo_evidence_panels(job_id, prefix) if completed: return completed return list(COMBO_EVIDENCE_PANELS) def _apply_standard_combo_panel_plan( *, stage_ids: list[str], stages_detail: list[Dict[str, Any]], plan_params: Dict[str, Any], plan_prefix: str, job_id: str = "", direct_combo: bool = False, ) -> tuple[list[str], list[Dict[str, Any]]]: evidence_in_plan = [panel for panel in COMBO_EVIDENCE_PANELS if panel in stage_ids] has_combo = "combo" in stage_ids if not has_combo and len(evidence_in_plan) < 2: return stage_ids, stages_detail selected = _resolve_standard_combo_panels_for_plan( stage_ids=stage_ids, plan_params=plan_params or {}, job_id=job_id, prefix=plan_prefix or str((plan_params or {}).get("prefix") or ""), direct_combo=direct_combo or has_combo, ) if len(selected) < 2: return stage_ids, stages_detail ordered_stage_ids = list(dict.fromkeys(stage_ids)) if not has_combo and len(evidence_in_plan) >= 2: ordered_stage_ids.append("combo") detail_by_stage: dict[str, Dict[str, Any]] = {} for detail in stages_detail or []: if not isinstance(detail, dict): continue sid = str(detail.get("id") or detail.get("name") or "").strip() if sid and sid not in detail_by_stage: detail_by_stage[sid] = detail normalized_details: list[Dict[str, Any]] = [] for order, sid in enumerate(ordered_stage_ids, start=1): detail = dict(detail_by_stage.get(sid) or _stage_detail_defaults(sid)) detail["id"] = sid detail["order"] = order detail_params = {**(plan_params or {}), **(detail.get("params") or {})} if plan_prefix and not detail_params.get("prefix"): detail_params["prefix"] = plan_prefix if sid in COMBO_EVIDENCE_PANELS: detail["dependencies"] = ["phylogeny"] elif sid == "combo": detail["dependencies"] = list(selected) detail_params["comboPanels"] = list(selected) plan_params["comboPanels"] = list(selected) if sid in GENE_FAMILY_SCOPE_STAGES: detail["params"] = enrich_stage_params_with_context(sid, detail_params.get("prefix") or plan_prefix, detail_params) else: detail["params"] = detail_params normalized_details.append(detail) return ordered_stage_ids, normalized_details def _rnaseq_auto_annotation_preflight(job_id: str, req: Any) -> dict[str, Any]: """Return confirmation payload when RNA-seq enrichment would start local annotation.""" requested_stage = str(getattr(req, "stage", "") or "") if requested_stage not in {"rnaseq_enrich", "rnaseq_go_enrich", "rnaseq_kegg_enrich"}: return {} if requested_stage == "rnaseq_go_enrich": enrich_type = "go" elif requested_stage == "rnaseq_kegg_enrich": enrich_type = "kegg" else: enrich_type = str(getattr(req, "enrichType", "go") or "go").strip().lower() if enrich_type not in {"go", "kegg", "both"}: enrich_type = "go" background_source = str(getattr(req, "enrichBackgroundSource", "auto") or "auto").strip().lower() if background_source in {"annotation", "eggnog", "auto_annotation"}: return {} if background_source != "auto": return {} if bool(getattr(req, "confirmedAutoAnnotation", False)): return {} final_dir = WORKSPACE_ROOT / job_id / "final" input_dir = INPUT_ROOT / job_id try: from apps.web.pipeline_backend import _filter_gene2go_files_by_de_overlap, _has_non_comment_data except Exception as exc: print(f"[API DEBUG] RNA-seq auto-annotation preflight skipped: {exc}") return {} need_go = enrich_type in {"go", "both"} need_kegg = enrich_type in {"kegg", "both"} gene2go_files = ( list(input_dir.glob("*gene2go*.txt")) + list(input_dir.glob("*gene2go*.tsv")) + list(final_dir.glob("*gene2go*.txt")) + list(final_dir.glob("*gene2go*.tsv")) ) gene2ko_files = ( list(input_dir.glob("*gene2ko*.txt")) + list(input_dir.glob("*gene2ko*.tsv")) + list(final_dir.glob("*gene2ko*.txt")) + list(final_dir.glob("*gene2ko*.tsv")) ) gene2go_files = [p for p in gene2go_files if _has_non_comment_data(p) and ".test." not in p.name] gene2ko_files = [p for p in gene2ko_files if _has_non_comment_data(p) and ".test." not in p.name] compatible, reports = _filter_gene2go_files_by_de_overlap(gene2go_files, final_dir) if need_go else ([], []) species = str(getattr(req, "enrichSpecies", "") or getattr(req, "speciesName", "") or "").strip().lower() supported_orgdb_species = {"ath", "hsa", "mmu", "rno", "dme", "sce"} mismatch = bool(gene2go_files and reports and not compatible) missing_usable_go = need_go and (mismatch or (not compatible and species not in supported_orgdb_species)) missing_gene2ko = need_kegg and not gene2ko_files if not missing_usable_go and not missing_gene2ko: return {} pep_candidates = ( list(input_dir.glob("*.pep.fasta")) + list(input_dir.glob("*.pep.fa")) + list(input_dir.glob("*.protein.fa")) + list(input_dir.glob("*protein*.fasta")) + list(final_dir.glob("*final_members.pep.fa")) ) required_backgrounds = [] if missing_usable_go: required_backgrounds.append("gene2go") if missing_gene2ko: required_backgrounds.append("gene2ko") if missing_usable_go and missing_gene2ko: reason = "missing_enrichment_background" elif mismatch: reason = "gene2go_id_mismatch" elif missing_usable_go: reason = "missing_usable_gene2go" else: reason = "missing_gene2ko" report = reports[0] if reports else {} if missing_usable_go and missing_gene2ko: message = "当前 GO/KEGG 富集背景缺失或与 DE 基因 ID 不匹配，需要启动本地自动注释。" elif missing_gene2ko: message = "当前 KEGG 富集背景缺失，需要启动本地自动注释生成 gene2ko。" elif mismatch: message = "当前 GO 背景与 DE 基因 ID 不匹配，需要启动本地自动注释生成匹配的 gene2go。" else: message = "当前 GO 富集背景缺失，需要启动本地自动注释生成 gene2go。" return { "reason": reason, "message": message, "required_backgrounds": required_backgrounds, "gene2go_files": [str(p) for p in gene2go_files], "compatible_gene2go_files": [str(p) for p in compatible], "gene2ko_files": [str(p) for p in gene2ko_files], "overlap_report": report, "protein_file": str(pep_candidates[0]) if pep_candidates else "", "has_protein_file": bool(pep_candidates), "enrich_type": enrich_type, "background_source": background_source, } def _pending_has_executable_fix(pending: Dict[str, Any]) -> bool: diagnosis = pending.get("diagnosis") or {} issues = diagnosis.get("issues") or [] primary_issue = diagnosis.get("primary_issue") if not isinstance(primary_issue, dict): primary_issue = issues[0] if issues else {} primary_code = str(primary_issue.get("code") or "") primary_layer = str(primary_issue.get("layer") or primary_issue.get("issue_layer") or "") if primary_layer == "planning" or ( primary_layer == "artifact_status" and primary_code in {"manifest_missing", "manifest_missing_stage_incomplete", "manifest_failed"} ): return False if diagnosis.get("safe_patch_candidates"): return True executable_codes = { "python_import_path_error", "protein_fasta_stop_markers", "r_table_comment_char", "success_with_zero_alignment_rate", "success_with_zero_qc_reads", "fgsea_rank_gene2ko_id_mismatch", "wgcna_matrix_comment_char", "quant_fpkm_tpm_comment_char", } for issue in issues: if issue.get("safe_patch_candidate") and issue.get("code") in executable_codes: return True return False def _pending_should_be_visible(pending: Dict[str, Any]) -> bool: """Show diagnostic reports even when they are read-only. Executable fixes still go through the confirmation path, but blocking/error diagnostics must not disappear just because the current agent cannot patch them automatically. """ if _pending_has_executable_fix(pending): return True diagnosis = pending.get("diagnosis") or {} if diagnosis.get("safe_patch_candidates"): return True severity = str(diagnosis.get("severity") or "").lower() if severity in {"blocking", "error", "critical", "warning"}: return True for issue in diagnosis.get("issues") or []: issue_severity = str(issue.get("severity") or "").lower() if issue_severity in {"blocking", "error", "critical", "warning"}: return True return bool(diagnosis) def _pending_resolved_by_manifest(pending: Dict[str, Any]) -> bool: job_id = pending.get("job_id", "") stage = pending.get("stage", "") prefix = pending.get("prefix", "") if not (job_id and stage): return False manifest_path = _find_stage_manifest(PROJECT_ROOT / "workspace" / job_id / "final", stage, prefix) if not manifest_path: return False try: data = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: return False return data.get("status") in {"success", "partial_success"} def _looks_like_gene_id_result_view(text: str) -> bool: value = str(text or "") lower = value.lower() has_existing_result = any( kw in lower or kw in value for kw in ["下载好的", "已下载", "已经下载", "下载结果", "下载产物", "已有结果", "已有的"] ) has_view_action = any( kw in lower or kw in value for kw in ["展示", "显示", "列出", "查看", "列表", "明细", "打印"] ) has_gene_id = any( kw in lower or kw in value for kw in ["基因id", "gene id", "gene ids", "id列表", "ids", "geneid"] ) return has_existing_result and has_view_action and has_gene_id def _extract_prefix_from_result_query(text: str) -> str: value = str(text or "") match = re.search(r"(? list[str]: species_map = {code.lower(): code for code in ["Ft", "Si", "At", "Os", "Ls", "Sl", "Zm", "Gm", "Br", "Vv", "Pt", "Sb", "Lc", "Dl", "Ppy", "Py", "Cp", "Lg"]} keyword_map = { "拟南芥": "At", "arabidopsis": "At", "thaliana": "At", "水稻": "Os", "rice": "Os", "生菜": "Ls", "莴苣": "Ls", "lettuce": "Ls", "lactuca": "Ls", "荔枝": "Lc", "litchi": "Lc", "龙眼": "Dl", "longan": "Dl", } value = str(text or "") lower = value.lower() hits: list[tuple[int, str]] = [] code_pattern = r"(? Dict[str, Any]: try: from gfa_core.mas.utils.intent_parser import parse_gene_family_scope_selection selection = parse_gene_family_scope_selection(text) return selection if isinstance(selection, dict) else {} except Exception: species = _extract_species_codes_from_text(text) return { "targetSpecies": species, "referenceSpecies": [], "comparativeSpecies": [], "scopeMode": "target_only", } def _apply_scope_selection_to_intent(intent: Dict[str, Any], message: str) -> None: selection = _extract_scope_selection_from_text(message) has_scope = any(selection.get(key) for key in ("targetSpecies", "referenceSpecies", "comparativeSpecies")) if not has_scope: return params = intent.setdefault("params", {}) params.update(selection) intent["target_species"] = selection.get("targetSpecies", []) intent["reference_species"] = selection.get("referenceSpecies", []) intent["comparative_species"] = selection.get("comparativeSpecies", []) intent["scopeMode"] = selection.get("scopeMode", "target_only") targets = selection.get("targetSpecies") or [] if targets: intent["species"] = targets[0] if len(targets) == 1 else "Multi" if str(intent.get("task_kind") or intent.get("taskType") or "").lower() == "utility": return family = str(intent.get("gene_family") or params.get("geneFamily") or "").strip() if family and targets: intent["prefix"] = f"{intent['species']}.{family}".rstrip(".") params["prefix"] = intent["prefix"] params["geneFamily"] = family def _configured_species_scope(job_id: str) -> Dict[str, list[str]]: """Read the species roles declared by the current job config.""" if not job_id: return {} config_path = INPUT_ROOT / job_id / "config.xlsx" try: import pandas as pd rows = pd.read_excel(config_path).fillna("") except Exception: return {} scope = {"targetSpecies": [], "referenceSpecies": [], "comparativeSpecies": []} role_to_key = { "target": "targetSpecies", "ref": "referenceSpecies", "reference": "referenceSpecies", "seed_reference": "referenceSpecies", "known_family_reference": "referenceSpecies", "comparative": "comparativeSpecies", "compactive": "comparativeSpecies", "outgroup": "comparativeSpecies", } for _, row in rows.iterrows(): species = str(row.get("Species_Code", "") or "").strip() key = role_to_key.get(str(row.get("Species_Role", "") or "").strip().lower()) if species and key and species not in scope[key]: scope[key].append(species) return scope if any(scope.values()) else {} def _confirmed_config_species_scope(job_id: str) -> Dict[str, list[str]]: """Read species roles only when the job has authoritative config confirmation.""" if not job_id: return {} try: from gfa_core.mas.confirmation_snapshot import build_confirmation_snapshot confirmation = build_confirmation_snapshot(PROJECT_ROOT, job_id) except (OSError, ValueError): return {} if not confirmation.get("config_confirmed"): return {} return _configured_species_scope(job_id) def _reconcile_confirmed_gene_family_species( job_id: str, message: str, intent: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Keep confirmed config roles separate from the current run selection.""" config_scope = _confirmed_config_species_scope(job_id) if not config_scope: return {"applicable": False, "conflicts": []} from gfa_core.mas.utils.intent_parser import parse_explicit_species_role_mentions role_keys = { "targetSpecies": "target", "referenceSpecies": "reference", "comparativeSpecies": "comparative", } canonical_config = { key: _canonicalize_species_codes_for_job(job_id, config_scope.get(key) or []) for key in role_keys } config_role_by_species: Dict[str, str] = {} conflicts: list[Dict[str, str]] = [] for key, role in role_keys.items(): for species in canonical_config[key]: existing = config_role_by_species.get(species) if existing and existing != role: conflicts.append({ "code": "config_species_role_conflict", "species": species, "config_role": f"{existing}/{role}", "requested_role": "", }) else: config_role_by_species[species] = role explicit_raw = parse_explicit_species_role_mentions(message or "") explicit = { key: _canonicalize_species_codes_for_job(job_id, explicit_raw.get(key) or []) for key in (*role_keys, "unassigned") } for key, requested_role in role_keys.items(): for species in explicit[key]: config_role = config_role_by_species.get(species) if config_role != requested_role: conflicts.append({ "code": "user_config_species_role_conflict", "species": species, "config_role": config_role or "not_configured", "requested_role": requested_role, }) routed_intent = intent or {} routed_params = routed_intent.get("params") or {} structured_targets = _canonicalize_species_codes_for_job( job_id, _param_list(routed_params.get("targetSpecies") or routed_intent.get("target_species")), ) structured_refs = _canonicalize_species_codes_for_job( job_id, _param_list(routed_params.get("referenceSpecies") or routed_intent.get("reference_species")), ) structured_comps = _canonicalize_species_codes_for_job( job_id, _param_list(routed_params.get("comparativeSpecies") or routed_intent.get("comparative_species")), ) explicit_reference_set = set(explicit["referenceSpecies"]) explicit_assigned_set = set( explicit["targetSpecies"] + explicit["referenceSpecies"] + explicit["comparativeSpecies"] ) for species in structured_targets: config_role = config_role_by_species.get(species) if ( config_role in {None, "reference"} and species not in explicit_reference_set and species not in explicit_assigned_set ): conflicts.append({ "code": "analysis_target_config_role_conflict", "species": species, "config_role": config_role or "not_configured", "requested_role": "analysis_target", }) mentioned = list(dict.fromkeys( species for key in (*role_keys, "unassigned") for species in explicit[key] )) target_candidates = list(dict.fromkeys([ *structured_targets, *explicit["targetSpecies"], *explicit["unassigned"], ])) comparative_candidates = list(dict.fromkeys([ *structured_comps, *structured_targets, *explicit["comparativeSpecies"], *explicit["unassigned"], ])) targets = [ species for species in canonical_config["targetSpecies"] if species in target_candidates ] or list(canonical_config["targetSpecies"]) comparatives = [ species for species in canonical_config["comparativeSpecies"] if species in comparative_candidates ] references = [ species for species in canonical_config["referenceSpecies"] if not structured_refs or species in structured_refs or species in mentioned ] or list(canonical_config["referenceSpecies"]) classification_species = list(dict.fromkeys([*targets, *comparatives])) unique_conflicts: list[Dict[str, str]] = [] seen_conflicts: set[tuple[str, str, str, str]] = set() for conflict in conflicts: identity = ( conflict["code"], conflict["species"], conflict["config_role"], conflict["requested_role"], ) if identity not in seen_conflicts: seen_conflicts.add(identity) unique_conflicts.append(conflict) return { "applicable": True, "config_scope": canonical_config, "target_species": targets, "reference_species": references, "comparative_species": comparatives, "classification_species": classification_species, "conflicts": unique_conflicts, } def _species_role_conflict_message(conflicts: list[Dict[str, str]]) -> str: role_labels = { "target": "目标物种", "reference": "参考物种", "comparative": "比较物种", "analysis_target": "本轮分析对象", "not_configured": "未登记", } details = [] for conflict in conflicts: species = conflict.get("species") or "未知物种" config_role = role_labels.get(conflict.get("config_role") or "", conflict.get("config_role") or "未知") requested_role = role_labels.get(conflict.get("requested_role") or "", conflict.get("requested_role") or "配置内部重复角色") details.append(f"{species}：config.xlsx 为{config_role}，本轮为{requested_role}") return ( "检测到物种角色不一致：" + "；".join(details) + "。为避免按错误范围执行，请发送“打开 config.xlsx”，核对并重新确认物种角色后再继续。" ) def _apply_reconciled_species_to_intent(intent: Dict[str, Any], resolution: Dict[str, Any]) -> None: if not resolution.get("applicable") or resolution.get("conflicts"): return targets = list(resolution.get("target_species") or []) references = list(resolution.get("reference_species") or []) comparatives = list(resolution.get("comparative_species") or []) params = intent.setdefault("params", {}) params.update({ "targetSpecies": targets, "referenceSpecies": references, "comparativeSpecies": comparatives, }) intent["target_species"] = targets intent["reference_species"] = references intent["comparative_species"] = comparatives scope_mode = ( "target_plus_reference_plus_comparative" if references and comparatives else "target_plus_reference" if references else "target_plus_comparative" if comparatives else "target_only" ) params["scopeMode"] = scope_mode intent["scopeMode"] = scope_mode def _is_gene_family_analysis_intent(intent: Optional[Dict[str, Any]]) -> bool: candidate = intent or {} stages = [str(stage or "").strip() for stage in candidate.get("stages") or []] if stages and all(stage.startswith("rnaseq_") or stage.startswith("wgs_") for stage in stages): return False params = candidate.get("params") or {} family = str(candidate.get("gene_family") or params.get("geneFamily") or "").strip() gene_family_stages = { "genome_prepare", "identify", "phylogeny", "motif", "domain", "structure", "combo", "reference_classification", "subfamily_confirm", "promoter", "tfbs", "synteny", "interspecies_synteny", "karyotype", "pep", "ortholog", "kaks", "ppi", "expression", } return bool(family or set(stages) & gene_family_stages) def _normalize_main_agent_analysis_intent( intent: Dict[str, Any], *, job_id: str, context_terms: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """Bind Router output to confirmed job facts without planning stages.""" normalized = dict(intent or {}) params = dict(normalized.get("params") or {}) targets = _param_list(params.get("targetSpecies") or normalized.get("target_species")) references = _param_list( params.get("referenceSpecies") or normalized.get("reference_species") ) comparatives = _param_list( params.get("comparativeSpecies") or normalized.get("comparative_species") ) params.update( { "targetSpecies": targets, "referenceSpecies": references, "comparativeSpecies": comparatives, } ) normalized.update( { "params": params, "target_species": targets, "reference_species": references, "comparative_species": comparatives, } ) config_scope = _confirmed_config_species_scope(job_id) if config_scope: params.update(config_scope) normalized["target_species"] = list(config_scope["targetSpecies"]) normalized["reference_species"] = list(config_scope["referenceSpecies"]) normalized["comparative_species"] = list(config_scope["comparativeSpecies"]) elif context_terms: for intent_key, params_key in ( ("target_species", "targetSpecies"), ("reference_species", "referenceSpecies"), ("comparative_species", "comparativeSpecies"), ): established = _param_list(context_terms.get(intent_key)) if established and not normalized.get(intent_key): normalized[intent_key] = established params[params_key] = established config_family = _infer_family_from_config_or_final(job_id) if config_scope else "" family = str( config_family or (context_terms or {}).get("gene_family") or normalized.get("gene_family") or params.get("geneFamily") or "" ).strip() targets = list(normalized.get("target_species") or []) references = list(normalized.get("reference_species") or []) comparatives = list(normalized.get("comparative_species") or []) scope_mode = ( "target_plus_reference" if references else ("target_plus_comparative" if comparatives else "target_only") ) normalized["scopeMode"] = scope_mode params["scopeMode"] = scope_mode if family: normalized["gene_family"] = family params["geneFamily"] = family if targets: normalized["species"] = targets[0] if len(targets) == 1 else "Multi" route_value = str( normalized.get("rna_route") or params.get("analysisRoute") or "" ).strip() is_rnaseq = bool( route_value in {"whole_genome", "family_member"} or any( str(stage).startswith("rnaseq_") for stage in normalized.get("stages") or () ) ) if is_rnaseq and len(targets) == 1: normalized["prefix"] = targets[0] params["prefix"] = targets[0] elif family and len(targets) == 1: normalized["prefix"] = f"{targets[0]}.{family}" params["prefix"] = normalized["prefix"] return normalized def _infer_family_from_config_or_final(job_id: str) -> str: if not job_id: return "" input_dir = INPUT_ROOT / job_id config_path = input_dir / "config.xlsx" if config_path.exists(): try: import pandas as pd xls = pd.ExcelFile(config_path) for sheet in xls.sheet_names: if sheet.strip().lower() not in {"analysis", "project", "metadata"}: continue df = pd.read_excel(config_path, sheet_name=sheet, header=None) for _, row in df.iterrows(): key = str(row.iloc[0] if len(row) > 0 else "").strip().lower() value = str(row.iloc[1] if len(row) > 1 else "").strip() if key in {"family", "gene_family", "analysis_family", "project_family"} and value and value.lower() not in {"nan", "none"}: return value df = pd.read_excel(config_path) for col in ["Family", "Analysis_Family", "Project_Family", "Gene_Family"]: if col in df.columns: values = [str(v).strip() for v in df[col].tolist() if str(v).strip() and str(v).strip().lower() not in {"nan", "none"}] unique = sorted(set(values)) if len(unique) == 1: return unique[0] except Exception as exc: print(f"[infer_family] config.xlsx read skipped: {exc}") final_dir = WORKSPACE_ROOT / job_id / "final" if final_dir.exists(): patterns = [ "*.ncbi_fetch.protein.fa", "*.pep.fa", "*.pep.fasta", "*.final_members.pep.fa", "*.manifest.json", ] for pattern in patterns: for path in sorted(final_dir.glob(pattern), key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True): if path.suffix == ".json": try: data = json.loads(path.read_text(encoding="utf-8")) family = str(data.get("gene_family") or "").strip() if family: return family prefix = str(data.get("prefix") or "").strip() except Exception: prefix = "" else: prefix = path.name match = re.match(r"^[A-Za-z][A-Za-z0-9_-]*\.([A-Za-z][A-Za-z0-9_-]*)\.", prefix) if match and match.group(1).lower() not in {"ncbi_fetch", "final_members"}: return match.group(1) return "" def _hydrate_goal_terms_from_current_job( terms: Dict[str, Any], job_id: str, ) -> Dict[str, Any]: """Fill missing conversation identity from authoritative current-job evidence. This is context hydration only. It deliberately does not infer or attach analysis stages, so IntentAgentV2 and PlanningAgentV2 remain responsible for analysis recognition and planning. """ hydrated = dict(terms or {}) explicit_term_keys = set(hydrated) if not job_id: return hydrated config_scope = _confirmed_config_species_scope(job_id) for config_key, term_key in ( ("targetSpecies", "target_species"), ("referenceSpecies", "reference_species"), ("comparativeSpecies", "comparative_species"), ): if not hydrated.get(term_key) and config_scope.get(config_key): hydrated[term_key] = list(config_scope[config_key]) if not str(hydrated.get("gene_family") or "").strip(): family = str(_infer_family_from_config_or_final(job_id) or "").strip() if family: hydrated["gene_family"] = family final_dir = WORKSPACE_ROOT / job_id / "final" manifests: list[Dict[str, Any]] = [] if final_dir.is_dir(): for manifest_path in final_dir.glob("*.manifest.json"): try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if manifest.get("status") in {"success", "partial_success"}: manifests.append(manifest) # Older completed jobs may predate confirmation snapshots. Prefer the # confirmed subfamily identity, then classification/identify identity. This # avoids letting technical stages such as PPI add a database taxon to the # biological reference-species role. identity_manifests = manifests for preferred_stage in ("subfamily_confirm", "reference_classification", "identify"): preferred = [ manifest for manifest in manifests if str(manifest.get("stage") or "") == preferred_stage ] if preferred: identity_manifests = preferred break for manifest_key, term_key in ( ("target_species", "target_species"), ("reference_species", "reference_species"), ("comparative_species", "comparative_species"), ): if hydrated.get(term_key): continue observed = { tuple(str(value).strip() for value in manifest.get(manifest_key) or () if str(value).strip()) for manifest in identity_manifests if manifest.get(manifest_key) } if len(observed) == 1: hydrated[term_key] = list(next(iter(observed))) if not str(hydrated.get("rna_route") or "").strip(): foreground_modes = { str(manifest.get("foreground_mode") or "").strip().lower() for manifest in manifests if str(manifest.get("stage") or "").startswith("rnaseq_") and str(manifest.get("foreground_mode") or "").strip().lower() in {"global", "family"} } if foreground_modes == {"global"}: hydrated["rna_route"] = "whole_genome" elif foreground_modes == {"family"}: hydrated["rna_route"] = "family_member" if hydrated.get("rna_route") == "whole_genome": # A job may later contain gene-family outputs as a separate analysis. # Do not let those facts redefine a whole-genome RNA conversation unless # the user/session had already established them explicitly. for key in ("gene_family", "reference_species", "comparative_species"): if key not in explicit_term_keys: hydrated.pop(key, None) return hydrated def _apply_inferred_family_to_intent(intent: Dict[str, Any], job_id: str, message: str) -> None: explicit_family = bool(re.search(r"\b[A-Z]{2,8}\b", str(message or ""))) family = str(intent.get("gene_family") or "").strip() if explicit_family and family: return inferred = _infer_family_from_config_or_final(job_id) if inferred: intent["gene_family"] = inferred species = str(intent.get("species") or "").strip() if not species or species == "Ft": species = "Multi" intent["species"] = species current_prefix = str(intent.get("prefix") or "") looks_auto_default = (current_prefix == f"{intent.get('species', '')}.{family}") if family else False if not current_prefix or current_prefix.endswith(".") or (family and current_prefix == f"Multi.{family}") or looks_auto_default: intent["prefix"] = f"{species}.{inferred}" if species else inferred def _species_codes_for_job(job_id: str) -> set[str]: codes: set[str] = set() if not job_id: return codes config_path = INPUT_ROOT / job_id / "config.xlsx" if config_path.exists(): try: import pandas as pd df = pd.read_excel(config_path) if "Species_Code" in df.columns: for value in df["Species_Code"].dropna().astype(str): code = value.strip() if code and code.lower() not in {"nan", "none", "null", "-"}: codes.add(code) except Exception as exc: print(f"[WARN] Failed to read species codes from {config_path}: {exc}") selection = _load_identify_species_selection(job_id) groups = (selection.get("groups") or {}) if isinstance(selection, dict) else {} for group in groups.values(): for item in group or []: code = str((item or {}).get("species") or "").strip() if code: codes.add(code) return codes def _message_explicitly_mentions_family(message: str, family: str) -> bool: family = str(family or "").strip() if not family: return False return bool(re.search(rf"(? str: """Resolve gene family with config as authority for job-scoped gene-family runs.""" params = params or {} config_family = str(_infer_family_from_config_or_final(job_id) or "").strip() requested = str(requested_family or params.get("geneFamily") or params.get("family") or "").strip() _prefix_species, prefix_family = _prefix_species_family(prefix) if not requested and prefix_family: requested = prefix_family species_codes = {code.lower() for code in _species_codes_for_job(job_id)} requested_is_species = bool(requested and requested.lower() in species_codes) explicit_requested = _message_explicitly_mentions_family(message, requested) if config_family: if not requested or requested_is_species: return config_family if requested != config_family and not explicit_requested: return config_family if requested != config_family and explicit_requested: raise ValueError( f"基因家族冲突：请求为 {requested}，但当前 job config.xlsx 的 Analysis.Family 为 {config_family}。" "请修改 config 或新建 job 后再运行。" ) return config_family if requested_is_species: raise ValueError(f"基因家族解析错误：{requested} 是物种代码，不是 gene family。") return requested or ("SPL" if subfamily else "") def _reference_species_from_config(job_id: str, family: str = "", exclude: Optional[list[str]] = None) -> list[str]: refs: list[str] = [] exclude_set = {str(item).strip() for item in (exclude or []) if str(item).strip()} if not job_id: return refs config_path = INPUT_ROOT / job_id / "config.xlsx" if not config_path.exists(): return refs try: import pandas as pd df = pd.read_excel(config_path) except Exception as exc: print(f"[WARN] Failed to read reference species from {config_path}: {exc}") return refs for _, row in df.fillna("").iterrows(): species = str(row.get("Species_Code", "") or "").strip() if not species or species in exclude_set or species.lower() in {"nan", "none", "null", "-"}: continue role = str(row.get("Species_Role", "") or "").strip().lower() file_scope = str(row.get("File_Scope", "") or "").strip().lower() row_family = str(row.get("Gene_Family", "") or "").strip() is_ref = _config_bool(row.get("Ref", "")) or role in {"ref", "reference", "seed", "seed_reference", "known_family_reference"} is_known_family = file_scope in {"known_family", "family", "seed_family"} or bool(row_family) family_matches = not family or not row_family or row_family == family if is_ref and is_known_family and family_matches and species not in refs: refs.append(species) return refs def _load_identify_species_selection(job_id: str) -> Dict[str, Any]: input_dir = INPUT_ROOT / job_id if job_id else None config_path = input_dir / "config.xlsx" if input_dir else None if not config_path or not config_path.exists(): return {} try: import pandas as pd df = pd.read_excel(config_path) except Exception as exc: print(f"[identify_species_selection] config read skipped: {exc}") return {} def resolve_declared_file(species: str, filename: str) -> Path | None: if not input_dir or not filename: return None name = Path(filename).name candidates = [ input_dir / species / f"{species}.pep.fasta", input_dir / species / f"{species}.pep.fa", input_dir / f"{species}.pep.fasta", input_dir / f"{species}.pep.fa", input_dir / species / f"{species}.protein.fasta", input_dir / species / f"{species}.protein.fa", input_dir / name, input_dir / species / name, ] for candidate in candidates: if candidate.exists(): return candidate return None def likely_protein_fasta(path: Path, max_len: int = 100000) -> tuple[bool, str]: if not path or not path.exists() or path.stat().st_size == 0: return False, "missing" lname = path.name.lower() if lname.endswith(".gz"): return False, "compressed file is not inspected; use standardized uncompressed *.pep.fasta" protein_name_hint = any(token in lname for token in ["pep", "protein", "proteome", ".faa"]) if (not protein_name_hint) and any(token in lname for token in ["chr", "chromosome", "genome", "promoter", "up2000", "pmol.fasta"]): return False, "filename suggests genome/promoter FASTA, not protein" current_id = "" current_len = 0 saw_header = False try: with path.open("r", encoding="utf-8", errors="ignore") as handle: for raw in handle: if raw.startswith(">"): if saw_header: if current_len > max_len: return False, f"{current_id} length {current_len} > {max_len}" return True, "protein-like" saw_header = True current_id = raw[1:].strip().split()[0] current_len = 0 continue if saw_header: current_len += len(raw.strip()) if current_len > max_len: return False, f"{current_id or 'first_record'} length {current_len} > {max_len}" if saw_header and current_len: if current_len > max_len: return False, f"{current_id} length {current_len} > {max_len}" return True, "protein-like" except Exception as exc: return False, f"read failed: {exc}" return False, "no FASTA records" by_species: Dict[str, Dict[str, Any]] = {} for _, row in df.iterrows(): species = str(row.get("Species_Code", "") or "").strip() if not species or species.lower() in {"nan", "none"}: continue role = str(row.get("Species_Role", "") or "").strip().lower() file_type = str(row.get("File_Type", "") or "").strip().lower() filename = str(row.get("Original_File_Name", "") or "").strip() entry = by_species.setdefault(species, {"species": species, "role": role or "unspecified", "files": [], "validProteinFiles": [], "warnings": []}) current_role = entry.get("role", "unspecified") if role == "target": entry["role"] = "target" elif role in {"ref", "reference", "seed_reference", "known_family_reference"} and current_role != "target": entry["role"] = role elif role in {"comparative", "compactive", "outgroup"} and current_role not in {"target", "ref", "seed_reference", "known_family_reference"}: entry["role"] = role elif role and current_role in {"", "unspecified"}: entry["role"] = role if file_type in {"pep", "protein", "all_pep"} and filename: entry["files"].append(filename) resolved = resolve_declared_file(species, filename) if resolved: ok, reason = likely_protein_fasta(resolved) if ok: if resolved.name not in entry["validProteinFiles"]: entry["validProteinFiles"].append(resolved.name) else: warning = f"{resolved.name}: {reason}" if warning not in entry["warnings"]: entry["warnings"].append(warning) # Some jobs standardize generated proteomes as Species.pep.fasta even when # config.xlsx only records genome/GFF rows. Treat that canonical file as the # identify input so explicit comparative species such as Ppy are selectable. for species, entry in by_species.items(): if entry.get("validProteinFiles"): continue for candidate in [ input_dir / f"{species}.pep.fasta" if input_dir else None, input_dir / f"{species}.pep.fa" if input_dir else None, input_dir / species / f"{species}.pep.fasta" if input_dir else None, input_dir / species / f"{species}.pep.fa" if input_dir else None, ]: if not candidate: continue ok, reason = likely_protein_fasta(candidate) if ok: entry["files"].append(candidate.name) entry["validProteinFiles"].append(candidate.name) break if candidate.exists(): warning = f"{candidate.name}: {reason}" if warning not in entry["warnings"]: entry["warnings"].append(warning) groups = {"target": [], "comparative": [], "reference": [], "other": []} for item in sorted(by_species.values(), key=lambda x: x["species"]): role = item.get("role", "unspecified") if role == "target": groups["target"].append(item) elif role in {"comparative", "compactive", "outgroup"}: groups["comparative"].append(item) elif role in {"ref", "reference", "seed_reference", "known_family_reference"}: groups["reference"].append(item) else: groups["other"].append(item) selectable = groups["target"] + groups["comparative"] + groups["other"] if len(selectable) <= 1: return {} default_selected = [item["species"] for item in groups["target"] if item.get("validProteinFiles")] or [ item["species"] for item in selectable if item.get("validProteinFiles") ] return { "type": "identify_species_selection", "message": "请选择本次 identify 要鉴定的物种；reference 仅作为参考来源，不参与目标鉴定。", "groups": groups, "defaultSelected": default_selected, "selectableCount": len(selectable), } def _canonicalize_species_codes_for_job(job_id: str, species_codes: list[str]) -> list[str]: """Map user-facing species aliases such as Ppy/PPy to config species codes such as PPY.""" if not species_codes: return [] selection = _load_identify_species_selection(job_id) if job_id else {} code_map: Dict[str, str] = {} groups = (selection.get("groups") or {}) if isinstance(selection, dict) else {} for group in groups.values(): for item in group or []: code = str((item or {}).get("species") or "").strip() if code: code_map[code.lower()] = code code_map.update(_species_alias_map_for_job(job_id)) canonical: list[str] = [] for raw in species_codes: code = str(raw or "").strip() if not code: continue mapped = code_map.get(code.lower(), code_map.get(_normalize_species_alias_for_job(code), code)) if mapped not in canonical: canonical.append(mapped) return canonical def _canonicalize_species_codes_for_job_strict(job_id: str, species_codes: list[str], role: str = "species") -> list[str]: """Map species names/aliases to config Species_Code and block unresolved names.""" values = [str(item or "").strip() for item in (species_codes or []) if str(item or "").strip()] if not values: return [] known_codes = _species_codes_for_job(job_id) alias_map = _species_alias_map_for_job(job_id) if not known_codes and not alias_map: return _canonicalize_species_codes_for_job(job_id, values) known_by_lower = {code.lower(): code for code in known_codes} canonical: list[str] = [] unresolved: list[str] = [] for raw in values: lower = raw.lower() mapped = known_by_lower.get(lower) if not mapped: mapped = alias_map.get(lower) or alias_map.get(_normalize_species_alias_for_job(raw)) if not mapped: unresolved.append(raw) continue if mapped not in canonical: canonical.append(mapped) if unresolved: raise ValueError( f"{role} 物种名称无法从 config.xlsx 映射到 Species_Code: {', '.join(unresolved)}；" "请在 config 中补充 Species_Code 及拉丁名/别名后再执行。" ) return canonical def _reference_bundle_species_available(job_id: str, species_code: str, require_cds: bool = False) -> tuple[bool, str]: """Return whether a downloaded job-local reference bundle can satisfy PPI/KaKs.""" code = str(species_code or "").strip() if not job_id or not code: return False, "missing job_id/species_code" species_dir = INPUT_ROOT / job_id / code pep_path = species_dir / f"{code}.pep.fasta" cds_path = species_dir / f"{code}.cds.fasta" if not pep_path.exists() or pep_path.stat().st_size <= 0: return False, f"missing PEP: {pep_path}" if require_cds and (not cds_path.exists() or cds_path.stat().st_size <= 0): return False, f"missing CDS: {cds_path}" return True, str(species_dir) def _normalize_species_alias_for_job(value: str) -> str: return re.sub(r"[^A-Za-z0-9]+", " ", str(value or "")).strip().lower() def _species_alias_map_for_job(job_id: str) -> Dict[str, str]: if not job_id: return {} config_path = INPUT_ROOT / job_id / "config.xlsx" if not config_path.exists(): return {} try: from gfa_core.gene_family_scope import read_config_workbook rows, _analysis = read_config_workbook(config_path) except Exception: return {} aliases: Dict[str, str] = {} def add(alias: Any, code: str) -> None: key = _normalize_species_alias_for_job(str(alias or "")) if key and code: aliases[key] = code aliases[str(alias or "").strip().lower()] = code for row in rows: code = str(row.get("Species_Code") or row.get("Species") or row.get("species") or "").strip() if not code: continue add(code, code) for key in ("Species_Name", "Scientific_Name", "Latin_Name", "Species", "species_name", "scientific_name", "latin_name"): add(row.get(key), code) filename = str(row.get("Original_File_Name") or "").strip() if filename: stem = Path(filename).name stem = re.sub(r"\.(?:gz|zip|fa|fasta|faa|fna|ffn|gff3?|gtf|cds|pep|all|chr|toplevel|dna|protein)+$", "", stem, flags=re.I) parts = [part for part in re.split(r"[._-]+", stem) if part] if len(parts) >= 2: add(" ".join(parts[:2]), code) add("_".join(parts[:2]), code) return aliases def _extract_species_codes_from_text_for_job(job_id: str, text: str) -> list[str]: """Extract species codes from text using built-ins plus this job's config codes.""" value = str(text or "") hits: list[tuple[int, str]] = [(idx, code) for idx, code in enumerate(_extract_species_codes_from_text(value))] selection = _load_identify_species_selection(job_id) if job_id else {} groups = (selection.get("groups") or {}) if isinstance(selection, dict) else {} for group in groups.values(): for item in group or []: code = str((item or {}).get("species") or "").strip() if not code: continue pattern = rf"(? Dict[str, list[str]]: """Preserve config roles while resolving species mentioned in a subfamily request.""" confirmed_resolution = _reconcile_confirmed_gene_family_species(job_id, message, intent) if confirmed_resolution.get("applicable") and not confirmed_resolution.get("conflicts"): return { "target_species": list(confirmed_resolution.get("target_species") or []), "reference_species": list(confirmed_resolution.get("reference_species") or []), "comparative_species": list(confirmed_resolution.get("comparative_species") or []), "classification_species": list(confirmed_resolution.get("classification_species") or []), } routed_intent = intent or {} params = routed_intent.get("params") or {} structured_targets = _canonicalize_species_codes_for_job( job_id, _param_list(params.get("targetSpecies") or routed_intent.get("target_species")), ) structured_refs = _canonicalize_species_codes_for_job( job_id, _param_list(params.get("referenceSpecies") or routed_intent.get("reference_species")), ) structured_comps = _canonicalize_species_codes_for_job( job_id, _param_list(params.get("comparativeSpecies") or routed_intent.get("comparative_species")), ) mentioned = _extract_species_codes_from_text_for_job(job_id, message or "") configured = _configured_species_scope(job_id) configured_targets = _canonicalize_species_codes_for_job(job_id, configured.get("targetSpecies") or []) configured_refs = _canonicalize_species_codes_for_job(job_id, configured.get("referenceSpecies") or []) configured_comps = _canonicalize_species_codes_for_job(job_id, configured.get("comparativeSpecies") or []) if configured: target_candidates = [*structured_targets, *mentioned] comp_candidates = [*structured_comps, *mentioned] ref_candidates = [*structured_refs, *mentioned] targets = [species for species in configured_targets if species in target_candidates] comps = [species for species in configured_comps if species in comp_candidates] refs = [species for species in configured_refs if species in ref_candidates] if not refs: refs = list(configured_refs) else: targets = structured_targets or mentioned refs = structured_refs comps = structured_comps classification_species = list(dict.fromkeys([*targets, *comps])) return { "target_species": targets, "reference_species": refs, "comparative_species": comps, "classification_species": classification_species, } def _species_groups_from_selection(selection: Dict[str, Any]) -> tuple[list[str], list[str], list[str]]: groups = selection.get("groups", {}) if isinstance(selection, dict) else {} targets = [str(item.get("species") or "").strip() for item in (groups.get("target") or []) if str(item.get("species") or "").strip()] refs = [str(item.get("species") or "").strip() for item in (groups.get("reference") or []) if str(item.get("species") or "").strip()] comps = [str(item.get("species") or "").strip() for item in (groups.get("comparative") or []) if str(item.get("species") or "").strip()] return targets, refs, comps def _primary_reference_species_from_scope(job_id: str, family: str = "", prefix: str = "") -> list[str]: if not job_id: return [] input_dir = INPUT_ROOT / job_id workspace = WORKSPACE_ROOT / job_id if not input_dir.exists(): return [] try: scope = build_gene_family_scope(input_dir=input_dir, workspace=workspace, prefix=prefix or "", family=family or "") return [str(x) for x in (getattr(scope, "primary_reference_species", None) or scope.reference_species or []) if str(x).strip()] except Exception as exc: print(f"[Scope] Failed to resolve primary reference species from config: {exc}") return [] def _confirmed_subfamily_species(job_id: str, family: str = "") -> list[dict[str, Any]]: roots = [WORKSPACE_ROOT / job_id / "final", INPUT_ROOT / job_id] if job_id else [] rows: list[dict[str, Any]] = [] seen: set[tuple[str, str, str]] = set() for root in roots: if not root.exists(): continue for pattern, source in (("*.confirmed_subfamily.tsv", "confirmed"), ("*.subfamily", "uploaded"), ("*.subfamily.tsv", "uploaded")): for path in sorted(root.glob(pattern)): if path.name.endswith(".review.tsv") or "reference_classification" in path.name or "target-" in path.name: continue parts = path.name.split(".") if len(parts) < 3: continue species, fam = parts[0], parts[1] if family and fam != family: continue if species in {"Target", "Multi"}: continue key = (species, fam, source) if key in seen: continue seen.add(key) try: mtime = path.stat().st_mtime except OSError: mtime = 0 rows.append({"species": species, "family": fam, "path": str(path), "mtime": mtime, "source": source}) rows.sort(key=lambda item: item.get("mtime", 0), reverse=True) return rows def _looks_like_gene_family_workflow_query(message: str) -> bool: text = str(message or "").lower() normalized = re.sub(r"\s+", "", text) if any(token in normalized for token in ("wgs全流程", "vcf全流程", "wgsfullpipeline", "vcffullpipeline")): return False return any(token in text for token in ["基因家族分析", "家族分析", "完整分析", "全流程", "全部阶段", "完整流程"]) def _looks_like_intraspecies_synteny_query(message: str) -> bool: text = str(message or "").lower() return ( any(token in text for token in ["种内共线性", "intraspecies synteny", "mcscanx"]) or ("共线性" in text and not any(token in text for token in ["种间", "物种间", "跨物种", "interspecies"])) ) def _looks_like_tfbs_query(message: str) -> bool: text = str(message or "").lower() return any(token in text for token in [ "tfbs", "fimo", "tf结合", "tf 结合", "转录因子结合", "转录因子结合位点", "启动子结合位点", "结合位点", "binding site", "binding-site", "binding_site", "tf motif", "motif binding" ]) def _looks_like_remaining_query(message: str) -> bool: text = str(message or "").lower() return any(token in text for token in ["剩余", "剩下", "未完成", "还没", "未做", "remaining", "unfinished"]) def _looks_like_all_species_query(message: str) -> bool: text = str(message or "").lower() return any( token in text for token in [ "全物种", "所有物种", "全部物种", "所有基因组", "全部基因组", "全体物种", "每个物种", "各物种", "all species", "all genomes", "every species", "each species", ] ) def _looks_like_all_subfamily_species_query(message: str) -> bool: text = str(message or "").lower() if not any(token in text for token in ["亚家族", "subfamily"]): return False return any( token in text for token in [ "所有的亚家族分类", "全部的亚家族分类", "所有亚家族分类", "全部亚家族分类", "全部的亚家族", "所有的亚家族", "全亚家族分类", ] ) def _remaining_subfamily_species(job_id: str, family: str, selection: Dict[str, Any]) -> list[str]: """Species still needing subfamily prediction/confirmation, excluding references.""" groups = selection.get("groups", {}) if isinstance(selection, dict) else {} candidates: list[str] = [] for group_name in ("target", "comparative", "other"): for item in groups.get(group_name) or []: species = str(item.get("species") or "").strip() if not species or not item.get("validProteinFiles"): continue if species not in candidates: candidates.append(species) remaining: list[str] = [] for species in candidates: status = str((_subfamily_state_for_species(job_id, species, family) or {}).get("status") or "") if status in {"missing_identify", "not_classified", "predicted_unconfirmed", "confirmed_mismatch"}: remaining.append(species) return remaining def _subfamily_group_species(selection: Dict[str, Any], group_names: list[str]) -> list[str]: groups = selection.get("groups", {}) if isinstance(selection, dict) else {} species_list: list[str] = [] for group_name in group_names: for item in groups.get(group_name) or []: species = str((item or {}).get("species") or "").strip() if not species or not (item or {}).get("validProteinFiles"): continue if species not in species_list: species_list.append(species) return species_list def _remaining_from_species(job_id: str, family: str, species_list: list[str]) -> list[str]: remaining: list[str] = [] for species in species_list or []: status = str((_subfamily_state_for_species(job_id, species, family) or {}).get("status") or "") if status in {"missing_identify", "not_classified", "predicted_unconfirmed", "confirmed_mismatch"}: remaining.append(species) return remaining def _subfamily_role_flags(message: str) -> tuple[bool, bool, bool]: text = str(message or "").lower() remaining = _looks_like_remaining_query(text) target_role = any(token in text for token in ["目标物种", "目标基因组", "target species", "target genomes", "targets"]) comparative_role = any(token in text for token in ["比较物种", "比较基因组", "comparative", "comparison species", "comparative species", "outgroup", "外群"]) return remaining, target_role, comparative_role def _select_subfamily_classification_species( job_id: str, family: str, selection: Dict[str, Any], message: str, current_targets: Optional[list[str]] = None, ) -> tuple[list[str], str]: explicit = _extract_species_codes_from_text_for_job(job_id, message or "") if explicit: return explicit, "explicit_species" remaining, target_role, comparative_role = _subfamily_role_flags(message) all_species = _looks_like_all_species_query(message) or _looks_like_all_subfamily_species_query(message) target_species = _subfamily_group_species(selection, ["target"]) comparative_species = _subfamily_group_species(selection, ["comparative", "other"]) all_classifiable = [] for species in [*target_species, *comparative_species]: if species and species not in all_classifiable: all_classifiable.append(species) if remaining and target_role: return _remaining_from_species(job_id, family, target_species), "remaining_target_role" if remaining and comparative_role: return _remaining_from_species(job_id, family, comparative_species), "remaining_comparative_role" if remaining: return _remaining_from_species(job_id, family, all_classifiable), "remaining_all" if all_species and target_role: return target_species, "all_target_role" if all_species and comparative_role: return comparative_species, "all_comparative_role" if all_species: return all_classifiable, "all_classifiable" if comparative_role: return comparative_species, "comparative_role_all" if target_role: return target_species, "target_role_all" canonical_current = _canonicalize_species_codes_for_job(job_id, current_targets or []) if canonical_current: return canonical_current, "default_current" default_selected = _canonicalize_species_codes_for_job(job_id, selection.get("defaultSelected", []) if isinstance(selection, dict) else []) if default_selected: return default_selected, "default_selection" return target_species, "default_target_role" def _subfamily_scope_mode_for_roles(refs: list[str], comps: list[str]) -> str: if refs and comps: return "target_plus_reference_plus_comparative" if refs: return "target_plus_reference" if comps: return "target_plus_comparative" return "target_only" def _split_subfamily_selected_species_by_config_roles( job_id: str, selected_species: list[str], identify_selection: Dict[str, Any], fallback_targets: Optional[list[str]] = None, ) -> tuple[list[str], list[str], list[str]]: selected = _canonicalize_species_codes_for_job(job_id, selected_species or []) selected_set = set(selected) target_role = _subfamily_group_species(identify_selection or {}, ["target"]) comparative_role = _subfamily_group_species(identify_selection or {}, ["comparative", "other"]) if not target_role and not comparative_role: configured = _configured_species_scope(job_id) target_role = _canonicalize_species_codes_for_job(job_id, configured.get("targetSpecies") or []) comparative_role = _canonicalize_species_codes_for_job(job_id, configured.get("comparativeSpecies") or []) targets = [sp for sp in target_role if sp in selected_set] comps = [sp for sp in comparative_role if sp in selected_set and sp not in targets] if not targets: fallback = _canonicalize_species_codes_for_job(job_id, fallback_targets or []) targets = [sp for sp in fallback if sp in selected_set and sp not in comps] if not targets: targets = [sp for sp in selected if sp not in comps] return selected, targets, comps def _identify_selection_with_default(selection: Dict[str, Any], selected_species: list[str], message: str) -> Dict[str, Any]: synced = dict(selection or {}) synced["defaultSelected"] = list(selected_species or []) if selected_species: synced["message"] = message or "已根据本次亚家族分类请求确定待分类物种；reference 仅作为参考来源。" return synced def _sync_subfamily_plan_species( *, job_id: str, family: str, message: str, selected_species: list[str], reference_species: list[str], identify_selection: Dict[str, Any], plan_params: Dict[str, Any], intent: Dict[str, Any], stages_detail: list, ) -> Dict[str, Any]: selected = _canonicalize_species_codes_for_job(job_id, selected_species or []) refs = _canonicalize_species_codes_for_job(job_id, reference_species or []) selected, role_targets, role_comps = _split_subfamily_selected_species_by_config_roles( job_id, selected, identify_selection or {}, plan_params.get("targetSpecies") or intent.get("target_species") or [], ) existing_mode = str(plan_params.get("analysisMode") or plan_params.get("analysis_mode") or "").strip() analysis_mode = existing_mode if existing_mode in {"per_species", "joint_multi_species"} else ("joint_multi_species" if len(selected) > 1 else "per_species") confirm_prefix_by_species = {sp: f"{sp}.{family}" for sp in selected if sp and family} subfamily_state = _subfamily_state_for_targets(job_id, selected, family) if selected and family else {} synced_selection = _identify_selection_with_default(identify_selection or {}, selected, message) plan_params["targetSpecies"] = role_targets plan_params["referenceSpecies"] = refs plan_params["comparativeSpecies"] = role_comps plan_params["scopeMode"] = _subfamily_scope_mode_for_roles(refs, role_comps) plan_params["identifySpeciesSelection"] = synced_selection plan_params["analysisMode"] = analysis_mode plan_params["speciesQueue"] = selected plan_params["confirmPrefixBySpecies"] = confirm_prefix_by_species plan_params["subfamilyState"] = subfamily_state intent["target_species"] = role_targets intent["reference_species"] = refs intent["comparative_species"] = role_comps intent["scopeMode"] = plan_params["scopeMode"] intent.setdefault("params", {}).update({ "targetSpecies": role_targets, "referenceSpecies": refs, "comparativeSpecies": role_comps, "scopeMode": plan_params["scopeMode"], "identifySpeciesSelection": synced_selection, "geneFamily": family, "analysisMode": analysis_mode, "speciesQueue": selected, "confirmPrefixBySpecies": confirm_prefix_by_species, "subfamilyState": subfamily_state, }) intent["species"] = selected[0] if len(selected) == 1 else ("Multi" if selected else "") for detail in stages_detail or []: if not isinstance(detail, dict): continue stage_id = str(detail.get("id") or detail.get("name") or "") detail_params = detail.setdefault("params", {}) detail_params["targetSpecies"] = list(selected) detail_params["identifySpeciesSelection"] = synced_selection detail_params["geneFamily"] = family detail_params["analysisMode"] = analysis_mode detail_params["speciesQueue"] = list(selected) detail_params["confirmPrefixBySpecies"] = dict(confirm_prefix_by_species) detail_params["subfamilyState"] = subfamily_state if stage_id == "phylogeny" and stage_id in methodology_target_stage_ids: detail["name"] = METHODOLOGY_DISPLAY_STAGE_LABELS["phylogeny"] if stage_id == "interspecies_synteny": detail["name"] = METHODOLOGY_DISPLAY_STAGE_LABELS["interspecies_synteny"] detail["description"] = METHODOLOGY_DISPLAY_STAGE_LABELS["interspecies_synteny"] if stage_id == "identify": detail_params["scopeMode"] = "target_only" detail_params["targetSpecies"] = list(selected) detail_params["referenceSpecies"] = [] detail_params["comparativeSpecies"] = [] detail_params.pop("analysisRoute", None) detail_params.pop("analysis_route", None) elif stage_id in {"phylogeny", "reference_classification", "motif", "domain", "structure", "combo"}: detail_params["targetSpecies"] = list(role_targets) detail_params["scopeMode"] = _subfamily_scope_mode_for_roles(refs, role_comps) detail_params["referenceSpecies"] = list(refs) detail_params["comparativeSpecies"] = list(role_comps) detail_params["analysisRoute"] = "subfamily_prediction" if stage_id == "reference_classification": detail_params["subfamilyPipeline"] = True detail_params["perTargetEvidenceOnly"] = True return synced_selection def _looks_like_phylogeny_query(message: str) -> bool: text = str(message or "").lower() return any(token in text for token in ["进化树", "系统发育", "phylogeny", "tree", "建树"]) def _downstream_gene_family_stages_for_context(message: str) -> list[str]: text = str(message or "").lower() if _looks_like_intraspecies_synteny_query(text): return ["synteny"] if _looks_like_tfbs_query(text) and not _looks_like_gene_family_workflow_query(text): return ["tfbs"] if _looks_like_phylogeny_query(text) and not _looks_like_gene_family_workflow_query(text): return ["phylogeny"] if _looks_like_gene_family_workflow_query(text): stages = [ "phylogeny", "motif", "domain", "structure", "combo", "promoter", "tfbs", "synteny", "interspecies_synteny", "karyotype", "pep", "ortholog", "kaks", "ppi", ] if any(token in text for token in ["表达", "expression", "rna"]): stages.append("expression") return stages stages = ["phylogeny", "motif", "domain", "structure", "promoter", "tfbs", "pep", "synteny", "karyotype"] if any(token in text for token in ["种间", "比较物种", "interspecies"]): stages.append("interspecies_synteny") if any(token in text for token in ["kaks", "ka/ks", "选择压力"]): stages.append("ortholog") stages.append("kaks") if any(token in text for token in ["ppi", "互作"]): stages.append("ppi") if any(token in text for token in ["表达", "expression", "rna"]): stages.append("expression") return list(dict.fromkeys(stages)) def _resolve_contextual_gene_family_request( job_id: str, message: str, request_prefix: str = "", requested_stages: list[str] | None = None, request_intent: Dict[str, Any] | None = None, ) -> Dict[str, Any]: """Resolve post-subfamily/default phylogeny context from real project state. This is deliberately rule-based: it decides scope/route boundaries from files and config, while LLM/intent parsing may still supply user wording. """ text = str(message or "") structured_stages = list(dict.fromkeys( str(stage).strip() for stage in (requested_stages or []) if str(stage).strip() )) contextual_stage_request = bool( set(structured_stages) & {"phylogeny", "motif", "domain", "structure", "combo"} ) routed_intent = request_intent or {} routed_params = routed_intent.get("params") or {} structured_targets = _canonicalize_species_codes_for_job( job_id, _param_list(routed_params.get("targetSpecies") or routed_intent.get("target_species")), ) structured_refs = _canonicalize_species_codes_for_job( job_id, _param_list(routed_params.get("referenceSpecies") or routed_intent.get("reference_species")), ) structured_comps = _canonicalize_species_codes_for_job( job_id, _param_list(routed_params.get("comparativeSpecies") or routed_intent.get("comparative_species")), ) if not ( _looks_like_phylogeny_query(text) or _looks_like_gene_family_workflow_query(text) or _looks_like_intraspecies_synteny_query(text) or _looks_like_tfbs_query(text) or contextual_stage_request ): return {} role_resolution = _reconcile_confirmed_gene_family_species( job_id, text, routed_intent, ) selection = _load_identify_species_selection(job_id) if role_resolution.get("applicable") and not role_resolution.get("conflicts"): confirmed_scope = role_resolution.get("config_scope") or {} config_targets = list(confirmed_scope.get("targetSpecies") or []) config_refs = list(confirmed_scope.get("referenceSpecies") or []) config_comps = list(confirmed_scope.get("comparativeSpecies") or []) else: config_targets, config_refs, config_comps = _species_groups_from_selection(selection) family = _nonempty_prefix(_gene_family_from_config(INPUT_ROOT / job_id / "config.xlsx")) if job_id else "" if not family and request_prefix and "." in request_prefix: family = request_prefix.split(".", 1)[1] confirmed = _confirmed_subfamily_species(job_id, family) confirmed_species = [item["species"] for item in confirmed if item.get("source") == "confirmed"] mentioned = _extract_species_codes_from_text_for_job(job_id, text) prefix_species = "" if request_prefix and "." in request_prefix: prefix_species = request_prefix.split(".", 1)[0].strip() if prefix_species in {"Target", "Multi", "Ref", "Reference"}: prefix_species = "" is_all_species_request = _looks_like_all_species_query(text) or _looks_like_all_subfamily_species_query(text) is_intraspecies_synteny_request = _looks_like_intraspecies_synteny_query(text) if role_resolution.get("applicable") and not role_resolution.get("conflicts"): targets = list(role_resolution.get("target_species") or []) refs = list(role_resolution.get("reference_species") or []) comps = list(role_resolution.get("comparative_species") or []) else: mentioned_targets = [sp for sp in mentioned if sp in config_targets or sp == prefix_species or sp in confirmed_species] mentioned_refs = [sp for sp in mentioned if sp in config_refs] mentioned_comps = [sp for sp in mentioned if sp in config_comps] unassigned_mentions = [sp for sp in mentioned if sp not in mentioned_targets and sp not in mentioned_refs and sp not in mentioned_comps] for sp in unassigned_mentions: if sp not in mentioned_targets: mentioned_targets.append(sp) if mentioned: targets = mentioned_targets or [sp for sp in mentioned if sp not in mentioned_refs and sp not in mentioned_comps] elif structured_targets: targets = list(structured_targets) elif is_all_species_request and config_targets: targets = list(config_targets) elif is_intraspecies_synteny_request and config_targets: targets = list(config_targets) elif prefix_species: targets = [prefix_species] elif config_targets: targets = [config_targets[0]] if len(config_targets) == 1 else list(config_targets) elif confirmed_species: targets = [confirmed_species[0]] else: targets = [] refs = mentioned_refs or list(structured_refs) or (list(config_refs) if is_all_species_request else []) comps = mentioned_comps or list(structured_comps) or (list(config_comps) if is_all_species_request else []) if refs and comps: scope_mode = "target_plus_reference_plus_comparative" elif refs: scope_mode = "target_plus_reference" elif comps: scope_mode = "target_plus_comparative" else: scope_mode = "target_only" if not family or not targets: return {} stages = structured_stages or _downstream_gene_family_stages_for_context(text) subfamily_label_stages = {"phylogeny", "motif", "domain", "structure", "combo"} needs_subfamily_labels_for_stage = any(stage in subfamily_label_stages for stage in stages) prefix = f"{targets[0]}.{family}" if len(targets) == 1 else f"Target.{family}" classifiable_species_for_labels = [] for sp in [*targets, *comps]: if sp and sp not in classifiable_species_for_labels: classifiable_species_for_labels.append(sp) subfamily_state = _subfamily_state_for_targets(job_id, classifiable_species_for_labels, family) target_confirmed = { sp: (data.get("confirmed_file") or data.get("uploaded_subfamily_file") or data.get("label_file") or "") for sp, data in (subfamily_state.get("by_species") or {}).items() } all_targets_confirmed = bool(classifiable_species_for_labels) and bool(subfamily_state.get("all_confirmed")) if role_resolution.get("applicable") and not role_resolution.get("conflicts"): explicit_custom_scope = ( targets != config_targets or refs != config_refs or comps != config_comps ) else: explicit_custom_scope = bool(refs or comps or len(targets) > 1) skip_subfamily_labels = _skip_subfamily_label_query(text) if skip_subfamily_labels: context_route = "plain_phylogeny" if _looks_like_phylogeny_query(text) else "gene_family_without_subfamily" elif all_targets_confirmed and not explicit_custom_scope: context_route = "post_subfamily_gene_family_analysis" elif explicit_custom_scope: context_route = "custom_phylogeny_scope" else: context_route = "gene_family_default_scope" wants_subfamily_context = ( needs_subfamily_labels_for_stage and not skip_subfamily_labels and ( context_route == "post_subfamily_gene_family_analysis" or explicit_custom_scope or _requires_confirmed_subfamily_query(text) or _looks_like_gene_family_workflow_query(text) or _looks_like_phylogeny_query(text) or contextual_stage_request ) ) requires_confirmed = wants_subfamily_context if wants_subfamily_context and not all_targets_confirmed and not refs: refs = list(config_refs) or _primary_reference_species_from_scope(job_id, family, prefix) if refs: if refs and comps: scope_mode = "target_plus_reference_plus_comparative" elif refs: scope_mode = "target_plus_reference" elif comps: scope_mode = "target_plus_comparative" else: scope_mode = "target_only" context_gate = _validate_gene_family_stage_context( job_id=job_id, stage=stages[0] if len(stages) == 1 else "gene_family_workflow", family=family, target_species=targets, comparative_species=comps, requires_confirmed_subfamily=requires_confirmed and all_targets_confirmed, ) context_gate["subfamily_state"] = subfamily_state if wants_subfamily_context and not all_targets_confirmed: context_gate["valid"] = False context_gate.setdefault("issues", []).extend(subfamily_state.get("issues") or []) context_gate["needs_subfamily_decision"] = True return { "family": family, "prefix": prefix, "stages": stages, "targetSpecies": targets, "referenceSpecies": refs, "comparativeSpecies": comps, "scopeMode": scope_mode, "confirmed_subfamily_available": any(bool(path) for path in target_confirmed.values()), "confirmed_subfamily_species": [sp for sp, path in target_confirmed.items() if path], "confirmed_subfamily_files": target_confirmed, "requiresConfirmedSubfamily": requires_confirmed, "contextGate": context_gate, "context_route": context_route, "needs_species_confirmation": len(targets) > 1 and not has_explicit_scope and _looks_like_phylogeny_query(text) and not _looks_like_gene_family_workflow_query(text), "subfamily_label_source": "confirmed_or_uploaded_subfamily" if (needs_subfamily_labels_for_stage and all_targets_confirmed) else ("none" if (skip_subfamily_labels or not needs_subfamily_labels_for_stage) else "auto"), "skipSubfamilyLabels": skip_subfamily_labels or not needs_subfamily_labels_for_stage, "subfamilyState": subfamily_state, } def _contextual_prerequisite_stages(contextual_request: Dict[str, Any]) -> tuple[list[str], list[str]]: """Return recoverable prerequisite stages and species that need manual label confirmation.""" skip_subfamily_labels = bool(contextual_request.get("skipSubfamilyLabels")) sub_state = contextual_request.get("subfamilyState") or {} by_species = sub_state.get("by_species") or {} stages: list[str] = [] confirm_species: list[str] = [] def add_stage(stage_id: str) -> None: if stage_id not in stages: stages.append(stage_id) for species, data in by_species.items(): status = str((data or {}).get("status") or "") if status == "missing_identify": add_stage("identify") if not skip_subfamily_labels: add_stage("reference_classification") if species not in confirm_species: confirm_species.append(species) elif not skip_subfamily_labels and status == "not_classified": add_stage("reference_classification") if species not in confirm_species: confirm_species.append(species) elif not skip_subfamily_labels and status in {"predicted_unconfirmed", "confirmed_mismatch"}: if status == "confirmed_mismatch": add_stage("reference_classification") if species not in confirm_species: confirm_species.append(species) if confirm_species: add_stage("subfamily_confirm") return stages, confirm_species def _target_family_member_files(job_id: str, species: str, family: str) -> Dict[str, str]: return resolve_target_member_files(WORKSPACE_ROOT, job_id, species, family) def _confirmed_subfamily_file(job_id: str, species: str, family: str) -> str: return resolve_confirmed_subfamily_file(WORKSPACE_ROOT, INPUT_ROOT, job_id, species, family) def _reference_classification_prediction_file(job_id: str, species: str, family: str) -> str: return resolve_prediction_subfamily_file(WORKSPACE_ROOT, job_id, species, family) def _read_first_column_ids(path: str) -> set[str]: ids: set[str] = set() if not path: return ids try: with open(path, "r", encoding="utf-8") as handle: first = True for raw in handle: line = raw.strip() if not line or line.startswith("#"): continue parts = line.split("\t") if "\t" in line else line.split() if not parts: continue token = parts[0].strip() if first and token.lower() in {"gene_id", "gene", "id"}: first = False continue first = False if token: ids.add(token) except Exception: return set() return ids def _skip_subfamily_label_query(message: str) -> bool: text = str(message or "").lower() return any(token in text for token in [ "不需要亚家族", "不用亚家族", "不标注亚家族", "跳过亚家族", "普通进化树", "只构树", "只按序列构树", "无亚家族", "without subfamily", "skip subfamily", "plain phylogeny", ]) def _subfamily_state_for_species(job_id: str, species: str, family: str) -> dict[str, Any]: return resolve_subfamily_state_for_species(WORKSPACE_ROOT, INPUT_ROOT, job_id, species, family, "target") def _subfamily_state_for_targets(job_id: str, species_list: list[str], family: str) -> dict[str, Any]: return subfamily_state_for_scope( WORKSPACE_ROOT, INPUT_ROOT, job_id, family, target_species=species_list or [], reference_species=[], comparative_species=[], ) def _requires_confirmed_subfamily_query(message: str) -> bool: text = str(message or "").lower() return any(token in text for token in [ "确认后的亚家族", "确认亚家族", "确认后的分类", "新的亚家族", "人工确认", "confirmed subfamily", "基于亚家族", "基于.*亚家族", ]) def _validate_gene_family_stage_context( job_id: str, stage: str, family: str, target_species: list[str], comparative_species: list[str] | None = None, requires_confirmed_subfamily: bool = False, ) -> Dict[str, Any]: """Deterministic context gate shared by planning/execution. identify membership is a hard dependency for downstream target/comparative analyses. confirmed_subfamily is a contextual dependency only when the request explicitly uses post-confirmation labels. """ comparative_species = comparative_species or [] species_to_check: list[str] = [] for sp in [*(target_species or []), *comparative_species]: sp = str(sp or "").strip() if sp and sp not in species_to_check: species_to_check.append(sp) issues: list[dict[str, Any]] = [] evidence: dict[str, Any] = {"family_members": {}, "confirmed_subfamily": {}} if stage not in {"genome_prepare", "genome_prepare_v2", "identify"}: for sp in species_to_check: member_files = _target_family_member_files(job_id, sp, family) evidence["family_members"][sp] = member_files if not member_files.get("pep"): issues.append({ "severity": "error", "category": "context", "code": "missing_target_family_members", "role": "family_members_pep", "species": sp, "message": f"{stage} 缺少 {sp} 的基因家族成员蛋白文件。", "suggestion": f"请先完成 {sp}/{family} 的 identify，生成 Target.{sp}.{family}.final_members.pep.fa。", "auto_fixable": True, "fix_action": "run_identify", }) if requires_confirmed_subfamily: label_state = subfamily_state_for_scope( WORKSPACE_ROOT, INPUT_ROOT, job_id, family, target_species=target_species or [], reference_species=[], comparative_species=comparative_species or [], ) evidence["confirmed_subfamily"] = { sp: (data.get("confirmed_file") or data.get("uploaded_subfamily_file") or data.get("label_file") or "") for sp, data in (label_state.get("by_species") or {}).items() } for sp, data in (label_state.get("by_species") or {}).items(): if data.get("status") not in {"confirmed_valid", "uploaded_confirmed"}: issues.append({ "severity": "error", "category": "context", "code": "missing_confirmed_or_uploaded_subfamily", "role": "confirmed_subfamily", "species": sp, "message": f"请求要求使用亚家族标签，但缺少 {sp}.{family}.confirmed_subfamily.tsv 或 {sp}.{family}.subfamily。", "suggestion": "请先完成参考亚家族分类并保存确认标签，或上传目标物种亚家族文件。", "auto_fixable": False, "fix_action": "run_reference_classification_then_subfamily_confirm", "subfamily_status": data.get("status"), }) return { "valid": not any(issue.get("severity") == "error" for issue in issues), "issues": issues, "evidence": evidence, "requires_confirmed_subfamily": bool(requires_confirmed_subfamily), } def _require_gene_family_target_species( stage: str, target_species: list[str], ) -> None: if stage not in GENE_FAMILY_SCOPE_STAGES: return selected = [ str(item or "").strip() for item in (target_species or []) if str(item or "").strip() ] if not selected: raise HTTPException( status_code=409, detail="基因家族分析尚未确认目标物种；请先核对并确认 config.xlsx。", ) def _stage_detail_defaults(stage_id: str) -> Dict[str, Any]: names = { "identify": "基因家族鉴定", "phylogeny": "系统发育分析", "motif": "Motif分析", "domain": "结构域分析", "structure": "基因结构分析", "promoter": "启动子元件预测", "tfbs": "TF结合位点预测", "pep": "蛋白理化性质与亚细胞定位", "synteny": "种内共线性分析", "interspecies_synteny": "种间共线性分析", "karyotype": "核型/染色体分布分析", "ortholog": "直系同源基因鉴定", "kaks": "Ka/Ks分析", "ppi": "PPI分析", "expression": "表达分析", "reference_classification": "参考亚家族分类", "combo": "组合图", } estimates = { "phylogeny": "30分钟", "motif": "30分钟", "domain": "20分钟", "structure": "20分钟", "promoter": "20分钟", "tfbs": "30分钟", "pep": "15分钟", "synteny": "30分钟", "interspecies_synteny": "40分钟", "karyotype": "20分钟", "combo": "10分钟", "ortholog": "15分钟", "kaks": "40分钟", "ppi": "20分钟", "expression": "30分钟", } deps = { "identify": [], "phylogeny": ["identify"], "motif": ["identify", "phylogeny"], "domain": ["identify", "phylogeny"], "structure": ["identify", "phylogeny"], "combo": ["motif", "domain", "structure"], "reference_classification": ["combo"], "promoter": ["identify", "phylogeny"], "tfbs": ["genome_prepare", "identify"], "pep": ["identify"], "synteny": ["identify"], "interspecies_synteny": ["identify"], "karyotype": ["identify", "synteny"], "ortholog": ["identify"], "kaks": ["identify", "ortholog"], "ppi": ["identify"], "expression": ["identify"], } return { "id": stage_id, "name": names.get(stage_id, stage_id), "description": names.get(stage_id, stage_id), "status": "pending", "dependencies": deps.get(stage_id, ["identify"] if stage_id != "identify" else []), "estimated_time": estimates.get(stage_id, "未知"), "params": {}, } FULL_GENE_FAMILY_WORKFLOW_STAGES = [ "phylogeny", "motif", "domain", "structure", "combo", "promoter", "tfbs", "synteny", "interspecies_synteny", "karyotype", "pep", "ortholog", "kaks", "ppi", ] FORMAL_GENE_FAMILY_CORE_STAGES = {"phylogeny", "motif", "domain", "structure", "combo"} FULL_GENE_FAMILY_TARGET_ONLY_STAGES = {"promoter", "tfbs", "synteny", "karyotype", "pep"} FULL_GENE_FAMILY_REFERENCE_STAGES = FORMAL_GENE_FAMILY_CORE_STAGES | { "interspecies_synteny", "ortholog", "kaks", "ppi", } SCOPE_IDENTITY_PARAM_KEYS = ( "scopeFile", "scope_file", "scopeId", "scope_id", "stageContext", "stage_context", "memberSetId", "member_set_id", "output_prefix", "legacy_output_prefix", ) def _formal_gene_family_route(params: Dict[str, Any]) -> str: refs = _param_list(params.get("referenceSpecies") or params.get("reference_species") or params.get("refSpeciesList")) comps = _param_list(params.get("comparativeSpecies") or params.get("comparative_species")) scope_mode = str(params.get("scopeMode") or params.get("scope_mode") or "").strip() if refs or comps or scope_mode in {"target_plus_reference", "target_plus_reference_plus_comparative", "target_plus_comparative"}: return "multi_species" return "single_species" def _single_target_family_prefix(plan_prefix: str, params: Dict[str, Any]) -> str: targets = _param_list(params.get("targetSpecies") or params.get("target_species")) family = str( params.get("geneFamily") or params.get("gene_family") or (plan_prefix.split(".")[-1] if "." in str(plan_prefix or "") else "") ).strip() if len(targets) == 1 and family: return f"{targets[0]}.{family}" return str(plan_prefix or params.get("prefix") or "").strip() def _drop_scope_identity_params(params: Dict[str, Any]) -> None: for stale_key in SCOPE_IDENTITY_PARAM_KEYS: params.pop(stale_key, None) def _normalize_full_workflow_stage_scope( stage_id: str, detail_params: Dict[str, Any], plan_params: Dict[str, Any], plan_prefix: str, route: str, ) -> None: """Keep mixed full-workflow stages from sharing one stale scope identity.""" targets = _param_list(detail_params.get("targetSpecies") or plan_params.get("targetSpecies")) refs = _param_list( plan_params.get("referenceSpecies") or plan_params.get("reference_species") or plan_params.get("refSpeciesList") or detail_params.get("referenceSpecies") or detail_params.get("reference_species") or detail_params.get("refSpeciesList") ) comps = _param_list( plan_params.get("comparativeSpecies") or plan_params.get("comparative_species") or detail_params.get("comparativeSpecies") or detail_params.get("comparative_species") ) if targets: detail_params["targetSpecies"] = targets target_prefix = _single_target_family_prefix(plan_prefix, {**plan_params, **detail_params}) if target_prefix: detail_params["prefix"] = target_prefix if stage_id in FULL_GENE_FAMILY_TARGET_ONLY_STAGES: detail_params["scopeMode"] = "target_only" detail_params["referenceSpecies"] = [] detail_params["refSpeciesList"] = [] detail_params["comparativeSpecies"] = [] detail_params["analysisRoute"] = "single_species" detail_params.pop("analysis_route", None) _drop_scope_identity_params(detail_params) return if stage_id in FULL_GENE_FAMILY_REFERENCE_STAGES: if refs and comps: detail_params["scopeMode"] = "target_plus_reference_plus_comparative" elif refs: detail_params["scopeMode"] = "target_plus_reference" elif comps: detail_params["scopeMode"] = "target_plus_comparative" else: detail_params["scopeMode"] = "target_only" detail_params["referenceSpecies"] = refs detail_params["refSpeciesList"] = refs detail_params["comparativeSpecies"] = comps detail_params["analysisRoute"] = route detail_params.pop("analysis_route", None) _drop_scope_identity_params(detail_params) def _normalize_full_gene_family_workflow_plan( *, message: str, stage_ids: list[str], stages_detail: list[Dict[str, Any]], plan_params: Dict[str, Any], plan_prefix: str, ) -> tuple[list[str], list[Dict[str, Any]]]: if not _looks_like_gene_family_workflow_query(message or ""): return stage_ids, stages_detail desired = list(FULL_GENE_FAMILY_WORKFLOW_STAGES) if "expression" in stage_ids or any(token in (message or "").lower() for token in ["表达", "expression", "rna"]): desired.append("expression") shared_refs = _param_list(plan_params.get("referenceSpecies") or plan_params.get("reference_species") or plan_params.get("refSpeciesList")) shared_comps = _param_list(plan_params.get("comparativeSpecies") or plan_params.get("comparative_species")) shared_scope = str(plan_params.get("scopeMode") or plan_params.get("scope_mode") or "").strip() if shared_refs and shared_scope in {"", "target_only"}: plan_params["scopeMode"] = "target_plus_reference" elif shared_comps and shared_scope in {"", "target_only"}: plan_params["scopeMode"] = "target_plus_comparative" ordered: list[str] = [] if "identify" in stage_ids: ordered.append("identify") for sid in desired: if sid not in ordered: ordered.append(sid) for sid in stage_ids: if sid and sid not in ordered: ordered.append(sid) detail_by_stage: dict[str, Dict[str, Any]] = {} for detail in stages_detail or []: if not isinstance(detail, dict): continue sid = str(detail.get("id") or detail.get("name") or "").strip() if sid and sid not in detail_by_stage: detail_by_stage[sid] = detail route = _formal_gene_family_route(plan_params or {}) normalized_details: list[Dict[str, Any]] = [] for order, sid in enumerate(ordered, start=1): detail = dict(detail_by_stage.get(sid) or _stage_detail_defaults(sid)) detail["id"] = sid detail["order"] = order detail_params = {**(plan_params or {}), **(detail.get("params") or {})} if plan_prefix and not detail_params.get("prefix"): detail_params["prefix"] = plan_prefix _normalize_full_workflow_stage_scope(sid, detail_params, plan_params or {}, plan_prefix, route) if sid in FORMAL_GENE_FAMILY_CORE_STAGES: if sid in {"motif", "domain", "structure"}: detail["dependencies"] = ["phylogeny"] elif sid == "combo": detail["dependencies"] = ["phylogeny", "motif", "domain", "structure"] detail_params["comboPanels"] = ["motif", "domain", "structure"] detail_params["comboPanelSource"] = "regular_gene_family" if sid in GENE_FAMILY_SCOPE_STAGES: detail["params"] = enrich_stage_params_with_context(sid, plan_prefix, detail_params) else: detail["params"] = detail_params normalized_details.append(detail) return ordered, normalized_details def _identify_completed_for_scope(job_id: str, family: str, targets: list[str]) -> bool: if not job_id or not family: return False final_dir = WORKSPACE_ROOT / job_id / "final" if (final_dir / f"Target.{family}.identify.manifest.json").exists(): if not targets: return True for species in targets: if (final_dir / f"Target.{species}.{family}.final_members.pep.fa").exists(): return True if (WORKSPACE_ROOT / job_id / "identify" / species / "final" / f"{species}.final_gfmembers.pep.fa").exists(): return True return False def _read_ncbi_mapping_rows(mapping_file: Path) -> list[Dict[str, str]]: try: lines = mapping_file.read_text(encoding="utf-8", errors="ignore").splitlines() except Exception: return [] if not lines: return [] header = lines[0].rstrip("\n").split("\t") rows: list[Dict[str, str]] = [] for line in lines[1:]: if not line.strip(): continue cols = line.rstrip("\n").split("\t") row = {header[i]: cols[i] if i < len(cols) else "" for i in range(len(header))} rows.append(row) return rows def _fasta_header_ids(fasta_file: Path) -> list[str]: ids: list[str] = [] try: with open(fasta_file, "r", encoding="utf-8", errors="ignore") as handle: for line in handle: if line.startswith(">"): token = line[1:].strip().split()[0] if token: ids.append(token) except Exception: return [] return ids def _fasta_header_records(fasta_file: Path) -> list[Dict[str, str]]: records: list[Dict[str, str]] = [] protein_patterns = [ re.compile(r"^(?:NP|XP|YP|WP|AP)_[0-9]+(?:\.\d+)?$"), re.compile(r"^(?:AAA|AAB|AAC|AAD|AAE|AAF|AAG|AAH|BAA|BAB|BAC|BAD|CAE|CAF)[0-9]{5,}(?:\.\d+)?$"), ] transcript_pattern = re.compile(r"^(?:NM|XM|NR|XR)_[0-9]+(?:\.\d+)?$") try: with open(fasta_file, "r", encoding="utf-8", errors="ignore") as handle: for line in handle: if not line.startswith(">"): continue header = line[1:].strip() tokens = header.split() if not tokens: continue protein_id = "" transcript_id = "" gene_id = "" for token in tokens[1:]: clean = token.rstrip(",;") if clean.startswith("GeneID:"): gene_id = clean.split(":", 1)[1] if not transcript_id and transcript_pattern.match(clean): transcript_id = clean if not protein_id and any(pattern.match(clean) for pattern in protein_patterns): protein_id = clean records.append({ "header_id": tokens[0], "gene_id": gene_id, "symbol": tokens[2] if len(tokens) > 2 and tokens[1].startswith("GeneID:") else "", "transcript_id": transcript_id, "protein_id": protein_id, "raw_header": header, }) except Exception: return [] return records def _extract_preview_filename(text: str) -> str: value = str(text or "") patterns = [ r"([A-Za-z0-9][A-Za-z0-9_.-]*\.(?:fa|fasta|faa|fna|ffn|pep|cds|txt|tsv|csv|md|json|yaml|yml|log))", r"([A-Za-z0-9][A-Za-z0-9_.-]*\.ncbi_fetch\.[A-Za-z0-9_.-]+)", ] for pattern in patterns: match = re.search(pattern, value, flags=re.IGNORECASE) if match: return Path(match.group(1)).name return "" def _looks_like_file_preview_request(text: str) -> bool: value = str(text or "") lower = value.lower() filename = _extract_preview_filename(value) if not filename: return False preview_words = [ "前几行", "前几列", "前几条", "前几项", "前几", "前10", "前 10", "前20", "前 20", "表头", "列名", "字段", "看一下", "查看", "展示", "显示", "列出来", "预览", "head", "tail", "first", "preview", ] return any(word in lower or word in value for word in preview_words) def _looks_like_fasta_id_filter_request(text: str) -> bool: value = str(text or "") lower = value.lower() filename = _extract_preview_filename(value) if not filename or not filename.lower().endswith((".fa", ".fasta", ".faa", ".pep")): return False has_filter_action = any(word in lower or word in value for word in [ "过滤", "只保留", "提取", "抽取", "列出", "展示", "显示", "查看", "filter", "extract", ]) has_id_target = any(word in lower or word in value for word in [ "id列", "id 列", "蛋白id", "蛋白 id", "protein id", "protein accession", "accession", "header id", "基因id", "基因 id", ]) return has_filter_action and has_id_target def _preview_limit_from_text(text: str, default: int = 10, maximum: int = 50) -> int: value = str(text or "") match = re.search(r"(?:前|head\s*-n\s*|first\s+)(\d+)", value, flags=re.IGNORECASE) if not match: match = re.search(r"(\d+)\s*(?:行|条|列)", value) if match: return max(1, min(maximum, int(match.group(1)))) return default def _resolve_preview_file(filename: str, job_id: str = "") -> Optional[Path]: if not filename or Path(filename).is_absolute() or ".." in Path(filename).parts: return None candidates: list[Path] = [] if job_id: job_ws = WORKSPACE_ROOT / job_id candidates.extend([ job_ws / "final" / filename, PROJECT_ROOT / "input" / job_id / filename, ]) candidates.extend(sorted(job_ws.glob(f"*/analysis/{filename}"))) candidates.extend(sorted(job_ws.glob(f"*/{filename}"))) candidates.extend(sorted(WORKSPACE_ROOT.glob(f"*/final/{filename}"), key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True)) for path in candidates: try: resolved = path.resolve() allowed_roots = [WORKSPACE_ROOT.resolve(), (PROJECT_ROOT / "input").resolve()] if path.exists() and path.is_file() and any(str(resolved).startswith(str(root) + os.sep) for root in allowed_roots): return path except Exception: continue return None def _preview_file_content(path: Path, request_text: str) -> Dict[str, Any]: lower = str(request_text or "").lower() max_lines = _preview_limit_from_text(request_text) max_bytes = 20000 suffix = path.suffix.lower() mode = "columns" if any(word in lower or word in request_text for word in ["前几列", "列名", "字段", "表头", "columns"]) else "lines" try: raw = path.read_text(encoding="utf-8", errors="ignore")[:max_bytes] except Exception as exc: return {"found": False, "message": f"读取文件失败: {exc}", "path": str(path)} lines = raw.splitlines() if mode == "columns" and suffix in {".tsv", ".csv"}: sep = "\t" if suffix == ".tsv" or ("\t" in lines[0] if lines else False) else "," header = lines[0].split(sep) if lines else [] limit = _preview_limit_from_text(request_text, default=min(10, len(header) or 10)) return { "found": True, "mode": "columns", "path": str(path), "filename": path.name, "columns": header[:limit], "total_preview_columns": min(limit, len(header)), "truncated": len(header) > limit, } return { "found": True, "mode": "lines", "path": str(path), "filename": path.name, "lines": lines[:max_lines], "line_count": len(lines[:max_lines]), "truncated": len(lines) > max_lines or path.stat().st_size > max_bytes, } def _load_file_preview_result(text: str, job_id: str = "") -> Dict[str, Any]: filename = _extract_preview_filename(text) path = _resolve_preview_file(filename, job_id) if not path: return {"found": False, "filename": filename, "message": f"没有在当前 job 的安全目录中找到文件：{filename}"} return _preview_file_content(path, text) def _format_file_preview_result(result: Dict[str, Any]) -> str: if not result.get("found"): return result.get("message") or "没有找到可预览的文件。" if result.get("mode") == "columns": columns = result.get("columns") or [] body = "\n".join(f"{idx}. {name}" for idx, name in enumerate(columns, 1)) return f"{result.get('filename')} 的前 {len(columns)} 列：\n\n{body}\n\n来源文件：{result.get('path')}" lines = result.get("lines") or [] body = "\n".join(lines) return f"{result.get('filename')} 的前 {len(lines)} 行：\n\n```text\n{body}\n```\n\n来源文件：{result.get('path')}" def _load_fasta_id_filter_result(text: str, job_id: str = "") -> Dict[str, Any]: filename = _extract_preview_filename(text) path = _resolve_preview_file(filename, job_id) if not path: return {"found": False, "filename": filename, "message": f"没有在当前 job 的安全目录中找到文件：{filename}"} records = _fasta_header_records(path) if not records: return {"found": False, "filename": filename, "path": str(path), "message": f"没有从 {filename} 读取到 FASTA header。"} lower = str(text or "").lower() target = "protein_id" if any(word in lower or word in text for word in ["蛋白id", "蛋白 id", "protein id", "protein accession", "accession"]) else "header_id" ids = [row.get(target, "") for row in records if row.get(target, "")] return { "found": True, "mode": "fasta_id_filter", "target": target, "path": str(path), "filename": path.name, "ids": ids, "records": records, "missing_count": len(records) - len(ids), } def _format_fasta_id_filter_result(result: Dict[str, Any]) -> str: if not result.get("found"): return result.get("message") or "没有找到可抽取的 FASTA ID。" target = result.get("target") title = "蛋白 ID" if target == "protein_id" else "FASTA header ID / 基因 ID" ids = result.get("ids") or [] lines = [ f"已从 {result.get('filename')} 的 FASTA header 中抽取 {title}。", f"共 {len(ids)} 个；未识别到该字段的 header 数：{result.get('missing_count', 0)}。", "", ] lines.extend(f"{idx}. {item}" for idx, item in enumerate(ids, 1)) lines.append("") lines.append(f"来源文件：{result.get('path')}") return "\n".join(lines) def _candidate_ncbi_result_dirs(prefix: str, job_id: str = "") -> list[Path]: candidates: list[Path] = [] if job_id: candidates.append(WORKSPACE_ROOT / job_id) if prefix and WORKSPACE_ROOT.exists(): for manifest in sorted( WORKSPACE_ROOT.glob(f"*/final/{prefix}.ncbi_fetch.manifest.json"), key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True, ): candidates.append(manifest.parent.parent) for manifest in sorted( WORKSPACE_ROOT.glob(f"*/final/{prefix}.ncbi_fetch_sequences.manifest.json"), key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True, ): candidates.append(manifest.parent.parent) seen: set[str] = set() unique: list[Path] = [] for path in candidates: key = str(path.resolve()) if path.exists() else str(path) if key not in seen: unique.append(path) seen.add(key) return unique def _load_ncbi_gene_id_result(prefix: str, job_id: str = "") -> Dict[str, Any]: if not prefix: return {"found": False, "message": "没有识别到结果前缀，例如 At.SPL。"} for workspace_dir in _candidate_ncbi_result_dirs(prefix, job_id): final_dir = workspace_dir / "final" if not final_dir.exists(): continue manifest_path = ( final_dir / f"{prefix}.ncbi_fetch.manifest.json" if (final_dir / f"{prefix}.ncbi_fetch.manifest.json").exists() else final_dir / f"{prefix}.ncbi_fetch_sequences.manifest.json" ) manifest: Dict[str, Any] = {} if manifest_path.exists(): try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: manifest = {} artifacts = manifest.get("artifacts") if isinstance(manifest.get("artifacts"), dict) else {} fasta_path = Path(artifacts.get("protein_fasta", {}).get("full_path", "")) if artifacts.get("protein_fasta") else final_dir / f"{prefix}.ncbi_fetch.protein.fa" fasta_records = _fasta_header_records(fasta_path) if fasta_path.exists() else [] if fasta_records: return { "found": True, "prefix": prefix, "workspace": workspace_dir.name, "source": str(fasta_path), "id_source": "protein_fasta_header", "ids": [row["header_id"] for row in fasta_records], "rows": [], "fasta_records": fasta_records, "manifest": str(manifest_path) if manifest_path.exists() else "", } mapping_path = Path(artifacts.get("mapping", {}).get("full_path", "")) if artifacts.get("mapping") else final_dir / f"{prefix}.ncbi_fetch.mapping.tsv" rows = _read_ncbi_mapping_rows(mapping_path) if mapping_path.exists() else [] if rows: ids = [row.get("input_id") or row.get("locus_tag") or row.get("gene_id") for row in rows] ids = [item for item in ids if item] return { "found": True, "prefix": prefix, "workspace": workspace_dir.name, "source": str(mapping_path), "id_source": "mapping_input_id", "ids": ids, "rows": rows, "manifest": str(manifest_path) if manifest_path.exists() else "", } queries = manifest.get("parameters", {}).get("queries") if manifest else None if isinstance(queries, list) and queries: return { "found": True, "prefix": prefix, "workspace": workspace_dir.name, "source": str(manifest_path), "ids": [str(item) for item in queries], "rows": [], "manifest": str(manifest_path), } return {"found": False, "message": f"没有在当前 workspace 中找到 {prefix} 的 NCBI 下载结果。"} def _format_ncbi_gene_id_result(result: Dict[str, Any]) -> str: if not result.get("found"): return result.get("message") or "没有找到可展示的基因 ID 列表。" rows = result.get("rows") or [] fasta_records = result.get("fasta_records") or [] ids = result.get("ids") or [] lines = [ f"已找到 {result.get('prefix')} 的 NCBI 下载结果，workspace: {result.get('workspace')}。", f"共 {len(ids)} 个基因 ID：", ] if fasta_records: lines.append("") lines.append("| FASTA header ID / 基因ID | NCBI GeneID | symbol | protein ID |") lines.append("|---|---:|---|---|") for row in fasta_records: lines.append( f"| {row.get('header_id', '')} | {row.get('gene_id', '')} | " f"{row.get('symbol', '')} | {row.get('protein_id', '')} |" ) elif rows: lines.append("") lines.append("| input_id / 基因ID | NCBI GeneID | symbol | protein ID |") lines.append("|---|---:|---|---|") for row in rows: lines.append( f"| {row.get('input_id', '')} | {row.get('gene_id', '')} | " f"{row.get('symbol', '')} | {row.get('protein_accession', '')} |" ) else: lines.append("") lines.extend(f"{idx}. {gene_id}" for idx, gene_id in enumerate(ids, 1)) lines.append("") lines.append(f"来源文件：{result.get('source')}") return "\n".join(lines) # 导入Celery任务 try: from tasks import execute_agent_task, execute_single_stage_task CELERY_AVAILABLE = True print("✅ Celery任务导入成功") except ImportError as e: CELERY_AVAILABLE = False print(f"⚠️ Celery任务导入失败: {e}") # 导入多Agent系统组件 try: from gfa_core.mas.core.orchestrator import Orchestrator from gfa_core.mas.core.langgraph_orchestrator import LangGraphOrchestrator, GraphState from gfa_core.mas.llm import LLMManager from gfa_core.mas.memory import MemoryManager from gfa_core.mas.api.conversation_routes import router as conv_router # 会话管理路由 MAS_AVAILABLE = True LANGGRAPH_AVAILABLE = True print("✅ MAS组件导入成功") print("✅ LangGraph编排器导入成功") except ImportError as e: MAS_AVAILABLE = False LANGGRAPH_AVAILABLE = False print(f"⚠️ MAS组件导入失败: {e}") Orchestrator = None LangGraphOrchestrator = None LLMManager = None MemoryManager = None WORKSPACE_ROOT = PROJECT_ROOT / "workspace" INPUT_ROOT = PROJECT_ROOT / "input" def _nonempty_prefix(value: object) -> str: text = str(value or "").strip() return text if text and text.lower() not in {"none", "null", "undefined", "nan", "-"} else "" def _gene_family_from_config(config_path: Path) -> Optional[str]: if not config_path.exists(): return None try: import pandas as pd sheets = pd.read_excel(config_path, sheet_name=None) for sheet_name, df in sheets.items(): if str(sheet_name).lower() in {"analysis", "meta", "metadata"}: for _, row in df.fillna("").iterrows(): cells = [str(v).strip() for v in row.tolist()] for idx, cell in enumerate(cells[:-1]): if cell.lower() in {"family", "gene_family", "analysis_family", "project_family"}: value = cells[idx + 1].strip() if value and value.lower() not in {"nan", "none", "null"}: return value if "Gene_Family" in df.columns: for value in df["Gene_Family"].dropna().astype(str): value = value.strip() if value and value.lower() not in {"nan", "none", "null"}: return value except Exception as exc: print(f"[WARN] Failed to infer gene family from {config_path}: {exc}") return None def _prefix_from_manifest(path: Path) -> str: if not path.exists(): return "" try: with open(path, "r", encoding="utf-8") as handle: data = json.load(handle) for key in ("prefix", "effective_prefix"): value = _nonempty_prefix(data.get(key)) if value: return value target = data.get("target") if isinstance(data.get("target"), dict) else None if target: value = _nonempty_prefix(target.get("prefix")) if value: return value artifacts = data.get("artifacts") if isinstance(data.get("artifacts"), dict) else {} for value in artifacts.values(): if isinstance(value, str): name = Path(value).name if (name.startswith("Target.") or name.startswith("Multi.")) and ".genome_prepare.manifest" in name: return name.split(".genome_prepare.manifest", 1)[0] except Exception as exc: print(f"[WARN] Failed to read prefix manifest {path}: {exc}") return "" def _prefix_from_final_members(final_dir: Path) -> str: if not final_dir.exists(): return "" for pattern in ("Target.*.final_members.pep.fa", "Multi.*.final_members.pep.fa"): for file in sorted(final_dir.glob(pattern)): name = file.name if name.endswith(".final_members.pep.fa"): return name[:-len(".final_members.pep.fa")] return "" def _prefix_from_ref_combined(job_dir: Path) -> str: for base in (job_dir, job_dir / "final"): if not base.exists(): continue for file in sorted(base.glob("Ref_combined.*.pep.fasta")): parts = file.name.split(".") if len(parts) >= 3: return f"Target.{parts[1]}" return "" def _rnaseq_prefix_from_final(final_dir: Path) -> str: if not final_dir.exists(): return "" preferred = [ "*.rnaseq_wgcna.manifest.json", "*.rnaseq_fgsea.manifest.json", "*.rnaseq_candidate_family.manifest.json", "*.rnaseq_sample_qc.manifest.json", "*.rnaseq_de.manifest.json", "*.rnaseq_quant.manifest.json", "*.rnaseq_align.manifest.json", "*.rnaseq_qc.manifest.json", "*.annotation.manifest.json", "*.annotation_go.manifest.json", "*.annotation_kegg.manifest.json", "*.genome_prepare.manifest.json", ] for pattern in preferred: matches = sorted(final_dir.glob(pattern)) if len(matches) == 1: suffix = pattern[1:] return matches[0].name[: -len(suffix)] return "" def _species_prefix_from_job_resources(job_dir: Path) -> str: """Infer RNA-seq output prefix from the unified project context.""" context = resolve_rnaseq_project_context(input_dir=job_dir) return context.get("output_prefix", "") RNASEQ_SPECIES_DEFAULTS = { "At": {"speciesName": "Arabidopsis thaliana", "keggOrganism": "ath"}, "Ath": {"speciesName": "Arabidopsis thaliana", "keggOrganism": "ath"}, "Os": {"speciesName": "Oryza sativa", "keggOrganism": "osa"}, "osa": {"speciesName": "Oryza sativa", "keggOrganism": "osa"}, "osa1": {"speciesName": "Oryza sativa", "keggOrganism": "osa"}, "osa1_r7": {"speciesName": "Oryza sativa", "keggOrganism": "osa"}, "Zm": {"speciesName": "Zea mays", "keggOrganism": "zma"}, "Gm": {"speciesName": "Glycine max", "keggOrganism": "gmx"}, "Sl": {"speciesName": "Solanum lycopersicum", "keggOrganism": "sly"}, "Lc": {"speciesName": "Litchi chinensis", "keggOrganism": ""}, "Dl": {"speciesName": "Dimocarpus longan", "keggOrganism": ""}, "Cp": {"speciesName": "Carica papaya", "keggOrganism": "cpap"}, } def _read_rnaseq_yaml_metadata(input_dir: Path) -> Dict[str, str]: meta = read_rnaseq_sample_metadata(input_dir=input_dir) out: Dict[str, str] = {} if meta.get("species_code"): out["speciesCode"] = meta["species_code"] if meta.get("species_name"): out["speciesName"] = meta["species_name"] if meta.get("kegg_organism"): out["keggOrganism"] = meta["kegg_organism"] if meta.get("output_prefix"): out["rnaseqPrefix"] = meta["output_prefix"] return out def _infer_rnaseq_species_defaults(job_id: str, prefix: str = "", stage: str = "rnaseq_fgsea") -> Dict[str, str]: input_dir = INPUT_ROOT / job_id final_dir = WORKSPACE_ROOT / job_id / "final" context = resolve_rnaseq_project_context(input_dir=input_dir, final_dir=final_dir, requested_prefix=prefix) species_code = context.get("species_code") or context.get("output_prefix", "").split(".", 1)[0].strip() defaults = dict(RNASEQ_SPECIES_DEFAULTS.get(species_code, {})) if context.get("species_name"): defaults["speciesName"] = context["species_name"] if context.get("kegg_organism"): defaults["keggOrganism"] = context["kegg_organism"] defaults.setdefault("speciesName", "") defaults.setdefault("keggOrganism", "") defaults["prefix"] = context.get("output_prefix") or prefix defaults["speciesCode"] = species_code defaults["referenceId"] = context.get("reference_id", "") defaults["source"] = context.get("source", "") return defaults def _is_gene_family_prefix(value: str) -> bool: value = str(value or "").strip() return "." in value or value.lower() in {"target", "multi"} def resolve_effective_prefix(job_id: str, params: Dict, stage: str) -> str: provided = _nonempty_prefix(params.get("prefix")) workspace_dir = WORKSPACE_ROOT / job_id final_dir = workspace_dir / "final" def normalize_gene_family_prefix(value: str) -> str: value = _nonempty_prefix(value) if stage not in GENE_FAMILY_SCOPE_STAGES or not value: return value species, family = _prefix_species_family(value) config_family = _nonempty_prefix(_gene_family_from_config(INPUT_ROOT / job_id / "config.xlsx")) species_codes = {code.lower() for code in _species_codes_for_job(job_id)} if config_family and (not family or family != config_family or family.lower() in species_codes): if species and species not in {"Multi", "Target"}: return f"{species}.{config_family}" return f"Target.{config_family}" if value.startswith("Multi.") and family: return f"Target.{family}" return value job_dir = INPUT_ROOT / job_id if stage in RNA_SEQ_STAGES: rnaseq_context = resolve_rnaseq_project_context( input_dir=job_dir, final_dir=final_dir, requested_prefix=provided, params=params, ) resolved = rnaseq_context.get("output_prefix", "") if resolved: if resolved != provided: print( f"[INFO] Using RNA-seq output prefix {resolved} for {stage} " f"(requested {provided or ''}, source={rnaseq_context.get('source', '')})" ) return resolved if stage in GENE_FAMILY_SCOPE_STAGES: targets = params.get("targetSpecies") or params.get("target_species") or [] if isinstance(targets, str): targets = [item.strip() for item in targets.split(",") if item.strip()] if provided and not targets: return normalize_gene_family_prefix(provided) family = _resolve_authoritative_gene_family(job_id, params.get("geneFamily") or params.get("family"), provided, params) if family: if len(targets) == 1: return f"{targets[0]}.{family}" return f"Target.{family}" if provided: return normalize_gene_family_prefix(provided) species_config = job_dir / "genome_prepare" / "species_config.json" if species_config.exists(): try: with open(species_config, "r", encoding="utf-8") as handle: data = json.load(handle) target = data.get("target") if isinstance(data.get("target"), dict) else None if target: value = _nonempty_prefix(target.get("prefix")) if value: return normalize_gene_family_prefix(value) value = _nonempty_prefix(data.get("prefix")) if value: return normalize_gene_family_prefix(value) except Exception as exc: print(f"[WARN] Failed to read species_config prefix: {exc}") manifests = [] manifests.extend(sorted(final_dir.glob("Target.*.genome_prepare.manifest.json"))) manifests.extend(sorted((job_dir / "final").glob("Target.*.genome_prepare.manifest.json"))) manifests.extend(sorted(final_dir.glob("Multi.*.genome_prepare.manifest.json"))) manifests.extend(sorted((job_dir / "final").glob("Multi.*.genome_prepare.manifest.json"))) manifests.extend(sorted(final_dir.glob("*.genome_prepare.manifest.json"))) manifests.extend(sorted((job_dir / "final").glob("*.genome_prepare.manifest.json"))) for manifest in manifests: value = _prefix_from_manifest(manifest) if value: return normalize_gene_family_prefix(value) family = _gene_family_from_config(job_dir / "config.xlsx") if family: return f"Target.{family}" value = _prefix_from_ref_combined(job_dir) if value: return normalize_gene_family_prefix(value) value = _prefix_from_final_members(final_dir) if value: return normalize_gene_family_prefix(value) return "" def _explicit_force_rerun_requested(message: str = "", params: Optional[Dict[str, Any]] = None) -> bool: params = params or {} if bool(params.get("forceRerun") or params.get("force_rerun")): return True force_stages = params.get("forceRerunStages") or params.get("force_rerun_stages") or [] if isinstance(force_stages, str): force_stages = [item.strip() for item in re.split(r"[,;，；\s]+", force_stages) if item.strip()] if force_stages: return True return bool(re.search(r"重跑|重新执行|重新分析|重新跑|强制|force\s*rerun|rerun|re-run", message or "", flags=re.IGNORECASE)) def _stage_status_for_plan_identity(job_id: str, stage_id: str, identity: Any, *, strict_identity: bool = True): from gfa_core.mas.tools.stage_identity import get_stage_status_for_identity status = get_stage_status_for_identity( job_id=job_id, identity=identity, workspace_root=WORKSPACE_ROOT, strict_identity=strict_identity, ) if status.completed: return status # Legacy/manual TFBS runs used the unscoped Ft.FAMILY.tfbs.manifest.json. # For a single-target target_only request that is equivalent to the scoped # target-only result and should prevent duplicate FIMO scans. if ( stage_id == "tfbs" and strict_identity and getattr(identity, "scope_mode", "") == "target_only" and getattr(identity, "target_species", None) and not getattr(identity, "reference_species", None) and not getattr(identity, "comparative_species", None) ): try: fallback_identity = type(identity)(**identity.to_dict()) fallback_identity.scope_id = "" fallback_identity.scope_file = "" fallback_identity.scope_mode = "" fallback_identity.analysis_route = "" fallback_identity.member_set_id = "" fallback_status = get_stage_status_for_identity( job_id=job_id, identity=fallback_identity, workspace_root=WORKSPACE_ROOT, strict_identity=False, ) if fallback_status.completed: return fallback_status except Exception as exc: print(f"[Agent Plan] TFBS unscoped fallback status check skipped: {exc}") return status def _wgs_manifest_identity_for_plan(job_id: str, stage_id: str, prefix: str) -> dict[str, str]: """Recover one unambiguous WGS identity from successful manifests in a job.""" if not job_id or not stage_id.startswith("wgs_"): return {} final_dir = WORKSPACE_ROOT / job_id / "final" if not final_dir.is_dir(): return {} candidates: set[tuple[str, str, str]] = set() for manifest_file in final_dir.glob(f"*.{stage_id}*.manifest.json"): try: manifest = json.loads(manifest_file.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if str(manifest.get("stage") or "").strip() != stage_id: continue if str(manifest.get("status") or "").strip() not in {"success", "partial_success"}: continue params = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {} context = manifest.get("stage_context") if isinstance(manifest.get("stage_context"), dict) else {} def value(key: str) -> str: return str(manifest.get(key) or params.get(key) or context.get(key) or "").strip() manifest_prefix = value("prefix") or value("output_prefix") if not manifest_prefix: suffix = f".{stage_id}.manifest.json" if manifest_file.name.endswith(suffix): manifest_prefix = manifest_file.name[:-len(suffix)] if prefix and manifest_prefix and manifest_prefix != prefix: continue candidates.add((value("wgs_sample_set_id"), value("wgs_reference_id"), value("wgs_analysis_id"))) required = { "wgs_qc": (0, 2), "wgs_index": (1, 2), }.get(stage_id, (0, 1, 2)) valid = { item for item in candidates if all(item[index] for index in required) } if len(valid) > 1: raise ValueError( f"WGS identity is ambiguous for prefix {prefix} and stage {stage_id}" ) if not valid: return {} sample_set_id, reference_id, analysis_id = next(iter(valid)) return { "wgsSampleSetId": sample_set_id, "wgsReferenceId": reference_id, "wgsAnalysisId": analysis_id, } def _wgs_prefix_for_plan(job_id: str, stage_ids: list[str], requested_prefix: str = "") -> str: """Resolve the single WGS prefix represented by successful manifests.""" requested_prefix = str(requested_prefix or "").strip() if not job_id: return requested_prefix final_dir = WORKSPACE_ROOT / job_id / "final" if not final_dir.is_dir(): return requested_prefix candidates: set[str] = set() for stage_id in stage_ids: if not str(stage_id).startswith("wgs_"): continue for manifest_file in final_dir.glob(f"*.{stage_id}*.manifest.json"): try: manifest = json.loads(manifest_file.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if str(manifest.get("stage") or "").strip() != stage_id: continue if str(manifest.get("status") or "").strip() not in {"success", "partial_success"}: continue params = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {} context = manifest.get("stage_context") if isinstance(manifest.get("stage_context"), dict) else {} prefix = str( manifest.get("prefix") or manifest.get("output_prefix") or params.get("prefix") or params.get("output_prefix") or context.get("prefix") or context.get("output_prefix") or "" ).strip() if not prefix: suffix = f".{stage_id}.manifest.json" if manifest_file.name.endswith(suffix): prefix = manifest_file.name[:-len(suffix)] if prefix: candidates.add(prefix) if requested_prefix in candidates: return requested_prefix if len(candidates) == 1: return next(iter(candidates)) if requested_prefix: return requested_prefix if len(candidates) > 1: raise ValueError( "WGS prefix is ambiguous; please select one explicitly: " + ", ".join(sorted(candidates)) ) return "" def _clear_planned_wgs_upstream_bindings( planned_stage_ids: list[str], plan_params: dict[str, Any], stages_detail: list[dict[str, Any]], ) -> None: """Drop stale upstream bindings when their producer will be rerun.""" binding_keys = { "wgs_qc": ( "wgsQcManifest", "wgs_qc_manifest", "wgsQcAnalysisId", "wgs_qc_analysis_id", "wgsSampleSetId", "wgs_sample_set_id", "wgsAnalysisId", "wgs_analysis_id", ), "wgs_index": ( "wgsIndexManifest", "wgs_index_manifest", "wgsIndexAnalysisId", "wgs_index_analysis_id", "wgsReferenceId", "wgs_reference_id", ), "wgs_align": ("wgsAlignManifest", "wgs_align_manifest", "wgsAnalysisId", "wgs_analysis_id"), "wgs_markdup": ("wgsMarkdupManifest", "wgs_markdup_manifest", "wgsMarkdupAnalysisId", "wgs_markdup_analysis_id"), "wgs_bam_qc": ("wgsBamQcManifest", "wgs_bam_qc_manifest", "wgsBamQcAnalysisId", "wgs_bam_qc_analysis_id"), "wgs_haplotype_calling": ("wgsHaplotypeManifest", "wgs_haplotype_manifest", "wgsHaplotypeAnalysisId", "wgs_haplotype_analysis_id"), "wgs_gvcf_combine": ("wgsCombineManifest", "wgs_combine_manifest", "wgsCombineAnalysisId", "wgs_combine_analysis_id"), "wgs_joint_genotyping": ("wgsJointManifest", "wgs_joint_manifest", "wgsJointAnalysisId", "wgs_joint_analysis_id"), "wgs_variant_select": ("wgsSelectManifest", "wgs_select_manifest", "wgsSelectAnalysisId", "wgs_select_analysis_id"), "wgs_variant_filter": ("wgsFilterManifest", "wgs_filter_manifest", "wgsFilterAnalysisId", "wgs_filter_analysis_id"), "wgs_impute": ("wgsImputeManifest", "wgs_impute_manifest", "wgsImputeAnalysisId", "wgs_impute_analysis_id"), } keys_to_clear = { key for stage_id in planned_stage_ids for key in binding_keys.get(stage_id, ()) } for key in keys_to_clear: plan_params.pop(key, None) for detail in stages_detail: if not isinstance(detail, dict): continue detail_params = detail.get("params") if not isinstance(detail_params, dict): continue for key in keys_to_clear: detail_params.pop(key, None) def _identity_completed_stages_for_plan( job_id: str, stage_ids: List[str], plan_params: Dict[str, Any], stages_detail: List[Dict[str, Any]], *, effective_prefix: str = "", request_prefix: str = "", force_rerun_stage_ids: Optional[set[str]] = None, ) -> List[str]: if not job_id or not stage_ids: return [] force_rerun_stage_ids = force_rerun_stage_ids or set() completed: List[str] = [] try: from gfa_core.mas.tools.stage_identity import resolve_stage_identity except Exception as exc: print(f"[Agent Plan] Identity-aware completed import failed: {exc}") return completed for stage_id in stage_ids: if stage_id in force_rerun_stage_ids: continue detail = next( ( item for item in stages_detail if isinstance(item, dict) and (item.get("id") or item.get("name")) == stage_id ), {}, ) detail_params = detail.get("params", {}) if isinstance(detail, dict) else {} merged_status_params = {**(plan_params or {}), **(detail_params or {})} detail_prefix = merged_status_params.get("prefix") or effective_prefix or request_prefix try: if stage_id.startswith("wgs_"): recovered_wgs_identity = _wgs_manifest_identity_for_plan( job_id, stage_id, str(detail_prefix or "").strip(), ) for key, value in recovered_wgs_identity.items(): merged_status_params.setdefault(key, value) if recovered_wgs_identity and isinstance(detail, dict): detail.setdefault("params", {}).update(recovered_wgs_identity) if stage_id in POPULATION_STAGES: population_identity = _materialize_population_scope_params( job_id, stage_id, detail_prefix, merged_status_params, ) if population_identity: merged_status_params.update(population_identity) if isinstance(detail, dict): detail.setdefault("params", {}).update(population_identity) identity = resolve_stage_identity( stage=stage_id, prefix=detail_prefix, params={**merged_status_params, "prefix": detail_prefix}, job_id=job_id, workspace_root=WORKSPACE_ROOT, ) status = _stage_status_for_plan_identity(job_id, stage_id, identity, strict_identity=True) technical_reference_keys = ( "referenceSpecies", "refSpeciesList", "ppiReferenceSpecies", "referenceBundleSpecies", "ppiKaksReferenceSpecies", "reference_species", ) explicit_technical_identity = any( _param_list(merged_status_params.get(key)) for key in technical_reference_keys ) or any( str(merged_status_params.get(key) or "").strip() for key in ("scopeFile", "scope_file") ) or any( str((plan_params or {}).get(key) or "").strip() for key in ("scopeId", "scope_id", "memberSetId", "member_set_id") ) if ( status.completed is not True and stage_id in {"ortholog", "kaks", "ppi"} and not explicit_technical_identity ): requested_family = str( merged_status_params.get("geneFamily") or merged_status_params.get("gene_family") or (str(detail_prefix).split(".")[-1] if "." in str(detail_prefix) else "") ).strip() requested_targets = _param_list( merged_status_params.get("targetSpecies") or merged_status_params.get("target_species") ) if not requested_targets and "." in str(detail_prefix): requested_targets = [str(detail_prefix).split(".", 1)[0]] final_dir = WORKSPACE_ROOT / job_id / "final" candidates = sorted( final_dir.glob(f"*.{stage_id}.manifest.json") if final_dir.is_dir() else [], key=lambda path: path.stat().st_mtime, reverse=True, ) for manifest_path in candidates: try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: continue stage_context = manifest.get("stage_context") if isinstance(manifest.get("stage_context"), dict) else {} parameters = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {} def manifest_value(*keys: str): for source in (manifest, stage_context, parameters): for key in keys: if source.get(key) not in (None, "", []): return source.get(key) return "" manifest_prefix = str(manifest.get("prefix") or manifest.get("output_prefix") or "").strip() manifest_family = str( manifest_value("family", "gene_family") or (manifest_prefix.split(".")[-1] if "." in manifest_prefix else "") ).strip() manifest_targets = _param_list(manifest_value("target_species", "targetSpecies")) if ( manifest.get("stage") != stage_id or manifest.get("status") != "success" or manifest_prefix != str(detail_prefix) or manifest_family != requested_family or set(manifest_targets) != set(requested_targets) ): continue inherited_params = { **merged_status_params, "prefix": detail_prefix, "targetSpecies": manifest_targets, "referenceSpecies": _param_list(manifest_value("reference_species", "referenceSpecies")), "scopeMode": manifest_value("scope_mode", "scopeMode"), "analysisRoute": manifest_value("analysis_route", "analysisRoute"), "memberSetId": manifest_value("member_set_id", "memberSetId"), } inherited_params.pop("scopeId", None) inherited_params.pop("scope_id", None) inherited_identity = resolve_stage_identity( stage=stage_id, prefix=detail_prefix, params=inherited_params, job_id=job_id, workspace_root=WORKSPACE_ROOT, ) status = _stage_status_for_plan_identity( job_id, stage_id, inherited_identity, strict_identity=True, ) break if status.completed is True: if stage_id == "combo": required_panels = _combo_panels_from_params(merged_status_params) manifest_path = Path(str(getattr(status, "manifest_path", "") or "")) manifest_panels = _manifest_standard_combo_panels(manifest_path) if manifest_path else [] if required_panels and not set(required_panels).issubset(set(manifest_panels)): print( "[Agent Plan] Combo manifest panels do not satisfy requested plan: " f"required={required_panels}, manifest={manifest_panels}, manifest_path={manifest_path}" ) continue completed.append(stage_id) except Exception as exc: print(f"[Agent Plan] Identity-aware completed check failed for {stage_id}: {exc}") return list(dict.fromkeys(completed)) def _attach_single_stage_status(task: Dict) -> None: """Derive task status from the unified manifest/artifact validator when possible.""" job_id = task.get("job_id") stage = task.get("stage") prefix = task.get("prefix", "") if not job_id or not stage: return current_status = str(task.get("status") or "").lower() has_result_payload = isinstance(task.get("result"), dict) and bool(task.get("result")) terminal_statuses = {"success", "completed", "failed", "cancelled", "waiting_confirmation"} if current_status not in terminal_statuses and not has_result_payload: return if ( task.get("force_rerun_policy") and task.get("status") not in {"success", "completed", "failed", "cancelled", "waiting_confirmation"} and not task.get("result") ): return if stage == "reference_bundle_download": result_data = task.get("result") if isinstance(task.get("result"), dict) else {} if current_status in {"success", "completed"} or result_data.get("success") is True: task["status"] = "success" task["progress"] = 100 task["current_stage"] = stage task["stage_status"] = { "stage": stage, "status": result_data.get("status") or "success", "completed": True, "manifest_path": result_data.get("manifest_path") or result_data.get("manifest"), "manifest_status": result_data.get("status") or "success", "artifacts_checked": len(result_data.get("files") or {}), "missing_artifacts": [], "errors": [], } return result_data = task.get("result") if isinstance(task.get("result"), dict) else {} if result_data.get("success") is False: return direct_result = result_data.get("result") if isinstance(result_data.get("result"), dict) else {} final_output = result_data.get("final_output") if isinstance(result_data.get("final_output"), dict) else {} manifest = None for candidate in (direct_result.get("manifest"), final_output.get("manifest")): if isinstance(candidate, dict): manifest = candidate break if not manifest: manifest_path_text = ( result_data.get("manifest_path") or result_data.get("manifest") or (result_data.get("stage_status") or {}).get("manifest_path") or (task.get("stage_status") or {}).get("manifest_path") ) if manifest_path_text: try: manifest_path = Path(str(manifest_path_text)) if not manifest_path.is_absolute(): manifest_path = WORKSPACE_ROOT / job_id / str(manifest_path_text).lstrip("/") if manifest_path.exists(): manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: manifest = None actual_stage = stage actual_prefix = prefix task_context = task.get("stage_context") if isinstance(task.get("stage_context"), dict) else {} actual_scope_id = str(task.get("scope_id") or task_context.get("scope_id") or "").strip() actual_analysis_route = str( task.get("analysisRoute") or task.get("analysis_route") or task_context.get("analysis_route") or "" ).strip() actual_member_set_id = str( task.get("memberSetId") or task.get("member_set_id") or task_context.get("member_set_id") or "" ).strip() if isinstance(manifest, dict): actual_stage = manifest.get("stage") or actual_stage actual_prefix = manifest.get("prefix") or actual_prefix actual_scope_id = str(manifest.get("scope_id") or actual_scope_id).strip() actual_analysis_route = str(manifest.get("analysis_route") or actual_analysis_route).strip() actual_member_set_id = str(manifest.get("member_set_id") or actual_member_set_id).strip() actual_stage = result_data.get("current_stage") or actual_stage if not actual_scope_id and task.get("scopeFile"): try: scope_path = Path(str(task.get("scopeFile"))).expanduser() if not scope_path.is_absolute(): scope_path = WORKSPACE_ROOT / job_id / scope_path scope_data = json.loads(scope_path.read_text(encoding="utf-8")) actual_scope_id = str(scope_data.get("scope_id") or "").strip() except Exception: actual_scope_id = "" if actual_stage == "tree_plot_revise": final_dir = WORKSPACE_ROOT / job_id / "final" manifest_path = final_dir / f"{actual_prefix}.tree_plot_revise.manifest.json" if manifest_path.exists(): try: data = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: data = {} if data.get("status") in {"success", "partial_success"}: task["stage_status"] = { "stage": actual_stage, "status": data.get("status"), "completed": True, "manifest_path": str(manifest_path), "manifest_status": data.get("status"), "artifacts_checked": len(data.get("artifacts") or {}), "missing_artifacts": [], "errors": [], } task["status"] = "success" task["progress"] = 100 task["current_stage"] = actual_stage if not task.get("end_time"): task["end_time"] = datetime.now().isoformat() if not task.get("message") or task.get("message", "").startswith("任务 ") or "未生成有效输出" in task.get("message", ""): task["message"] = "✅ 进化树图修订完成。" return if actual_stage == "synteny_plot_revise": final_dir = WORKSPACE_ROOT / job_id / "final" manifest_path = final_dir / f"{actual_prefix}.synteny_plot_revise.manifest.json" if not manifest_path.exists() and final_dir.exists(): candidates = sorted( final_dir.glob("*.synteny_plot_revise.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True, ) manifest_path = candidates[0] if candidates else manifest_path if manifest_path.exists(): try: data = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: data = {} if data.get("status") in {"success", "partial_success"}: task["stage_status"] = { "stage": actual_stage, "status": data.get("status"), "completed": True, "manifest_path": str(manifest_path), "manifest_status": data.get("status"), "artifacts_checked": len(data.get("artifacts") or {}), "missing_artifacts": [], "errors": [], } task["status"] = "success" task["progress"] = 100 task["current_stage"] = actual_stage if not task.get("end_time"): task["end_time"] = datetime.now().isoformat() if not task.get("message") or task.get("message", "").startswith("任务 ") or "未生成有效输出" in task.get("message", ""): task["message"] = "✅ 种内共线性 Circos 图修订完成。" return if actual_stage == "promoter_plot_revise": final_dir = WORKSPACE_ROOT / job_id / "final" manifest_path = final_dir / f"{actual_prefix}.promoter_plot_revise.manifest.json" if not manifest_path.exists() and final_dir.exists(): candidates = sorted( final_dir.glob("*.promoter_plot_revise.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True, ) manifest_path = candidates[0] if candidates else manifest_path if manifest_path.exists(): try: data = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: data = {} if data.get("status") in {"success", "partial_success"}: task["stage_status"] = { "stage": actual_stage, "status": data.get("status"), "completed": True, "manifest_path": str(manifest_path), "manifest_status": data.get("status"), "artifacts_checked": len(data.get("artifacts") or {}), "missing_artifacts": [], "errors": [], } task["status"] = "success" task["progress"] = 100 task["current_stage"] = actual_stage if not task.get("end_time"): task["end_time"] = datetime.now().isoformat() if not task.get("message") or task.get("message", "").startswith("任务 ") or "未生成有效输出" in task.get("message", ""): task["message"] = "✅ 启动子自定义绘图完成。" return if actual_stage == "msa_region_plot": final_dir = WORKSPACE_ROOT / job_id / "final" region_manifests = sorted( final_dir.glob(f"{actual_prefix}.msa_region_*.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True, ) if final_dir.exists() else [] if region_manifests: try: data = json.loads(region_manifests[0].read_text(encoding="utf-8")) except Exception: data = {} if data.get("status") in {"success", "partial_success"}: task["stage_status"] = { "stage": actual_stage, "status": data.get("status"), "completed": True, "manifest_path": str(region_manifests[0]), "manifest_status": data.get("status"), "artifacts_checked": len(data.get("artifacts") or {}), "missing_artifacts": [], "errors": [], } task["status"] = "success" task["progress"] = 100 task["current_stage"] = actual_stage if not task.get("end_time"): task["end_time"] = datetime.now().isoformat() if not task.get("message") or task.get("message", "").startswith("任务 ") or "未生成有效输出" in task.get("message", ""): task["message"] = "✅ MSA区间绘图完成。" return try: task_identity = task.get("stage_identity") if isinstance(task.get("stage_identity"), dict) else {} if isinstance(manifest, dict) and manifest: from gfa_core.mas.tools.stage_identity import ( get_stage_status_for_identity, merge_manifest_identity_params, resolve_stage_identity, ) identity_params = merge_manifest_identity_params( { "prefix": actual_prefix, "scopeFile": task.get("scopeFile") or "", "scopeId": actual_scope_id, "scopeMode": task_context.get("scope_mode") or "", "analysisRoute": actual_analysis_route, "memberSetId": actual_member_set_id, }, manifest, ) strict_manifest_identity = bool( identity_params.get("scopeId") or identity_params.get("scope_id") or identity_params.get("analysisRoute") or identity_params.get("analysis_route") or identity_params.get("memberSetId") or identity_params.get("member_set_id") ) identity = resolve_stage_identity( stage=actual_stage, prefix=actual_prefix, params=identity_params, job_id=job_id, workspace_root=WORKSPACE_ROOT, ) if not strict_manifest_identity: identity.scope_id = "" identity.analysis_route = "" identity.member_set_id = "" report = get_stage_status_for_identity( job_id=job_id, identity=identity, strict_identity=strict_manifest_identity, ) elif task_identity: from gfa_core.mas.tools.stage_identity import StageIdentity, get_stage_status_for_identity identity = StageIdentity(**{k: v for k, v in task_identity.items() if k in StageIdentity.__dataclass_fields__}) report = get_stage_status_for_identity( job_id=job_id, identity=identity, strict_identity=True, ) else: from gfa_core.mas.tools.stage_status import get_stage_status as get_unified_stage_status report = get_unified_stage_status( job_id=job_id, prefix=actual_prefix, stage=actual_stage, scope_id=actual_scope_id, analysis_route=actual_analysis_route, member_set_id=actual_member_set_id, strict_identity=bool(actual_scope_id or actual_analysis_route or actual_member_set_id), ) if actual_stage in {"rnaseq_go_enrich", "rnaseq_kegg_enrich"} and report.completed: final_dir = WORKSPACE_ROOT / job_id / "final" suffix = ".GO_enrich.txt" if actual_stage == "rnaseq_go_enrich" else ".KEGG_enrich.txt" if not any(final_dir.glob(f"*{suffix}")): report.completed = False report.status = "not_found" report.errors.append(f"{actual_stage} specific output not found: *{suffix}") task["stage_status"] = report.to_dict() if report.completed: recovered_message = f"✅ {actual_stage} 分析已完成" task["status"] = "success" task["progress"] = 100 task["current_stage"] = actual_stage if not task.get("end_time"): task["end_time"] = datetime.now().isoformat() current_message = str(task.get("message") or "") if ( not current_message or current_message.startswith("任务 ") or "failed" in current_message.lower() or "失败" in current_message ): task["message"] = recovered_message result = task.get("result") if isinstance(result, dict) and not bool(result.get("success", True)): task["result"] = { "success": True, "status": "success", "message": task.get("message") or recovered_message, "stage": actual_stage, "recovered_from_manifest": True, "stage_status": report.to_dict(), } resource_task = task.get("resource_task") resource_task_id = task.get("id") or task.get("task_id") if resource_task_id and isinstance(resource_task, dict) and str(resource_task.get("status") or "").lower() == "failed": try: from gfa_core.mas.resource_governor import release as release_resource_task release_result = release_resource_task(resource_task_id, status="success", message=task.get("message") or recovered_message) if isinstance(release_result, dict) and isinstance(release_result.get("task"), dict): task["resource_task"] = release_result["task"] except Exception as release_exc: print(f"[API DEBUG] Failed to repair completed resource task status: {release_exc}") elif task.get("status") in {"success", "completed"}: task["status"] = "failed" task["message"] = f"❌ {actual_stage} 未生成有效输出" except Exception as e: print(f"[API DEBUG] Failed to attach unified stage status: {e}") def _attach_annotation_progress(task: Dict) -> None: """Attach long-running local annotation progress when available.""" job_id = task.get("job_id") stage = task.get("stage") if not job_id or stage not in {"annotation", "rnaseq_enrich", "rnaseq_go_enrich", "rnaseq_kegg_enrich"}: return if stage == "rnaseq_go_enrich": progress_paths = [WORKSPACE_ROOT / job_id / "annotation" / "analysis" / "go.progress.json"] elif stage == "rnaseq_kegg_enrich": progress_paths = [WORKSPACE_ROOT / job_id / "annotation" / "analysis" / "kegg.progress.json"] else: progress_paths = [ WORKSPACE_ROOT / job_id / "annotation" / "analysis" / "go.progress.json", WORKSPACE_ROOT / job_id / "annotation" / "analysis" / "kegg.progress.json", WORKSPACE_ROOT / job_id / "annotation" / "analysis" / "progress.json", WORKSPACE_ROOT / job_id / "rnaseq_annotation" / "progress.json", ] progress_path = next((path for path in progress_paths if path.exists()), None) if progress_path is None: return try: progress = json.loads(progress_path.read_text(encoding="utf-8")) except Exception as e: print(f"[API DEBUG] Failed to read annotation progress: {e}") return task["annotation_progress"] = progress progress_status = str(progress.get("status") or "").lower() if task.get("status") in {"pending", "running", "failed"}: if progress_status == "running" and _annotation_progress_stale(job_id, stage, progress): if stage in {"rnaseq_go_enrich", "rnaseq_kegg_enrich"} and _annotation_parent_still_active(task): progress["warning"] = "自动背景注释进程刚结束，等待后端写入最终注释状态。" task["status"] = "running" task["progress"] = progress.get("progress", task.get("progress", 0)) task["current_stage"] = "annotation" task["message"] = "自动背景注释状态同步中，正在等待后端最终结果..." return task["status"] = "failed" task["message"] = "自动背景注释已中断：progress 仍为 running，但没有检测到对应的运行进程。请重新执行该阶段。" task.setdefault("result", {"success": False, "error": task["message"]}) return if progress_status in {"failed", "error"}: task["status"] = "failed" task["message"] = progress.get("message") or "自动背景注释失败" task.setdefault("result", {"success": False, "error": task["message"]}) return if progress_status in {"success", "completed"}: if stage == "annotation": task["status"] = "success" task["progress"] = 100 task["message"] = "✅ 自动背景注释已完成" if not task.get("end_time"): task["end_time"] = datetime.now().isoformat() else: task["status"] = "running" task["progress"] = max(int(task.get("progress", 0) or 0), 90) task["current_stage"] = stage task["message"] = "自动背景注释已完成，正在继续执行富集分析..." return task["status"] = "running" task["progress"] = progress.get("progress", task.get("progress", 0)) task["current_stage"] = "annotation" total = progress.get("total", 0) processed = progress.get("processed", 0) annotated = progress.get("annotated", 0) failed = progress.get("failed", 0) task["message"] = ( f"自动背景注释进行中：{processed}/{total}，" f"已注释 {annotated}，失败 {failed}" ) def _annotation_parent_still_active(task: Dict) -> bool: """Return True when the enclosing enrichment task should not be downgraded yet.""" stage_status = task.get("stage_status") if isinstance(stage_status, dict) and stage_status.get("completed") is True: return True task_id = str(task.get("id") or "") if not task_id: return False try: from gfa_core.mas.resource_governor import get_task payload = get_task(task_id) resource_task = payload.get("task") if isinstance(payload, dict) else None status = str((resource_task or {}).get("status") or "").lower() return status in { "queued", "pending", "submitted", "running", "waiting_resource", "active", } except Exception: return str(task.get("status") or "").lower() in {"pending", "running", "queued", "waiting_resource"} def _annotation_progress_stale(job_id: str, stage: str, progress: Dict) -> bool: updated_raw = str(progress.get("updated_at") or "") try: updated = datetime.fromisoformat(updated_raw) if (datetime.now() - updated).total_seconds() < 60: return False except Exception: pass scope = "go" if stage == "rnaseq_go_enrich" else ("kegg" if stage == "rnaseq_kegg_enrich" else "") workspace = WORKSPACE_ROOT / job_id markers = [str(workspace.resolve()), str((workspace / "annotation" / "analysis").resolve())] if scope == "go": tool_markers = ["run_annotation.py", "interproscan"] elif scope == "kegg": tool_markers = ["run_annotation.py", "exec_annotation", "kofamscan", "hmmscan"] else: tool_markers = ["run_annotation.py", "interproscan", "exec_annotation", "hmmscan"] try: for proc in Path("/proc").iterdir(): if not proc.name.isdigit(): continue try: cmdline = (proc / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", errors="ignore") except Exception: continue if cmdline and any(marker in cmdline for marker in markers) and any(marker in cmdline for marker in tool_markers): return False except Exception: return False return True def _attach_agent_manifest_status(task: Dict) -> None: """Attach completed stages from manifests to an agent task.""" job_id = task.get("job_id") prefix = task.get("prefix", "") if not job_id: return try: from gfa_core.mas.tools.stage_status import list_completed_stages completed = list_completed_stages(job_id=job_id, prefix=prefix) task["stages_completed"] = completed task["completed_stages"] = completed except Exception as e: print(f"[API DEBUG] Failed to attach agent manifest status: {e}") app = FastAPI(title="BotanicalBaseAgent API") # MainAgent runtime state. Analysis planning and execution remain owned by # the original IntentAgent/PlanningAgent/frontend flow below. llm_manager = None agent_conversation_store = None @app.on_event("startup") def initialize_agent_conversation_store() -> None: """Initialize the persistent conversation state used only by MainAgent.""" global agent_conversation_store from gfa_core.mas.agent_state_repository import SQLiteAgentStateRepository from gfa_core.mas.persistent_conversation import PersistentConversationStore database_path = Path( os.environ.get( "PLANTAI_AGENT_STATE_DB", str(PROJECT_ROOT / "workspace" / ".agent_state" / "agent-state.sqlite3"), ) ).expanduser() database_path.parent.mkdir(parents=True, exist_ok=True) repository = SQLiteAgentStateRepository(database_path) repository.initialize() agent_conversation_store = PersistentConversationStore(repository) class AgentConversationCreateRequest(BaseModel): user_id: str = "default" title: str = "" job_id: Optional[str] = None class AgentConversationIdentityRequest(BaseModel): conversation_id: str class AgentConversationTitleRequest(AgentConversationIdentityRequest): title: str class AgentConversationMessageRequest(AgentConversationIdentityRequest): role: str content: str metadata: Optional[dict] = None class AgentConversationJobRequest(AgentConversationIdentityRequest): job_id: str def _agent_conversation_repository(): store = globals().get("agent_conversation_store") if store is None: raise HTTPException(status_code=503, detail="会话存储尚未初始化") return store.repository def _agent_conversation_principal(user_id: str) -> str: if str(user_id or "").strip() != "default": raise HTTPException(status_code=404, detail="用户不存在") return "web-session" @app.post("/api/conversations/create") async def create_agent_conversation(request: AgentConversationCreateRequest): repository = _agent_conversation_repository() conversation = repository.create_conversation( conversation_id=str(uuid.uuid4()), principal_id=_agent_conversation_principal(request.user_id), job_id=str(request.job_id or ""), title=str(request.title or "")[:200], ) return { "conversation_id": conversation.conversation_id, "title": conversation.title or "新会话", "created_at": conversation.created_at, "message": "会话创建成功", } @app.get("/api/conversations/list/{user_id}") async def list_agent_conversations( user_id: str, status: str = "active", limit: int = 20, offset: int = 0, ): repository = _agent_conversation_repository() try: conversations = repository.list_conversations( principal_id=_agent_conversation_principal(user_id), status=status, limit=limit, offset=offset, ) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc summaries = [] for conversation in conversations: message_count, last_message = repository.get_message_summary( conversation.conversation_id ) first_messages = repository.list_messages( conversation.conversation_id, limit=1, ) first_user = ( first_messages[0].content if first_messages and first_messages[0].role == "user" else "" ) ledger = repository.get_goal_ledger(conversation.conversation_id) active_job_id = ( str((ledger.user_terms if ledger else {}).get("active_job_id") or "") or conversation.job_id ) summaries.append( { "conversation_id": conversation.conversation_id, "title": conversation.title or first_user[:50] or "新会话", "last_message": last_message, "updated_at": conversation.updated_at, "message_count": message_count, "gene_family": None, "job_id": active_job_id or None, "input_dir": ( str(INPUT_ROOT / active_job_id) if active_job_id else None ), "output_dir": ( str(WORKSPACE_ROOT / active_job_id / "final") if active_job_id else None ), "working_dir": ( str(WORKSPACE_ROOT / active_job_id) if active_job_id else None ), } ) return { "user_id": user_id, "conversations": summaries, "total": len(summaries), } @app.get("/api/conversations/{conversation_id}/messages") async def get_agent_conversation_messages( conversation_id: str, limit: int = 500, offset: int = 0, ): repository = _agent_conversation_repository() try: conversation = repository.get_conversation(conversation_id) except ValueError as exc: raise HTTPException(status_code=404, detail="会话不存在") from exc if conversation is None or conversation.principal_id != "web-session": raise HTTPException(status_code=404, detail="会话不存在") if not 1 <= limit <= 500 or offset < 0: raise HTTPException(status_code=400, detail="无效的消息分页参数") total, _last_message = repository.get_message_summary(conversation_id) messages = repository.list_messages( conversation_id, limit=limit, offset=offset, ) return { "conversation_id": conversation_id, "title": conversation.title or "新会话", "messages": [ { "role": item.role, "content": item.content, "timestamp": item.created_at, "metadata": None, } for item in messages ], "total": total, } @app.post("/api/conversations/message") async def add_agent_conversation_message( request: AgentConversationMessageRequest, ): repository = _agent_conversation_repository() conversation = repository.get_conversation(request.conversation_id) if conversation is None or conversation.principal_id != "web-session": raise HTTPException(status_code=404, detail="会话不存在") repository.append_message( conversation_id=request.conversation_id, message_id=str(uuid.uuid4()), role=request.role, content=request.content, expected_revision=conversation.revision, ) return {"success": True, "message": "消息添加成功"} @app.post("/api/conversations/update-title") async def update_agent_conversation_title( request: AgentConversationTitleRequest, ): _agent_conversation_repository().update_conversation( conversation_id=request.conversation_id, principal_id="web-session", title=request.title, ) return {"success": True, "message": "标题更新成功"} @app.post("/api/conversations/archive") async def archive_agent_conversation( request: AgentConversationIdentityRequest, ): _agent_conversation_repository().update_conversation( conversation_id=request.conversation_id, principal_id="web-session", status="archived", ) return {"success": True, "message": "会话已归档"} @app.post("/api/conversations/delete") async def delete_agent_conversation( request: AgentConversationIdentityRequest, ): _agent_conversation_repository().update_conversation( conversation_id=request.conversation_id, principal_id="web-session", status="deleted", ) return {"success": True, "message": "会话已从列表移除"} @app.post("/api/conversations/link-to-job") async def link_agent_conversation_to_job( request: AgentConversationJobRequest, ): store = globals().get("agent_conversation_store") if store is None: raise HTTPException(status_code=503, detail="会话存储尚未初始化") conversation_id = store.rebind_job( session_id=request.conversation_id, job_id=request.job_id, ) return { "success": True, "conversation_id": conversation_id, "message": "会话已关联 Working Directory", } AGENT_TOKEN_TRACK_PATHS = {"/api/agent/plan", "/api/agent/chat", "/api/chat"} def _estimate_text_tokens(value: object) -> int: text = str(value or "") return max(1, len(text) // 4) if text else 0 def _track_agent_interaction_tokens(path: str, request_body: bytes, response_body: bytes) -> None: try: if path not in AGENT_TOKEN_TRACK_PATHS: return request_text = request_body.decode("utf-8", errors="ignore") if request_body else "" response_text = response_body.decode("utf-8", errors="ignore") if response_body else "" if not request_text and not response_text: return from gfa_core.mas.cost_tracker import track_cost agent_name = { "/api/agent/plan": "AgentPlanEstimate", "/api/agent/chat": "AgentChatEstimate", "/api/chat": "ChatEstimate", }.get(path, "AgentInteractionEstimate") track_cost( agent_name=agent_name, provider="local", model="deterministic-estimate", input_tokens=_estimate_text_tokens(request_text), output_tokens=_estimate_text_tokens(response_text), duration_ms=0, query=request_text[:100], ) except Exception as exc: print(f"[CostTracker] interaction token estimate failed: {exc}") async def _replay_body_receive(body: bytes): sent = False async def receive(): nonlocal sent if sent: return {"type": "http.request", "body": b"", "more_body": False} sent = True return {"type": "http.request", "body": body, "more_body": False} return receive # 挂载静态文件目录 WEB_DIR = Path(__file__).parent STATIC_DIR = WEB_DIR / "static" STATIC_DIR.mkdir(exist_ok=True) app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") # MainAgent uses the durable conversation routes registered on this app below. # The legacy in-memory router is intentionally not mounted a second time. # 添加CORS headers到所有响应，并对本地Agent交互做token估算 @app.middleware("http") async def add_cors_headers(request, call_next): if request.method == "OPTIONS": return Response( status_code=200, headers={ "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS", "Access-Control-Allow-Headers": "Content-Type, Authorization", } ) request_body = b"" track_interaction = request.method == "POST" and request.url.path in AGENT_TOKEN_TRACK_PATHS if track_interaction: request_body = await request.body() request._receive = await _replay_body_receive(request_body) response = await call_next(request) response.headers["Access-Control-Allow-Origin"] = "*" if not track_interaction: return response response_body = b"" async for chunk in response.body_iterator: response_body += chunk _track_agent_interaction_tokens(request.url.path, request_body, response_body) headers = dict(response.headers) headers["Content-Length"] = str(len(response_body)) return Response( content=response_body, status_code=response.status_code, headers=headers, media_type=response.media_type, ) # 确保目录存在 WORKSPACE_ROOT.mkdir(exist_ok=True) INPUT_ROOT.mkdir(exist_ok=True) # 内存存储 jobs = {} current_job_id = "default_job" # 异步任务存储 tasks: Dict[str, dict] = {} executor = ThreadPoolExecutor(max_workers=4) CELERY_PENDING_WARN_SEC = int(os.environ.get("BBAV5_CELERY_PENDING_WARN_SEC", "60")) CELERY_PENDING_STALE_SEC = int(os.environ.get("BBAV5_CELERY_PENDING_STALE_SEC", "180")) CELERY_HEALTH_TIMEOUT_SEC = float(os.environ.get("BBAV5_CELERY_HEALTH_TIMEOUT_SEC", "1.0")) CELERY_AUTO_RESTART = os.environ.get("BBAV5_CELERY_AUTO_RESTART", "1").strip().lower() not in {"0", "false", "no"} CELERY_RESTART_COOLDOWN_SEC = int(os.environ.get("BBAV5_CELERY_RESTART_COOLDOWN_SEC", "60")) CELERY_RESTART_WAIT_SEC = float(os.environ.get("BBAV5_CELERY_RESTART_WAIT_SEC", "2.0")) ADMIN_OPS_TOKEN = os.environ.get("BBAV5_ADMIN_OPS_TOKEN", "").strip() ADMIN_OPS_ALLOW_CONFIRM_ONLY = os.environ.get("BBAV5_ADMIN_OPS_ALLOW_CONFIRM_ONLY", "1").strip().lower() not in {"0", "false", "no"} def _parse_iso_datetime(value: Any) -> Optional[datetime]: if not value: return None try: return datetime.fromisoformat(str(value)) except Exception: return None def _elapsed_seconds_since(value: Any) -> float: dt = _parse_iso_datetime(value) if not dt: return 0.0 return max(0.0, (datetime.now() - dt).total_seconds()) def _default_celery_queue() -> str: try: from gfa_core.mas.stage_catalog import get_queue_prefix return f"{get_queue_prefix()}.default" except Exception: prefix = os.environ.get("BBAV5_QUEUE_PREFIX", "plantai_dev_7880").strip() or "plantai_dev_7880" return f"{prefix}.default" def _get_celery_queue_health(queue_name: str = "", timeout: float = CELERY_HEALTH_TIMEOUT_SEC) -> Dict[str, Any]: """Return whether at least one live worker consumes queue_name.""" queue_name = str(queue_name or "").strip() health: Dict[str, Any] = { "ok": False, "available": False, "queue": queue_name, "worker_count": 0, "workers": [], "online_workers": [], "active_queues": [], "checked_at": datetime.now().isoformat(), "error": "", } if not CELERY_AVAILABLE: health["error"] = "Celery is not available in this API process" return health try: from celery_app import app as celery_app inspector = celery_app.control.inspect(timeout=timeout) active_queues = inspector.active_queues() or {} if not active_queues: health["error"] = "No Celery workers replied to active_queues inspect" return health all_workers = sorted(active_queues.keys()) health["online_workers"] = all_workers seen_queues: set[str] = set() matched_workers: list[str] = [] for worker_name, queues in active_queues.items(): for queue in queues or []: qname = str(queue.get("name") or queue.get("routing_key") or "") if qname: seen_queues.add(qname) if queue_name and qname == queue_name: matched_workers.append(worker_name) health["active_queues"] = sorted(seen_queues) if queue_name: health["workers"] = sorted(set(matched_workers)) health["worker_count"] = len(health["workers"]) health["available"] = health["worker_count"] > 0 health["ok"] = health["available"] if not health["available"]: health["error"] = f"No online Celery worker consumes queue {queue_name}" else: health["workers"] = all_workers health["worker_count"] = len(all_workers) health["available"] = bool(all_workers) health["ok"] = health["available"] return health except Exception as exc: health["error"] = f"Celery health check failed: {exc}" return health def _celery_restart_state_path() -> Path: state_dir = WORKSPACE_ROOT / "resource_governor" state_dir.mkdir(parents=True, exist_ok=True) return state_dir / "celery_restart_watchdog.json" def _celery_start_script_for_current_profile() -> Path: prefix = os.environ.get("BBAV5_QUEUE_PREFIX", "").strip() redis_url = os.environ.get("BBAV5_REDIS_URL", "").strip() port = os.environ.get("PLANTAI_WEB_PORT", "").strip() if prefix.endswith("_9000") or redis_url.endswith("/10") or port == "9000": return PROJECT_ROOT / "start_celery_9000.sh" return PROJECT_ROOT / "start_celery_7880.sh" def _current_celery_profile() -> Dict[str, Any]: try: from gfa_core.mas.stage_catalog import get_queue_prefix prefix = get_queue_prefix() except Exception: prefix = os.environ.get("BBAV5_QUEUE_PREFIX", "plantai_dev_7880").strip() or "plantai_dev_7880" redis_default = "redis://localhost:6379/10" if prefix.endswith("_9000") or os.environ.get("PLANTAI_WEB_PORT", "").strip() == "9000" else "redis://localhost:6379/9" redis_url = os.environ.get("BBAV5_REDIS_URL", redis_default).strip() or redis_default script = _celery_start_script_for_current_profile() log_dir = Path(os.environ.get("BBAV5_CELERY_LOG_DIR", "") or f"logs/celery_{prefix}") if not log_dir.is_absolute(): log_dir = PROJECT_ROOT / log_dir worker_name_prefix = os.environ.get("BBAV5_WORKER_NAME_PREFIX", "").strip() or re.sub(r"[^A-Za-z0-9_]", "_", prefix) expected_queues = [f"{prefix}.{name}" for name in ["default", "heavy", "medium", "light", "rnaseq"]] expected_workers = [f"{worker_name_prefix}_{name}" for name in ["default", "heavy", "medium", "light", "rnaseq"]] restart_script = Path(os.environ.get("PLANTAI_RESTART_SCRIPT", "").strip() or (PROJECT_ROOT / "restart_plantai_9000.sh")) if not restart_script.is_absolute(): restart_script = PROJECT_ROOT / restart_script return { "queue_prefix": prefix, "redis_url": redis_url, "start_script": str(script), "start_script_exists": script.exists(), "log_dir": str(log_dir), "worker_name_prefix": worker_name_prefix, "expected_queues": expected_queues, "expected_workers": expected_workers, "project_restart_script": str(restart_script), "project_restart_configured": restart_script.exists(), "admin_auth_mode": "token" if ADMIN_OPS_TOKEN else ("confirm_only" if ADMIN_OPS_ALLOW_CONFIRM_ONLY else "disabled"), } def _read_process_cmdline(pid: int) -> str: try: return (Path("/proc") / str(pid) / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", "replace") except Exception: return "" def _find_celery_worker_pids(profile: Dict[str, Any]) -> List[Dict[str, Any]]: matches: Dict[int, Dict[str, Any]] = {} log_dir = Path(str(profile.get("log_dir") or "")) for worker_name in profile.get("expected_workers") or []: pidfile = log_dir / f"{worker_name}.pid" try: pid = int(pidfile.read_text(encoding="utf-8").strip()) except Exception: pid = 0 if pid > 0: cmdline = _read_process_cmdline(pid) if "celery" in cmdline and f"--hostname={worker_name}@" in cmdline: matches[pid] = {"pid": pid, "worker": worker_name, "source": "pidfile", "cmdline": cmdline} try: completed = subprocess.run( ["pgrep", "-af", f"--hostname={worker_name}@"], cwd=str(PROJECT_ROOT), capture_output=True, text=True, timeout=5, check=False, ) except Exception: completed = None if not completed or completed.returncode not in {0, 1}: continue for line in (completed.stdout or "").splitlines(): parts = line.strip().split(maxsplit=1) if not parts: continue try: pid = int(parts[0]) except Exception: continue cmdline = parts[1] if len(parts) > 1 else _read_process_cmdline(pid) if "celery" in cmdline and f"--hostname={worker_name}@" in cmdline: matches[pid] = {"pid": pid, "worker": worker_name, "source": matches.get(pid, {}).get("source") or "pgrep", "cmdline": cmdline} return sorted(matches.values(), key=lambda item: int(item.get("pid") or 0)) def _terminate_celery_worker_pids(processes: List[Dict[str, Any]], timeout_sec: float = 8.0) -> Dict[str, Any]: stopped: List[int] = [] still_running: List[int] = [] errors: List[str] = [] for proc in processes: pid = int(proc.get("pid") or 0) if pid <= 0: continue try: os.kill(pid, signal.SIGTERM) stopped.append(pid) except ProcessLookupError: stopped.append(pid) except Exception as exc: errors.append(f"{pid}: {exc}") deadline = time.time() + max(0.5, timeout_sec) while time.time() < deadline: alive = [] for pid in stopped: try: os.kill(pid, 0) alive.append(pid) except ProcessLookupError: pass except Exception: pass if not alive: break time.sleep(0.25) for pid in stopped: try: os.kill(pid, 0) still_running.append(pid) except ProcessLookupError: pass except Exception: pass return {"stopped_pids": stopped, "still_running_pids": still_running, "errors": errors} def _run_celery_start_script(profile: Dict[str, Any]) -> Dict[str, Any]: script = Path(str(profile.get("start_script") or "")) result: Dict[str, Any] = { "attempted": False, "success": False, "script": str(script), "stdout_tail": "", "stderr_tail": "", "returncode": None, } if not script.exists(): result["message"] = f"Celery start script not found: {script}" return result env = os.environ.copy() env["BBAV5_QUEUE_PREFIX"] = str(profile.get("queue_prefix") or "") env["BBAV5_REDIS_URL"] = str(profile.get("redis_url") or "") env["BBAV5_CELERY_LOG_DIR"] = str(profile.get("log_dir") or "") env["BBAV5_WORKER_NAME_PREFIX"] = str(profile.get("worker_name_prefix") or "") result["attempted"] = True try: completed = subprocess.run( ["bash", str(script)], cwd=str(PROJECT_ROOT), env=env, capture_output=True, text=True, timeout=45, check=False, ) result["returncode"] = completed.returncode result["stdout_tail"] = completed.stdout[-3000:] result["stderr_tail"] = completed.stderr[-3000:] result["success"] = completed.returncode == 0 result["message"] = "Celery start script executed" if result["success"] else "Celery start script failed" except Exception as exc: result["message"] = f"Celery start script failed: {exc}" return result def _try_restart_celery_workers(reason: str, queue_name: str) -> Dict[str, Any]: script = _celery_start_script_for_current_profile() result: Dict[str, Any] = { "attempted": False, "success": False, "reason": reason, "queue": queue_name, "script": str(script), "cooldown_sec": CELERY_RESTART_COOLDOWN_SEC, "message": "", "stdout_tail": "", "stderr_tail": "", "returncode": None, "started_at": datetime.now().isoformat(), } if not CELERY_AUTO_RESTART: result["message"] = "Celery auto restart disabled by BBAV5_CELERY_AUTO_RESTART" return result if not script.exists(): result["message"] = f"Celery start script not found: {script}" return result state_path = _celery_restart_state_path() try: if state_path.exists(): previous = json.loads(state_path.read_text(encoding="utf-8")) last_ts = float(previous.get("last_attempt_ts") or 0) wait_left = CELERY_RESTART_COOLDOWN_SEC - (time.time() - last_ts) if wait_left > 0: result["message"] = f"Celery restart skipped by cooldown ({int(wait_left)}s remaining)" result["last_attempt"] = previous return result except Exception: pass result["attempted"] = True try: completed = subprocess.run( ["bash", str(script)], cwd=str(PROJECT_ROOT), capture_output=True, text=True, timeout=30, check=False, ) result["returncode"] = completed.returncode result["stdout_tail"] = completed.stdout[-2000:] result["stderr_tail"] = completed.stderr[-2000:] result["success"] = completed.returncode == 0 result["message"] = "Celery start script executed" if result["success"] else "Celery start script failed" except Exception as exc: result["message"] = f"Celery restart failed: {exc}" result["finished_at"] = datetime.now().isoformat() try: state_path.write_text(json.dumps({"last_attempt_ts": time.time(), "last_result": result}, ensure_ascii=False, indent=2), encoding="utf-8") except Exception as exc: result["state_write_error"] = str(exc) return result def _ensure_celery_queue_health(queue_name: str, reason: str) -> tuple[Dict[str, Any], Dict[str, Any]]: health = _get_celery_queue_health(queue_name) restart_result: Dict[str, Any] = {} if health.get("available"): return health, restart_result restart_result = _try_restart_celery_workers(reason, queue_name) if restart_result.get("attempted"): time.sleep(max(0.0, CELERY_RESTART_WAIT_SEC)) health = _get_celery_queue_health(queue_name) health["restart_result"] = restart_result return health, restart_result def _queue_unavailable_payload(stage: str, queue_name: str, health: Dict[str, Any], submitted: bool, restart_result: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: return { "success": False, "status": "queue_unavailable", "stage": stage, "queue": queue_name, "celery_queue_health": health, "celery_restart": restart_result or health.get("restart_result") or {}, "action_required": True, "retryable": True, "submitted_to_celery": submitted, "recommended_action": "restart_worker", "message": f"{stage} 队列当前没有在线 worker，任务尚未开始执行", } def _apply_pending_watchdog(task: Dict[str, Any], celery_state: str) -> None: if celery_state != "PENDING": return queue_name = str(task.get("celery_queue") or _default_celery_queue()) age_sec = _elapsed_seconds_since(task.get("created_at") or task.get("start_time")) task["pending_seconds"] = round(age_sec, 1) if age_sec < CELERY_PENDING_WARN_SEC: return health, restart_result = _ensure_celery_queue_health(queue_name, reason="pending_watchdog") task["celery_queue_health"] = health if restart_result: task["celery_restart"] = restart_result task["action_required"] = False task["retryable"] = True if not health.get("available"): task["status"] = "queue_unavailable" task["action_required"] = True task["recommended_action"] = "restart_worker" task["message"] = ( f"任务已等待 {int(age_sec)} 秒但目标队列 {queue_name} 没有在线 worker；" "任务尚未真正开始执行。" ) task["result"] = _queue_unavailable_payload( str(task.get("stage") or task.get("current_stage") or "unknown"), queue_name, health, submitted=True, restart_result=restart_result, ) elif restart_result.get("attempted") and health.get("available"): task["status"] = "queued" task["recommended_action"] = "wait" task["message"] = f"检测到队列 {queue_name} 曾无 worker，已尝试恢复；任务仍在等待 worker 接收。" elif age_sec >= CELERY_PENDING_STALE_SEC: task["status"] = "stalled" task["recommended_action"] = "inspect_queue" task["message"] = ( f"任务已等待 {int(age_sec)} 秒仍未被 worker 接收；" f"队列 {queue_name} 有 worker 在线，需检查 routing key、Redis broker 或 worker 日志。" ) else: task["status"] = "stalled_warning" task["recommended_action"] = "wait_or_recheck" task["message"] = f"任务已等待 {int(age_sec)} 秒仍未被 worker 接收，正在监控队列状态。" # 初始化多Agent系统 orchestrator = None langgraph_orchestrator = None if MAS_AVAILABLE: try: # 清除Python模块缓存，确保配置重新加载 import sys modules_to_remove = [k for k in sys.modules.keys() if 'gfa_core.mas' in k] for m in modules_to_remove: del sys.modules[m] # 加载MAS配置 from gfa_core.mas.config import config as mas_config llm_config = mas_config.load_llm_config() # 注入阿里云百炼配置 llm_config["openai_api_key"] = mas_config.settings.openai_api_key llm_config["openai_base_url"] = mas_config.settings.openai_base_url print(f"[DEBUG] openai_api_key configured: {bool(llm_config.get('openai_api_key'))}") print(f"[DEBUG] openai_base_url: {llm_config.get('openai_base_url', 'NOT SET')}") # 强制重新创建LLMManager，不使用任何缓存 from gfa_core.mas.llm import LLMManager llm_manager = LLMManager(config=llm_config) # 清除provider缓存 llm_manager._providers = {} llm_manager._available_providers = None memory_manager = MemoryManager() # 初始化原 Orchestrator（保留作为参考） orchestrator = Orchestrator( llm_manager=llm_manager, memory_manager=memory_manager, papers_dir=PROJECT_ROOT / "paper" / "gfpaper" / "buchong" ) print("✅ Orchestrator初始化成功（保留参考）") # 初始化 LangGraph Orchestrator（新版本） if LANGGRAPH_AVAILABLE: langgraph_orchestrator = LangGraphOrchestrator(llm_manager=llm_manager) print("✅ LangGraph Orchestrator初始化成功") except Exception as e: print(f"⚠️ Orchestrator初始化失败: {e}") import traceback print(traceback.format_exc()) orchestrator = None langgraph_orchestrator = None # 需要异步轮询的阶段（耗时长，可能超过HTTP超时） # 分析阶段耗时评估： # - identify: HMMER+BLAST搜索，通常3-10分钟，必须异步 # - phylogeny: MAFFT建树+IQTree引导树，通常5-30分钟，必须异步 # - motif: MEME分析，通常5-20分钟，必须异步 # - domain: PfamScan搜索，通常3-10分钟，必须异步 # - structure: AlphaFold预测（如启用），可能数小时，必须异步 # - synteny: MCScanX共线性分析，通常5-15分钟，已包含 # - interspecies_synteny: 跨物种共线性分析，通常10-30分钟，已包含 # - karyotype: 染色体可视化，通常2-5分钟，已包含 # - pep: 多物种pep分析，通常5-15分钟，已包含 # - promoter: PlantCARE提交+等待，通常数小时，已包含 # - ppi: STRING蛋白互作网络，通常2-10分钟，加入异步 # - kaks: KaKs计算，通常2-10分钟，加入异步 # - expression: GEO数据下载+分析，通常10-30分钟，加入异步 # - rnaseq_*: RNA-seq全流程，每个阶段10-60分钟，已包含 PREPARE_SUBSTAGES = { "prepare_organize", "prepare_normalize_gff", "prepare_longest_transcript", "prepare_extract_pep", "prepare_extract_cds", "prepare_filter_fasta", "prepare_simplify_ncbi", } RNA_SEQ_STAGES = { "rnaseq_qc", "rnaseq_align", "rnaseq_quant", "rnaseq_sample_qc", "rnaseq_de", "annotation", "rnaseq_enrich", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea", "rnaseq_wgcna", "rnaseq_candidate_family", } ASYNC_STAGES = { # 核心分析阶段（耗时长） "identify", "identify_refine", "phylogeny", "reference_classification", "motif", "domain", "structure", # 共线性分析 "synteny", "interspecies_synteny", "karyotype", # 组合图 "combo", # 其他分析 "pep", "ppi", "kaks", "expression", "msa", "ortholog", # 启动子分析（外部服务） "promoter", "tfbs", # 基因组准备 "genome_prepare", "sequence_extract_by_id", *PREPARE_SUBSTAGES, # RNA-seq全流程 *RNA_SEQ_STAGES, } def _config_bool(value: object) -> bool: if isinstance(value, bool): return value text = str(value or "").strip().lower() return text in {"true", "1", "yes", "y", "ref"} def _ordered_species_options(job_id: str) -> Dict[str, list]: config_path = INPUT_ROOT / job_id / "config.xlsx" targets = [] refs = [] if config_path.exists(): try: import pandas as pd df = pd.read_excel(config_path) for species, species_rows in df.groupby("Species_Code", dropna=True): species = str(species).strip() if not species or species.lower() == "nan" or species == "-": continue roles = {str(v or "").strip().lower() for v in species_rows.get("Species_Role", [])} ref_flags = [_config_bool(v) for v in species_rows.get("Ref", [])] if roles & {"seed", "seed_reference", "known_family_reference"}: group = refs elif roles & {"comparative", "comparison", "outgroup", "reference", "ref", "comparison_reference"}: group = refs elif roles & {"target"} and not any(ref_flags): group = targets elif any(ref_flags): group = refs else: group = targets if species not in group: group.append(species) except Exception as exc: print(f"[WARN] Failed to read interspecies species options from {config_path}: {exc}") preferred_refs = [sp for sp in refs if sp.lower() in {"at", "ath"}] other_refs = [sp for sp in refs if sp.lower() not in {"at", "ath"}] default_refs = preferred_refs + other_refs if refs else [] return { "targets": targets, "refs": refs, "default_targets": list(targets), "default_refs": default_refs, } GENE_FAMILY_SCOPE_STAGES = { "identify", "phylogeny", "reference_classification", "structure", "motif", "domain", "combo", "promoter", "tfbs", "synteny", "interspecies_synteny", "karyotype", "pep", "ortholog", "ppi", "kaks", "expression", } def _default_scope_mode_for_stage(stage: str) -> str: if stage in {"reference_classification", "interspecies_synteny", "kaks"}: return "target_plus_reference" return "target_only" def _param_list(value: Any) -> list[str]: if value is None: return [] if isinstance(value, str): return [item.strip() for item in re.split(r"[,;，；、\s]+", value) if item.strip()] if isinstance(value, (list, tuple, set)): return [str(item).strip() for item in value if str(item).strip()] return [str(value).strip()] if str(value).strip() else [] def _prepare_execute_technical_reference_scope( req: Any, scope_params: Dict[str, Any], ) -> tuple[list[str], bool, bool]: stage = str(getattr(req, "stage", "") or "").strip() ppi_owner_intent = ( stage == "ortholog" and str(getattr(req, "dependencyOwner", "") or "").strip().lower() == "ppi" ) if ppi_owner_intent: missing_fields = [] if not bool(getattr(req, "ppiReferenceUserConfirmed", False)): missing_fields.append("ppiReferenceUserConfirmed") if not str(getattr(req, "ppiReferenceTaxonId", "") or "").strip(): missing_fields.append("ppiReferenceTaxonId") if not str(getattr(req, "ppiReferenceStorageKey", "") or "").strip(): missing_fields.append("ppiReferenceStorageKey") if missing_fields: raise HTTPException( status_code=400, detail={ "code": "ppi_owner_reference_incomplete", "issue_code": "ppi_reference_confirmation_required", "message": "PPI-owned Ortholog requires a confirmed STRING taxon and storage key.", "missing_fields": missing_fields, }, ) ppi_owned_ortholog = ppi_owner_intent if ppi_owned_ortholog: technical_reference_species = [ str(getattr(req, "ppiReferenceStorageKey", "") or "").strip() ] existing_scope_file = str(scope_params.get("scopeFile") or "").strip() if existing_scope_file: from gfa_core.mas.tools.stage_identity import ( load_scope_file, params_from_identity, resolve_stage_identity, ) job_id = str(getattr(req, "job_id", "") or "").strip() workspace = WORKSPACE_ROOT / job_id if job_id else None scope_data = load_scope_file(existing_scope_file, workspace) if not scope_data: raise HTTPException( status_code=400, detail={ "code": "ppi_owner_scope_invalid", "message": "PPI-owned Ortholog scopeFile is missing or unreadable.", "scope_file": existing_scope_file, }, ) scope_mode = str(scope_data.get("scope_mode") or scope_data.get("scopeMode") or "").strip() if scope_mode != "target_only": raise HTTPException( status_code=400, detail={ "code": "ppi_owner_scope_mode_incompatible", "message": "PPI-owned Ortholog only accepts an existing target_only biological scope.", "scope_file": existing_scope_file, "scope_mode": scope_mode, }, ) if not str(scope_data.get("scope_id") or scope_data.get("scopeId") or "").strip() or not _param_list( scope_data.get("target_species") or scope_data.get("targetSpecies") ): raise HTTPException( status_code=400, detail={ "code": "ppi_owner_scope_invalid", "message": "PPI-owned Ortholog scopeFile lacks scope_id or target_species.", "scope_file": existing_scope_file, }, ) identity = resolve_stage_identity( stage=stage, prefix=str(getattr(req, "prefix", "") or "").strip(), params={"scopeFile": existing_scope_file}, job_id=job_id, ) scope_params.update(params_from_identity(identity)) original_scope_id = str(scope_data.get("scope_id") or scope_data.get("scopeId") or "").strip() scope_params["scopeFile"] = existing_scope_file scope_params["scopeId"] = original_scope_id scope_params["scope_id"] = original_scope_id scope_params["refSpeciesList"] = list(identity.reference_species) scope_params["_validatedPpiOwnerScopeId"] = original_scope_id else: scope_params["scopeMode"] = "target_only" scope_params["referenceSpecies"] = [] scope_params["refSpeciesList"] = [] scope_params["comparativeSpecies"] = [] scope_params["scopeFile"] = "" else: technical_reference_species = [] if stage in {"ortholog", "ppi", "kaks"}: raw_technical_refs: list[Any] = [] raw_technical_refs.extend(_param_list(getattr(req, "refSpecies", "") or "")) raw_technical_refs.extend(_param_list(getattr(req, "ppiKaksReferenceSpecies", []) or [])) raw_technical_refs.extend(_param_list(getattr(req, "ppiReferenceSpecies", "") or "")) raw_technical_refs.extend(_param_list(getattr(req, "referenceBundleSpecies", "") or "")) for item in raw_technical_refs: code = str(item or "").strip() if code and code not in technical_reference_species: technical_reference_species.append(code) if technical_reference_species and not ppi_owned_ortholog: scope_params["referenceSpecies"] = list(technical_reference_species) scope_params["refSpeciesList"] = list(technical_reference_species) if not scope_params.get("scopeMode") or scope_params.get("scopeMode") == "target_only": scope_params["scopeMode"] = "target_plus_reference" needs_reference_fallback = ( stage in {"ortholog", "kaks", "interspecies_synteny"} and not ppi_owned_ortholog and not (scope_params.get("referenceSpecies") or scope_params.get("refSpeciesList")) ) return technical_reference_species, ppi_owned_ortholog, needs_reference_fallback def _apply_kaks_scope_defaults(job_id: str, prefix: str, plan_params: Dict[str, Any], intent: Dict[str, Any], stages_detail: list[Dict[str, Any]]) -> None: """Ka/Ks is a target-vs-reference analysis; never plan it with an empty reference scope.""" if not job_id: return input_dir = INPUT_ROOT / job_id if not input_dir.exists(): return workspace = WORKSPACE_ROOT / job_id effective_prefix = str(prefix or plan_params.get("prefix") or intent.get("prefix") or "").strip() prefix_species, prefix_family = _prefix_species_family(effective_prefix) family = str(plan_params.get("geneFamily") or intent.get("gene_family") or prefix_family or "").strip() target_species = ( _param_list(plan_params.get("targetSpecies")) or _param_list(intent.get("target_species")) or _param_list((intent.get("params") or {}).get("targetSpecies")) ) reference_species = ( _param_list(plan_params.get("referenceSpecies")) or _param_list(plan_params.get("refSpeciesList")) or _param_list(intent.get("reference_species")) or _param_list((intent.get("params") or {}).get("referenceSpecies")) or _param_list((intent.get("params") or {}).get("refSpeciesList")) ) # Ka/Ks in this pipeline is executed as target-vs-reference. Comparative # species from the broader config must not leak into the execution identity. comparative_species: list[str] = [] if not target_species and prefix_species and prefix_species not in {"Target", "Multi"}: target_species = [prefix_species] try: scope = build_gene_family_scope( input_dir=input_dir, workspace=workspace, prefix=effective_prefix, family=family, target_species=target_species if target_species else None, reference_species=reference_species if reference_species else None, comparative_species=comparative_species if comparative_species else None, scope_mode="target_plus_reference", ) except Exception as exc: print(f"[Agent Plan] KaKs scope default resolution skipped: {exc}") return if not family: family = scope.family if not target_species: target_species = list(scope.target_species or []) if not reference_species: reference_species = list(scope.reference_species or []) ref_set = set(reference_species or []) if ref_set: target_species = [sp for sp in target_species if sp not in ref_set] if not target_species or not reference_species: print(f"[Agent Plan] KaKs scope defaults incomplete: targets={target_species}, refs={reference_species}") return plan_prefix = effective_prefix or scope.prefix if len(target_species) == 1 and family and (not prefix_species or prefix_species in {"Target", "Multi"}): plan_prefix = f"{target_species[0]}.{family}" plan_params.update({ "prefix": plan_prefix, "geneFamily": family, "targetSpecies": target_species, "referenceSpecies": reference_species, "refSpeciesList": reference_species, "comparativeSpecies": comparative_species, "scopeMode": "target_plus_reference", }) intent.update({ "prefix": plan_params["prefix"], "gene_family": family, "species": target_species[0] if len(target_species) == 1 else "Multi", "target_species": target_species, "reference_species": reference_species, "comparative_species": comparative_species, "scopeMode": "target_plus_reference", }) intent_params = intent.setdefault("params", {}) intent_params.update({ "prefix": plan_params["prefix"], "geneFamily": family, "targetSpecies": target_species, "referenceSpecies": reference_species, "refSpeciesList": reference_species, "comparativeSpecies": comparative_species, "scopeMode": "target_plus_reference", }) for detail in stages_detail: if not isinstance(detail, dict): continue stage_id = detail.get("id") or detail.get("name") or "" if stage_id == "interspecies_synteny": continue if stage_id not in {"kaks", "ortholog"}: continue detail_params = {**plan_params, **(detail.get("params") or {})} for stale_key in ( "stageContext", "stage_context", "scopeId", "scope_id", "memberSetId", "member_set_id", "output_prefix", "legacy_output_prefix", "analysisRoute", "analysis_route", ): detail_params.pop(stale_key, None) detail_params.update({ "prefix": plan_params["prefix"], "geneFamily": family, "targetSpecies": target_species, "referenceSpecies": reference_species, "refSpeciesList": reference_species, "comparativeSpecies": comparative_species, "scopeMode": "target_plus_reference", }) detail["params"] = enrich_stage_params_with_context(stage_id, plan_params["prefix"], detail_params) def _apply_interspecies_scope_defaults( job_id: str, prefix: str, plan_params: Dict[str, Any], intent: Dict[str, Any], stages_detail: list[Dict[str, Any]], ) -> None: """Interspecies synteny compares target against all configured ref-side species.""" if not job_id: return options = _ordered_species_options(job_id) all_refs = _param_list(options.get("refs")) if not all_refs: return effective_prefix = str(prefix or plan_params.get("prefix") or intent.get("prefix") or "").strip() prefix_species, prefix_family = _prefix_species_family(effective_prefix) family = str(plan_params.get("geneFamily") or intent.get("gene_family") or prefix_family or "").strip() target_species = _param_list( plan_params.get("targetSpecies") or intent.get("target_species") or (intent.get("params") or {}).get("targetSpecies") ) if not target_species and prefix_species and prefix_species not in {"Target", "Multi"}: target_species = [prefix_species] target_set = set(target_species) ref_side = [sp for sp in all_refs if sp not in target_set] if not ref_side: return existing_ref_list = _param_list(plan_params.get("refSpeciesList")) if len(existing_ref_list) > 1: ref_side = list(dict.fromkeys(existing_ref_list)) primary_candidates = _param_list(plan_params.get("referenceSpecies")) or _param_list(intent.get("reference_species")) primary = next((sp for sp in primary_candidates if sp in ref_side), ref_side[0]) comparative = [sp for sp in ref_side if sp != primary] scope_mode = "target_plus_reference_plus_comparative" if comparative else "target_plus_reference" if len(target_species) == 1 and family: plan_prefix = f"{target_species[0]}.{family}" else: plan_prefix = effective_prefix or plan_params.get("prefix") or "" payload = { "prefix": plan_prefix, "geneFamily": family or plan_params.get("geneFamily") or "", "targetSpecies": target_species, "referenceSpecies": [primary], "refSpeciesList": ref_side, "comparativeSpecies": comparative, "scopeMode": scope_mode, "analysisRoute": "multi_species", "blastEngine": str(plan_params.get("blastEngine") or (intent.get("params") or {}).get("blastEngine") or "diamond").strip().lower() or "diamond", "interspeciesPlotMode": str(plan_params.get("interspeciesPlotMode") or (intent.get("params") or {}).get("interspeciesPlotMode") or "both").strip().lower() or "both", } if payload["interspeciesPlotMode"] not in {"multi", "pairwise", "both"}: payload["interspeciesPlotMode"] = "both" if payload["blastEngine"] not in {"diamond", "blastp"}: payload["blastEngine"] = "diamond" plan_params.update(payload) intent.update({ "prefix": payload["prefix"], "gene_family": payload["geneFamily"], "target_species": target_species, "reference_species": [primary], "comparative_species": comparative, "scopeMode": scope_mode, }) intent_params = intent.setdefault("params", {}) intent_params.update(payload) stale_keys = ( "stageContext", "stage_context", "scopeId", "scope_id", "memberSetId", "member_set_id", "output_prefix", "legacy_output_prefix", "scopeFile", "scope_file", ) for detail in stages_detail: if not isinstance(detail, dict): continue stage_id = detail.get("id") or detail.get("name") or "" if stage_id != "interspecies_synteny": continue detail_params = {**plan_params, **(detail.get("params") or {}), **payload} for stale_key in stale_keys: detail_params.pop(stale_key, None) detail["params"] = enrich_stage_params_with_context(stage_id, payload["prefix"], detail_params) print(f"[Agent Plan] Interspecies scope defaults: targets={target_species}, ref_side={ref_side}") def _ensure_gene_family_scope_file(job_id: str, prefix: str, stage: str, params: Dict[str, Any]) -> tuple[str, str]: if stage not in GENE_FAMILY_SCOPE_STAGES: return "", "" input_dir = INPUT_ROOT / job_id workspace = WORKSPACE_ROOT / job_id if not input_dir.exists() or not (input_dir / "config.xlsx").exists(): return "", "" scope_mode = str(params.get("scopeMode") or params.get("scope_mode") or "").strip() or _default_scope_mode_for_stage(stage) target_species = params.get("targetSpecies") or params.get("target_species") or [] reference_species = params.get("referenceSpecies") or params.get("reference_species") or [] comparative_species = params.get("comparativeSpecies") or params.get("comparative_species") or [] target_species = _canonicalize_species_codes_for_job(job_id, _param_list(target_species)) reference_species = _canonicalize_species_codes_for_job(job_id, _param_list(reference_species)) comparative_species = _canonicalize_species_codes_for_job(job_id, _param_list(comparative_species)) ref_species_list = _canonicalize_species_codes_for_job(job_id, _param_list(params.get("refSpeciesList") or params.get("ref_species_list") or [])) identify_blast_refs = _canonicalize_species_codes_for_job(job_id, _param_list(params.get("identifyBlastReferenceSpecies") or params.get("identify_blast_reference_species") or [])) primary_refs = _canonicalize_species_codes_for_job(job_id, _param_list(params.get("primaryReferenceSpecies") or params.get("primary_reference_species") or [])) params["targetSpecies"] = target_species params["referenceSpecies"] = reference_species params["comparativeSpecies"] = comparative_species params["refSpeciesList"] = ref_species_list if identify_blast_refs: params["identifyBlastReferenceSpecies"] = identify_blast_refs if primary_refs: params["primaryReferenceSpecies"] = primary_refs if stage == "interspecies_synteny": ref_side = _param_list(ref_species_list) if ref_side: reference_species = _param_list(reference_species) or [ref_side[0]] targets = _param_list(target_species) extra_refs = [sp for sp in ref_side if sp not in reference_species and sp not in targets] comparative_species = list(dict.fromkeys([*_param_list(comparative_species), *extra_refs])) if comparative_species and scope_mode == "target_plus_reference": scope_mode = "target_plus_reference_plus_comparative" elif comparative_species and scope_mode == "target_only": scope_mode = "target_plus_comparative" if scope_mode == "target_plus_reference_plus_comparative" and not _param_list(comparative_species): scope_mode = "target_plus_reference" if not ref_side and not _param_list(reference_species): options = _ordered_species_options(job_id) ref_side = _param_list(options.get("refs")) targets = _param_list(target_species) ref_side = [sp for sp in ref_side if sp not in targets] if ref_side: reference_species = [ref_side[0]] comparative_species = list(dict.fromkeys([*_param_list(comparative_species), *ref_side[1:]])) if len(ref_side) > 1: scope_mode = "target_plus_reference_plus_comparative" elif not reference_species: reference_species = ref_species_list or [] params["referenceSpecies"] = reference_species family = str(params.get("geneFamily") or params.get("family") or "").strip() prefix_species, _prefix_family = _prefix_species_family(prefix) if not target_species and prefix_species and prefix_species not in {"Target", "Multi"}: target_species = [prefix_species] print(f"[Scope] targetSpecies empty for {stage}; derived from prefix: {target_species}") family = _resolve_authoritative_gene_family(job_id, family, prefix, params, subfamily=(stage == "reference_classification")) if target_species: prefix = f"{target_species[0]}.{family}" if len(_param_list(target_species)) == 1 else f"Target.{family}" elif prefix_species and prefix_species not in {"Target", "Multi"}: prefix = f"{prefix_species}.{family}" else: prefix = f"Target.{family}" params["geneFamily"] = family params["prefix"] = prefix selected_species_codes = {sp.lower() for sp in [*_param_list(target_species), *_param_list(reference_species), *_param_list(comparative_species)]} if family.lower() in selected_species_codes: raise ValueError(f"Scope identity conflict: planned family={family} is a species code in this job, not a gene family.") analysis_route = str(params.get("analysisRoute") or params.get("analysis_route") or "").strip() if stage == "reference_classification" and analysis_route == "subfamily_prediction": if not _param_list(reference_species): reference_species = _reference_species_from_config(job_id, family, _param_list(target_species)) if reference_species: params["referenceSpecies"] = reference_species if not _param_list(reference_species): raise ValueError(f"Subfamily prediction requires referenceSpecies for Family={family}; none were resolved from config.xlsx.") scope_mode = "target_plus_reference" params["scopeMode"] = scope_mode scope = build_gene_family_scope( input_dir=input_dir, workspace=workspace, prefix=prefix, family=family, target_species=target_species, reference_species=reference_species, comparative_species=comparative_species, scope_mode=scope_mode, ) analysis_family = str(((scope.analysis or {}).get("Family") if isinstance(scope.analysis, dict) else "") or "").strip() if analysis_family and analysis_family != family: raise ValueError(f"Scope identity conflict: scope family={family}, config Analysis.Family={analysis_family}.") existing = str(params.get("scopeFile") or params.get("scope_file") or "").strip() if existing: existing_path = Path(existing) try: data = json.loads(existing_path.read_text(encoding="utf-8")) existing_scope_id = str(data.get("scope_id") or "").strip() if existing_scope_id == scope.scope_id: return existing, existing_scope_id print( f"[WARN] Ignoring stale scopeFile for {stage}: {existing} " f"(file scope_id={existing_scope_id or ''}, expected={scope.scope_id}); regenerating" ) except Exception as exc: print(f"[WARN] Ignoring unreadable scopeFile for {stage}: {existing} ({exc}); regenerating from request params") output = scope_manifest_path(workspace, prefix or scope.prefix, scope.scope_id) write_scope(scope, output) return str(output), scope.scope_id def _prefix_species_family(prefix: str) -> tuple[str, str]: prefix = str(prefix or "").strip() if "." in prefix: species, family = prefix.split(".", 1) return species.strip(), family.strip() return "", prefix def _resolve_population_request_path(value: str) -> Path: path = Path(str(value or "").strip()).expanduser() if not path.is_absolute(): path = PROJECT_ROOT / path return path.resolve(strict=False) def _auto_bind_population_inputs(job_id: str) -> tuple[Path | None, Path | None]: """Bind one unambiguous population VCF and optional group table for a job.""" input_dir = INPUT_ROOT / str(job_id or "").strip() if not input_dir.is_dir(): return None, None vcfs = sorted( path.resolve() for path in input_dir.iterdir() if path.is_file() and path.name.lower().endswith((".vcf", ".vcf.gz")) ) groups = sorted( path.resolve() for path in input_dir.iterdir() if path.is_file() and path.name.lower().endswith((".groups.tsv", ".group.tsv", ".sample_group.tsv")) ) return (vcfs[0] if len(vcfs) == 1 else None), (groups[0] if len(groups) == 1 else None) def _materialize_population_scope_params( job_id: str, stage: str, prefix: str, params: Dict[str, Any], ) -> Dict[str, str]: vcf_value = str(params.get("populationVcf") or "").strip() vcf = _resolve_population_request_path(vcf_value) if vcf_value else None group_value = str(params.get("populationSampleGroup") or "").strip() group = _resolve_population_request_path(group_value) if group_value else None resolved = resolve_population_manifest_identity( workspace=WORKSPACE_ROOT / job_id, stage=stage, prefix=prefix, genotype_scope_id=str(params.get("genotypeScopeId") or params.get("genotype_scope_id") or ""), analysis_scope_id=str(params.get("analysisScopeId") or params.get("analysis_scope_id") or ""), expected_vcf_sha256=sha256_file(vcf) if vcf is not None and vcf.is_file() else "", expected_group_sha256=( sha256_file(group) if stage in {"pop_pca", "pop_ld_decay"} and group is not None and group.is_file() else "" ), ) return { key: value for key, value in resolved.items() if key in {"genotypeScopeId", "analysisScopeId", "runId"} } def _resolve_wgs_execute_prefix( *, job_id: str, stage: str, prefix: str, params: Dict[str, Any], ) -> str: if not stage.startswith("wgs_") or stage in {"wgs_qc", "wgs_index"}: return prefix from gfa_core.wgs_upstream_manifest import resolve_unique_wgs_prefix return resolve_unique_wgs_prefix( stage=stage, job_id=job_id, params=params, workspace=WORKSPACE_ROOT / job_id, ) def _api_semantic_preflight(job_id: str, stage: str, prefix: str, params: Dict[str, Any]) -> Dict[str, Any]: """Read-only semantic preflight for direct /api/execute submissions.""" if stage.startswith("wgs_") and stage not in {"wgs_qc", "wgs_index"}: from gfa_core.wgs_upstream_manifest import ( WgsUpstreamManifestError, materialize_wgs_identity_params, ) try: materialized = materialize_wgs_identity_params( stage=stage, job_id=job_id, prefix=prefix, params=params, workspace=WORKSPACE_ROOT / job_id, ) except WgsUpstreamManifestError as exc: return { "valid": False, "issues": [{ "severity": "error", "code": "invalid_wgs_manifest_identity", "message": str(exc), }], "decisions": {}, "materialized_params": {}, } return { "valid": True, "issues": [], "decisions": {}, "materialized_params": materialized, } if stage in POPULATION_STAGES: vcf_value = str(params.get("populationVcf") or "").strip() vcf = _resolve_population_request_path(vcf_value) if vcf_value else None group_value = str(params.get("populationSampleGroup") or "").strip() group = _resolve_population_request_path(group_value) if group_value else None if vcf is None or not vcf.is_file(): auto_vcf, auto_group = _auto_bind_population_inputs(job_id) if vcf is None and auto_vcf is not None: vcf = auto_vcf vcf_value = str(vcf) if group is None and auto_group is not None: group = auto_group group_value = str(group) issues = [] if vcf is None or not vcf.is_file(): issues.append({ "severity": "error", "code": "missing_population_vcf", "message": f"Population VCF does not exist: {vcf_value or ''}", }) if group is not None and not group.is_file(): issues.append({ "severity": "error", "code": "missing_population_sample_group", "message": f"Population sample-group table does not exist: {group_value}", }) materialized_params = {} if not issues: try: materialized_params = { "populationVcf": str(vcf), "populationSampleGroup": str(group) if group is not None else "", **_materialize_population_scope_params(job_id, stage, prefix, params), } except PopulationManifestError as exc: issues.append({ "severity": "error", "code": "invalid_population_manifest_identity", "message": str(exc), }) return { "valid": not issues, "issues": issues, "decisions": {}, "materialized_params": materialized_params, } if stage == "gwas_fmt": resolution = resolve_gwas_fmt_input( job_id=job_id, prefix=prefix, params=params, project_root=PROJECT_ROOT, ) if not resolution.get("valid"): return resolution decision = (resolution.get("decisions") or {}).get("external_vcf") or {} params["gwasVcf"] = decision.get("path", "") semantic = semantic_external_vcf_preflight(params) tassel_decision = (semantic.get("decisions") or {}).get("tassel") or {} if tassel_decision.get("executable") and tassel_decision.get("path"): params["tasselPipeline"] = tassel_decision["path"] semantic["candidates"] = resolution.get("candidates") or [] if semantic.get("decisions", {}).get("external_vcf"): semantic["decisions"]["external_vcf"].update({ key: decision[key] for key in ("source", "source_manifest", "sha256", "genotype_scope_id") if decision.get(key) }) return semantic if stage == "gwas_kinship": try: from gfa_core.gwas_upstream_manifest import resolve_upstream_artifact decision = resolve_upstream_artifact( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, producer_stage="gwas_fmt", artifact_role="sorted_vcf", expected_scopes={ "genotype_scope_id": str(params.get("genotypeScopeId") or ""), }, ) return { "valid": True, "issues": [], "decisions": {"sorted_vcf": decision}, } except Exception as exc: return { "valid": False, "issues": [{ "severity": "error", "code": "invalid_gwas_fmt_upstream", "message": str(exc), "suggestion": "先完成同一 job、同一 genotype scope 的 gwas_fmt 阶段。", }], "decisions": {}, } if stage == "gwas_trait": from gfa_core.gwas_root_inputs import resolve_root_input resolution = resolve_root_input( project_root=PROJECT_ROOT, job_id=job_id, role="phenotype_table", selected=str(params.get("phenotypeFile") or params.get("phenotype_file") or ""), ) if not resolution.get("valid"): return { "valid": False, "needs_confirmation": bool(resolution.get("needs_confirmation")), "issues": resolution.get("issues") or [], "decisions": {"phenotype_table": resolution.get("decision") or {}}, "candidates": resolution.get("candidates") or [], } decision = resolution["decision"] phenotype_path = Path(decision["path"]) phenotype_lines = phenotype_path.read_text(encoding="utf-8-sig").splitlines() first_line = phenotype_lines[0] if phenotype_lines else "" columns = [part.strip() for part in first_line.split("\t")] if "\t" in first_line else [] sample_column = str(params.get("phenotypeSampleColumn") or params.get("sample_column") or "").strip() trait_column = str(params.get("phenotypeTraitColumn") or params.get("trait_column") or "").strip() if not sample_column and not trait_column and len(columns) == 2: sample_column, trait_column = columns if sample_column not in columns or trait_column not in columns or sample_column == trait_column: return { "valid": False, "needs_confirmation": len(columns) > 2, "issues": [{ "severity": "warning" if len(columns) > 2 else "error", "code": "phenotype_columns_require_confirmation", "message": "请选择 phenotype 的样本列和性状列。", }], "decisions": {"phenotype_table": decision, "columns": columns}, "candidates": resolution.get("candidates") or [], } trait_name = str(params.get("phenotypeTraitName") or params.get("trait_name") or trait_column).strip() materialized = { "phenotypeFile": str(phenotype_path), "phenotypeSampleColumn": sample_column, "phenotypeTraitColumn": trait_column, "phenotypeTraitName": trait_name, } return { "valid": True, "needs_confirmation": False, "issues": [], "decisions": {"phenotype_table": {"resolved": True, **decision}, "columns": columns}, "materialized_params": materialized, } if stage == "gwas_cov": from gfa_core.gwas_root_inputs import resolve_root_pair resolution = resolve_root_pair( project_root=PROJECT_ROOT, job_id=job_id, first_role="admixture_q", second_role="sample_fam", first_selected=str(params.get("gwasQFile") or params.get("gwas_q_file") or ""), second_selected=str(params.get("gwasFamFile") or params.get("gwas_fam_file") or ""), ) if not resolution.get("valid"): return { "valid": False, "needs_confirmation": bool(resolution.get("needs_confirmation")), "issues": resolution.get("issues") or [], "decisions": {}, } try: k = int(params.get("gwasQK") or params.get("gwas_q_k") or 2) except (TypeError, ValueError): k = 0 source_scope = str(resolution.get("source_genotype_scope_id") or "") requested_scope = str(params.get("genotypeScopeId") or params.get("genotype_scope_id") or "") issues = [] if k < 2: issues.append({"severity": "error", "code": "invalid_gwas_q_k", "message": "ADMIXTURE K 必须至少为2。"}) if not re.fullmatch(r"genotype:[0-9a-f]{64}", source_scope): issues.append({"severity": "error", "code": "unverified_covariate_lineage", "message": "Q/FAM 缺少可验证的源 genotype scope。"}) if requested_scope and source_scope != requested_scope: issues.append({"severity": "error", "code": "covariate_genotype_scope_mismatch", "message": "Q/FAM 与当前 genotype scope 不一致。"}) materialized = { "gwasQFile": resolution["first"]["path"], "gwasFamFile": resolution["second"]["path"], "gwasQK": k, "genotypeScopeId": source_scope, } return { "valid": not issues, "needs_confirmation": False, "issues": issues, "decisions": { "admixture_q": {"resolved": True, **resolution["first"]}, "sample_fam": {"resolved": True, **resolution["second"]}, }, "materialized_params": materialized, } if stage == "gwas_mlm": from gfa_core.gwas_upstream_manifest import resolve_upstream_artifact genotype_scope = str(params.get("genotypeScopeId") or params.get("genotype_scope_id") or "") phenotype_scope = str(params.get("phenotypeScopeId") or params.get("phenotype_scope_id") or "") covariate_scope = str(params.get("covariateScopeId") or params.get("covariate_scope_id") or "") requested_scopes = (genotype_scope, phenotype_scope, covariate_scope) if any(requested_scopes) and not all(requested_scopes): return {"valid": False, "issues": [{ "severity": "error", "code": "invalid_gwas_mlm_upstream", "message": "gwas_mlm scope identity is partial; provide all three scopes or leave all three empty.", }], "decisions": {}} expected_scopes = { "sorted_vcf": {"genotype_scope_id": genotype_scope}, "kinship_matrix": {"genotype_scope_id": genotype_scope}, "trait_table": {"phenotype_scope_id": phenotype_scope}, "covariate_table": { "genotype_scope_id": genotype_scope, "covariate_scope_id": covariate_scope, }, } if all(requested_scopes) else { "sorted_vcf": {}, "kinship_matrix": {}, "trait_table": {}, "covariate_table": {}, } requests = { "sorted_vcf": ("gwas_fmt", "sorted_vcf"), "kinship_matrix": ("gwas_kinship", "kinship_matrix"), "trait_table": ("gwas_trait", "trait_table"), "covariate_table": ("gwas_cov", "covariate_table"), } try: decisions = { role: resolve_upstream_artifact( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, producer_stage=producer, artifact_role=artifact_role, expected_scopes=expected_scopes[role], ) for role, (producer, artifact_role) in requests.items() } if not any(requested_scopes): genotype_scopes = { str(decisions[role].get("genotype_scope_id") or "") for role in ("sorted_vcf", "kinship_matrix", "covariate_table") } if len(genotype_scopes) != 1: raise ValueError("GWAS upstream manifests have conflicting genotype scopes") genotype_scope = genotype_scopes.pop() phenotype_scope = str(decisions["trait_table"].get("phenotype_scope_id") or "") covariate_scope = str(decisions["covariate_table"].get("covariate_scope_id") or "") for value, pattern, label in ( (genotype_scope, r"genotype:[0-9a-f]{64}", "genotype"), (phenotype_scope, r"phenotype:[0-9a-f]{64}", "phenotype"), (covariate_scope, r"covariate:[0-9a-f]{64}", "covariate"), ): if not re.fullmatch(pattern, value): raise ValueError(f"GWAS upstream manifest {label} scope is invalid") except Exception as exc: return {"valid": False, "issues": [{ "severity": "error", "code": "invalid_gwas_mlm_upstream", "message": str(exc), "suggestion": "先完成同一 job 和精确 scientific scope 的四个 GWAS 上游阶段。", }], "decisions": {}} return { "valid": True, "needs_confirmation": False, "issues": [], "decisions": decisions, "materialized_params": { "gwasSortedVcf": decisions["sorted_vcf"]["path"], "gwasKinshipFile": decisions["kinship_matrix"]["path"], "gwasTraitFile": decisions["trait_table"]["path"], "gwasCovariateFile": decisions["covariate_table"]["path"], "genotypeScopeId": genotype_scope, "phenotypeScopeId": phenotype_scope, "covariateScopeId": covariate_scope, }, } if stage == "gwas_plot": from gfa_core.gwas_upstream_manifest import resolve_gwas_plot_upstream, resolve_upstream_artifact genotype_scope = str(params.get("genotypeScopeId") or params.get("genotype_scope_id") or "") phenotype_scope = str(params.get("phenotypeScopeId") or params.get("phenotype_scope_id") or "") covariate_scope = str(params.get("covariateScopeId") or params.get("covariate_scope_id") or "") source_model_scope = str(params.get("sourceModelScopeId") or params.get("source_model_scope_id") or "") requested_scopes = (genotype_scope, phenotype_scope, covariate_scope, source_model_scope) if any(requested_scopes) and not all(requested_scopes): return {"valid": False, "issues": [{ "severity": "error", "code": "invalid_gwas_plot_upstream", "message": "gwas_plot scope identity is partial; provide all four scopes or leave all four empty.", }], "decisions": {}} try: if not any(requested_scopes): identity = resolve_upstream_artifact( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, producer_stage="gwas_mlm", artifact_role="mlm_pvalues", ) genotype_scope = str(identity.get("genotype_scope_id") or "") phenotype_scope = str(identity.get("phenotype_scope_id") or "") covariate_scope = str(identity.get("covariate_scope_id") or "") source_model_scope = str(identity.get("model_scope_id") or "") for value, pattern, label in ( (genotype_scope, r"genotype:[0-9a-f]{64}", "genotype"), (phenotype_scope, r"phenotype:[0-9a-f]{64}", "phenotype"), (covariate_scope, r"covariate:[0-9a-f]{64}", "covariate"), (source_model_scope, r"gwas-model:[0-9a-f]{64}", "source-model"), ): if not re.fullmatch(pattern, value): raise ValueError(f"gwas_mlm manifest {label} scope is invalid") resolved = resolve_gwas_plot_upstream( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, genotype_scope_id=genotype_scope, phenotype_scope_id=phenotype_scope, covariate_scope_id=covariate_scope, source_model_scope_id=source_model_scope, ) for role, camel_key, snake_key in ( ("mlm_pvalues", "gwasMlmPvalues", "gwas_mlm_pvalues"), ("cmlm_pvalues", "gwasCmlmPvalues", "gwas_cmlm_pvalues"), ): supplied = str(params.get(camel_key) or params.get(snake_key) or "") if supplied and Path(supplied).resolve() != Path(resolved[role]["path"]).resolve(): raise ValueError(f"{role} does not match the authoritative gwas_mlm Manifest") count = resolved["candidate_marker_count"] for model, key in ( ("mlm", "gwasMlmTestedHypotheses"), ("cmlm", "gwasCmlmTestedHypotheses"), ): supplied = params.get(key) if supplied not in (None, ""): if isinstance(supplied, bool) or int(supplied) != count: raise ValueError(f"{model} tested hypotheses does not match candidate_marker_count") except Exception as exc: return {"valid": False, "issues": [{ "severity": "error", "code": "invalid_gwas_plot_upstream", "message": str(exc), "suggestion": "先完成同一 job 和精确五范围的 gwas_mlm 阶段。", }], "decisions": {}} decisions = { "mlm_pvalues": resolved["mlm_pvalues"], "cmlm_pvalues": resolved["cmlm_pvalues"], } return { "valid": True, "needs_confirmation": False, "issues": [], "decisions": decisions, "materialized_params": { "gwasMlmPvalues": resolved["mlm_pvalues"]["path"], "gwasCmlmPvalues": resolved["cmlm_pvalues"]["path"], "gwasMlmTestedHypotheses": count, "gwasCmlmTestedHypotheses": count, "gwasRscript": str(params.get("gwasRscript") or ASAW_RSCRIPT), "genotypeScopeId": genotype_scope, "phenotypeScopeId": phenotype_scope, "covariateScopeId": covariate_scope, "sourceModelScopeId": source_model_scope, }, } if stage == "gwas_emmax": from gfa_core.gwas_upstream_manifest import resolve_gwas_emmax_upstream, resolve_upstream_artifact genotype_scope = str(params.get("genotypeScopeId") or params.get("genotype_scope_id") or "") phenotype_scope = str(params.get("phenotypeScopeId") or params.get("phenotype_scope_id") or "") requested_scopes = (genotype_scope, phenotype_scope) if any(requested_scopes) and not all(requested_scopes): return {"valid": False, "issues": [{ "severity": "error", "code": "invalid_gwas_emmax_upstream", "message": "gwas_emmax scope identity is partial; provide both scopes or leave both empty.", }], "decisions": {}} try: if not any(requested_scopes): fmt_identity = resolve_upstream_artifact( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, producer_stage="gwas_fmt", artifact_role="sorted_vcf", ) trait_identity = resolve_upstream_artifact( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, producer_stage="gwas_trait", artifact_role="trait_table", ) genotype_scope = str(fmt_identity.get("genotype_scope_id") or "") phenotype_scope = str(trait_identity.get("phenotype_scope_id") or "") if not re.fullmatch(r"genotype:[0-9a-f]{64}", genotype_scope): raise ValueError("gwas_fmt manifest genotype scope is invalid") if not re.fullmatch(r"phenotype:[0-9a-f]{64}", phenotype_scope): raise ValueError("gwas_trait manifest phenotype scope is invalid") resolved = resolve_gwas_emmax_upstream( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, genotype_scope_id=genotype_scope, phenotype_scope_id=phenotype_scope, ) for role, camel_key, snake_key in ( ("sorted_vcf", "gwasSortedVcf", "gwas_sorted_vcf"), ("trait_table", "gwasTraitFile", "gwas_trait_file"), ): supplied = str(params.get(camel_key) or params.get(snake_key) or "") if supplied and Path(supplied).resolve() != Path(resolved[role]["path"]).resolve(): raise ValueError(f"{role} does not match its authoritative upstream Manifest") requested_pcs = params.get("gwasRequestedPcs", params.get("gwas_requested_pcs", 3)) if isinstance(requested_pcs, bool) or int(requested_pcs) < 0: raise ValueError("gwasRequestedPcs must be a non-negative integer") except Exception as exc: return {"valid": False, "issues": [{ "severity": "error", "code": "invalid_gwas_emmax_upstream", "message": str(exc), "suggestion": "先完成同一 job 和精确 genotype/phenotype scope 的 gwas_fmt 与 gwas_trait 阶段。", }], "decisions": {}} return { "valid": True, "needs_confirmation": False, "issues": [], "decisions": resolved, "materialized_params": { "gwasSortedVcf": resolved["sorted_vcf"]["path"], "gwasTraitFile": resolved["trait_table"]["path"], "genotypeScopeId": genotype_scope, "phenotypeScopeId": phenotype_scope, "gwasPlink": str(params.get("gwasPlink") or ASAW_BIN / "plink"), "gwasEmmax": str(params.get("gwasEmmax") or GWAS_EMMAX_BIN), "gwasRscript": str(params.get("gwasRscript") or ASAW_RSCRIPT), "gwasRequestedPcs": int(requested_pcs), }, } if stage == "gwas_emplot": from gfa_core.gwas_upstream_manifest import resolve_gwas_emplot_upstream, resolve_upstream_artifact genotype_scope = str(params.get("genotypeScopeId") or params.get("genotype_scope_id") or "") phenotype_scope = str(params.get("phenotypeScopeId") or params.get("phenotype_scope_id") or "") source_model_scope = str(params.get("sourceModelScopeId") or params.get("source_model_scope_id") or "") requested_scopes = (genotype_scope, phenotype_scope, source_model_scope) if any(requested_scopes) and not all(requested_scopes): return {"valid": False, "issues": [{ "severity": "error", "code": "invalid_gwas_emplot_upstream", "message": "gwas_emplot scope identity is partial; provide all three scopes or leave all three empty.", }], "decisions": {}} try: if not any(requested_scopes): identity = resolve_upstream_artifact( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, producer_stage="gwas_emmax", artifact_role="emmax_pvalues", ) genotype_scope = str(identity.get("genotype_scope_id") or "") phenotype_scope = str(identity.get("phenotype_scope_id") or "") source_model_scope = str(identity.get("model_scope_id") or "") for value, pattern, label in ( (genotype_scope, r"genotype:[0-9a-f]{64}", "genotype"), (phenotype_scope, r"phenotype:[0-9a-f]{64}", "phenotype"), (source_model_scope, r"gwas-model:[0-9a-f]{64}", "source-model"), ): if not re.fullmatch(pattern, value): raise ValueError(f"gwas_emmax manifest {label} scope is invalid") resolved = resolve_gwas_emplot_upstream( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, genotype_scope_id=genotype_scope, phenotype_scope_id=phenotype_scope, source_model_scope_id=source_model_scope, ) supplied_path = str(params.get("gwasEmmaxPvalues") or params.get("gwas_emmax_pvalues") or "") if supplied_path and Path(supplied_path).resolve() != Path(resolved["emmax_pvalues"]["path"]).resolve(): raise ValueError("emmax_pvalues does not match the authoritative gwas_emmax Manifest") supplied_manifest = str(params.get("gwasEmmaxManifest") or params.get("gwas_emmax_manifest") or "") if supplied_manifest and Path(supplied_manifest).resolve() != Path(resolved["source_manifest"]).resolve(): raise ValueError("emmax Manifest does not match the authoritative gwas_emmax Manifest") supplied_count = params.get("gwasEmmaxTestedHypotheses", params.get("gwas_emmax_tested_hypotheses")) if supplied_count not in (None, "") and ( isinstance(supplied_count, bool) or int(supplied_count) != resolved["emmax_tested_count"] ): raise ValueError("tested hypotheses does not match emmax_tested_count") except Exception as exc: return {"valid": False, "issues": [{ "severity": "error", "code": "invalid_gwas_emplot_upstream", "message": str(exc), "suggestion": "先完成同一 job 和精确 scientific scope 的 gwas_emmax 阶段。", }], "decisions": {}} return { "valid": True, "needs_confirmation": False, "issues": [], "decisions": resolved, "materialized_params": { "gwasEmmaxPvalues": resolved["emmax_pvalues"]["path"], "gwasEmmaxManifest": resolved["source_manifest"], "gwasEmmaxTestedHypotheses": resolved["emmax_tested_count"], "gwasRscript": str(params.get("gwasRscript") or ASAW_RSCRIPT), "genotypeScopeId": genotype_scope, "phenotypeScopeId": phenotype_scope, "sourceModelScopeId": source_model_scope, }, } if stage == "gwas_glm": from gfa_core.gwas_upstream_manifest import resolve_gwas_glm_upstream, resolve_upstream_artifact genotype_scope = str(params.get("genotypeScopeId") or params.get("genotype_scope_id") or "") phenotype_scope = str(params.get("phenotypeScopeId") or params.get("phenotype_scope_id") or "") covariate_scope = str(params.get("covariateScopeId") or params.get("covariate_scope_id") or "") requested_scopes = (genotype_scope, phenotype_scope, covariate_scope) if any(requested_scopes) and not all(requested_scopes): return {"valid": False, "issues": [{ "severity": "error", "code": "invalid_gwas_glm_upstream", "message": "gwas_glm scope identity is partial; provide all three scopes or leave all three empty.", }], "decisions": {}} try: if not any(requested_scopes): fmt_identity = resolve_upstream_artifact( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, producer_stage="gwas_fmt", artifact_role="sorted_vcf", ) trait_identity = resolve_upstream_artifact( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, producer_stage="gwas_trait", artifact_role="trait_table", ) cov_identity = resolve_upstream_artifact( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, producer_stage="gwas_cov", artifact_role="covariate_table", ) genotype_scope = str(fmt_identity.get("genotype_scope_id") or "") phenotype_scope = str(trait_identity.get("phenotype_scope_id") or "") covariate_scope = str(cov_identity.get("covariate_scope_id") or "") for value, pattern, label in ( (genotype_scope, r"genotype:[0-9a-f]{64}", "genotype"), (phenotype_scope, r"phenotype:[0-9a-f]{64}", "phenotype"), (covariate_scope, r"covariate:[0-9a-f]{64}", "covariate"), ): if not re.fullmatch(pattern, value): raise ValueError(f"GWAS upstream manifest {label} scope is invalid") resolved = resolve_gwas_glm_upstream( workspace=WORKSPACE_ROOT / job_id, prefix=prefix, genotype_scope_id=genotype_scope, phenotype_scope_id=phenotype_scope, covariate_scope_id=covariate_scope, ) supplied = { "sorted_vcf": ("gwasSortedVcf", "gwasVcfManifest"), "trait_table": ("gwasTraitFile", "gwasTraitManifest"), "covariate_table": ("gwasCovariateFile", "gwasCovariateManifest"), } for role, (path_key, manifest_key) in supplied.items(): selected_path = str(params.get(path_key) or "") selected_manifest = str(params.get(manifest_key) or "") if selected_path and Path(selected_path).resolve() != Path(resolved[role]["path"]).resolve(): raise ValueError(f"{role} does not match its authoritative upstream Manifest") if selected_manifest and Path(selected_manifest).resolve() != Path(resolved[role]["source_manifest"]).resolve(): raise ValueError(f"{role} producer Manifest does not match the authoritative upstream Manifest") except Exception as exc: return {"valid": False, "issues": [{ "severity": "error", "code": "invalid_gwas_glm_upstream", "message": str(exc), "suggestion": "先完成同一 job 和精确 scientific scope 的 gwas_fmt、gwas_trait 与 gwas_cov 阶段。", }], "decisions": {}} return { "valid": True, "needs_confirmation": False, "issues": [], "decisions": resolved, "materialized_params": { "gwasSortedVcf": resolved["sorted_vcf"]["path"], "gwasVcfManifest": resolved["sorted_vcf"]["source_manifest"], "gwasTraitFile": resolved["trait_table"]["path"], "gwasTraitManifest": resolved["trait_table"]["source_manifest"], "gwasCovariateFile": resolved["covariate_table"]["path"], "gwasCovariateManifest": resolved["covariate_table"]["source_manifest"], "tasselPipeline": str(params.get("tasselPipeline") or "run_pipeline.pl"), "tasselXmx": str(params.get("tasselXmx") or "10g"), "genotypeScopeId": genotype_scope, "phenotypeScopeId": phenotype_scope, "covariateScopeId": covariate_scope, }, } capability = get_stage_capability(stage) if not capability: return {"valid": True, "issues": [], "decisions": {}} context = build_project_context(job_id=job_id, prefix=prefix, project_root=PROJECT_ROOT) prefix_species, prefix_family = _prefix_species_family(prefix) family = str(params.get("geneFamily") or params.get("family") or prefix_family or context.family or "").strip() target_species = params.get("targetSpecies") or params.get("target_species") or [] if isinstance(target_species, str): target_species = [item.strip() for item in re.split(r"[,;，；\s]+", target_species) if item.strip()] reference_species = params.get("referenceSpecies") or params.get("reference_species") or [] if isinstance(reference_species, str): reference_species = [item.strip() for item in re.split(r"[,;，；\s]+", reference_species) if item.strip()] comparative_species = params.get("comparativeSpecies") or params.get("comparative_species") or [] if isinstance(comparative_species, str): comparative_species = [item.strip() for item in re.split(r"[,;，；\s]+", comparative_species) if item.strip()] # Make semantic resolver score candidates against the current request scope, # not the whole job config. if target_species: context.target_species = list(target_species) context.reference_species = list(reference_species) context.comparative_species = list(comparative_species) setattr(context, "expected_scope_mode", str(params.get("scopeMode") or params.get("scope_mode") or "")) setattr(context, "expected_analysis_route", str(params.get("analysisRoute") or params.get("analysis_route") or "")) resolver = ArtifactResolver(context) requires_confirmed_subfamily = bool(params.get("requiresConfirmedSubfamily") or params.get("requires_confirmed_subfamily")) analysis_route = str(params.get("analysisRoute") or params.get("analysis_route") or "").strip() if ( stage == "reference_classification" and analysis_route == "subfamily_prediction" and target_species and reference_species ): return { "valid": True, "needs_confirmation": False, "issues": [], "decisions": { "reference_classification": { "resolved": True, "reason": "per-target target+reference evidence is generated inside reference_classification", } }, "context": { "family": context.family, "target_species": context.target_species, "reference_species": context.reference_species, "comparative_species": context.comparative_species, "config_path": context.config_path, }, } def default_species() -> str: if len(target_species) == 1: return target_species[0] if prefix_species and prefix_species.lower() not in {"multi", "target"}: return prefix_species if context.target_species and len(context.target_species) == 1: return context.target_species[0] return "" if stage == "sequence_extract_by_id": ids_file = str(params.get("sequenceExtractIdsFile") or "").strip() inline_ids = str(params.get("sequenceExtractIds") or "").strip() clean_headers = bool(params.get("sequenceExtractCleanHeaders")) if ids_file or inline_ids or clean_headers: return { "valid": True, "needs_confirmation": False, "issues": [], "decisions": { "sequence_extract_by_id": { "resolved": True, "reason": "utility parameters provide IDs; source FASTA may be auto-resolved from job or bundled species reference", } }, "context": { "family": context.family, "target_species": context.target_species, "reference_species": context.reference_species, "comparative_species": context.comparative_species, "config_path": context.config_path, }, } issues: List[Dict[str, Any]] = [] decisions: Dict[str, Any] = {} materialized_params: Dict[str, Any] = {} species_to_check = target_species if stage == "identify" and target_species else [default_species()] for species in species_to_check: stage_decisions = resolver.resolve_stage(stage, species=species, family=family) for need in capability.consumes: if not need.required: continue decision = stage_decisions.get(need.role) decision_key = f"{species or 'auto'}:{need.role}" decisions[decision_key] = decision.to_dict() if decision else None if decision and decision.resolved: if ( need.role == "tree" and decision.selected and decision.selected.path and not str(params.get("treeFile") or "").strip() ): materialized_params["treeFile"] = decision.selected.path if ( stage == "wgs_index" and need.role == "reference_fasta" and decision.selected and decision.selected.path and not str(params.get("wgsReferenceFasta") or "").strip() ): materialized_params["wgsReferenceFasta"] = decision.selected.path if stage == "identify" and need.role == "target_proteome" and not species: species_tokens = set() for candidate in decision.candidates: if not candidate.exists: continue name = Path(candidate.path).name.lower() if any(token in name for token in ["ref", "final_members", "final_gfmembers"]): continue token = Path(candidate.path).name.split(".", 1)[0].split("_", 1)[0] if token: species_tokens.add(token) if len(species_tokens) > 1: issues.append({ "severity": "warning", "category": "scope_selection", "code": "identify_target_species_required", "message": "Identify 阶段检测到多个目标物种蛋白，需要确认 target species，不自动开始执行。", "suggestion": "请在弹窗中选择 target species；系统会据此自动生成内部 prefix。", "auto_fixable": True, "fix_action": "select_target_species", }) continue suggestion = "请检查 config.xlsx、input、final 或先运行可生成该输入的上游阶段。" if need.role == "target_genome": suggestion = "请检查 config.xlsx 中目标物种 genome FASTA，或先运行 genome_prepare 生成目标物种 genome 输入。" elif need.role == "plant_tf_motif_database": suggestion = "请确认 data/tfbs/jaspar2026/combined/JASPAR2026_CORE_plants_non-redundant.meme 存在，或配置 PlantTFDB/JASPAR MEME 资源。" issues.append({ "severity": "error", "category": "input", "code": f"missing_{need.role}", "message": f"{stage} 缺少所需输入: {need.role} ({need.kind})", "suggestion": suggestion, "auto_fixable": need.role in {"target_proteome", "family_members_pep", "family_members_id", "target_gff", "target_genome", "stage_manifests"}, "fix_action": "artifact_resolve_or_prepare", }) context_gate = _validate_gene_family_stage_context( job_id=job_id, stage=stage, family=family, target_species=target_species, comparative_species=comparative_species, requires_confirmed_subfamily=requires_confirmed_subfamily, ) if context_gate.get("issues"): issues.extend(context_gate.get("issues") or []) errors = [issue for issue in issues if issue.get("severity") == "error"] return { "valid": not errors, "needs_confirmation": any(issue.get("code") == "identify_target_species_required" for issue in issues), "issues": issues, "decisions": decisions, "materialized_params": materialized_params, "context_gate": context_gate, "decisions_context": context_gate.get("evidence", {}), "context": { "family": context.family, "target_species": context.target_species, "reference_species": context.reference_species, "comparative_species": context.comparative_species, "config_path": context.config_path, }, } def _semantic_resolved_artifacts(semantic_preflight: Dict[str, Any]) -> Dict[str, Any]: resolved: Dict[str, Any] = {} decisions = semantic_preflight.get("decisions") if isinstance(semantic_preflight, dict) else {} if not isinstance(decisions, dict): return resolved for key, decision in decisions.items(): if not isinstance(decision, dict): continue selected = decision.get("selected") if isinstance(decision.get("selected"), dict) else None if not selected or not selected.get("path"): continue resolved[key] = { "path": selected.get("path"), "source": selected.get("source", ""), "kind": selected.get("kind", ""), "role": selected.get("role", ""), "species": selected.get("species", ""), "family": selected.get("family", ""), } return resolved def _suggest_upstream_stages_for_preflight(stage: str, semantic_preflight: Dict[str, Any]) -> List[str]: """Translate semantic missing inputs into executable upstream stages. This is intentionally deterministic. It is not a replacement for the planner; it gives the frontend enough structure to recover when a direct stage submission is blocked before a task is created. """ role_to_stage = { "tree": "phylogeny", "family_members_pep": "identify", "family_members_id": "identify", "target_proteome": "genome_prepare", "target_gff": "genome_prepare", "target_genome": "genome_prepare", "raw_protein": "genome_prepare", "raw_gff": "genome_prepare", "raw_genome": "genome_prepare", "prepared_protein": "genome_prepare", "stage_manifests": "identify", "confirmed_subfamily": "reference_classification", } suggestions: List[str] = [] for issue in semantic_preflight.get("issues", []) if isinstance(semantic_preflight, dict) else []: if not isinstance(issue, dict) or issue.get("severity") != "error": continue code = str(issue.get("code") or "") role = code.removeprefix("missing_") if code.startswith("missing_") else str(issue.get("role") or "") if code == "missing_confirmed_subfamily": role = "confirmed_subfamily" if code == "missing_target_family_members": role = "family_members_pep" upstream = role_to_stage.get(role) if upstream and upstream != stage and upstream not in suggestions: suggestions.append(upstream) if not suggestions: return [] try: from gfa_core.mas.stage_catalog import complete_dependencies expanded = complete_dependencies(suggestions) return [item for item in expanded if item != stage] except Exception: return suggestions def _run_preflight_diagnosis_for_task( job_id: str, stage: str, prefix: str, params: Dict[str, Any], semantic_preflight: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """DiagnosisAgent is disabled; return only semantic preflight context.""" return { "status": "diagnosis_disabled", "stage": stage, "prefix": prefix, "issues": [], "semantic_preflight": semantic_preflight or {}, } try: from gfa_core.mas.agents.diagnosis import DiagnosisAgent diagnosis_agent = DiagnosisAgent(None) diagnosis_prefix = "genome_prepare" if stage in {"genome_prepare", "genome_prepare_v2"} else prefix diagnosis = diagnosis_agent.diagnose_stage_gate( job_id=job_id, stage=stage, prefix=diagnosis_prefix, gate="preflight", project_root=PROJECT_ROOT, params=params, ) if semantic_preflight: diagnosis["semantic_preflight"] = semantic_preflight diagnosis["suggested_upstream_stages"] = _suggest_upstream_stages_for_preflight(stage, semantic_preflight) return diagnosis except Exception as exc: return { "status": "diagnosis_unavailable", "stage": stage, "prefix": prefix, "error": str(exc), "semantic_preflight": semantic_preflight or {}, } def _run_failure_diagnosis_for_task( job_id: str, stage: str, prefix: str, error_message: str = "", params: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """DiagnosisAgent is disabled; keep failure handling deterministic.""" return { "status": "diagnosis_disabled", "stage": stage, "prefix": prefix, "issues": [], "runtime_error": error_message, } try: from gfa_core.mas.agents.diagnosis import DiagnosisAgent diagnosis_agent = DiagnosisAgent(None) diagnosis_prefix = "genome_prepare" if stage in {"genome_prepare", "genome_prepare_v2"} else prefix diagnosis = diagnosis_agent.diagnose_stage_gate( job_id=job_id, stage=stage, prefix=diagnosis_prefix, gate="failure", project_root=PROJECT_ROOT, params=params or {}, ) if error_message: diagnosis.setdefault("runtime_error", error_message) return diagnosis except Exception as exc: return { "status": "diagnosis_unavailable", "stage": stage, "prefix": prefix, "error": str(exc), "runtime_error": error_message, } def _safe_upload_name(filename: str) -> str: name = Path(filename or "uploaded_file").name return name.replace("/", "_").replace("\\", "_") or "uploaded_file" def _is_excel_config_upload(filename: str) -> bool: suffix = Path(filename or "").suffix.lower() return suffix in {".xlsx", ".xls"} def _is_sample_info_upload(filename: str) -> bool: lower = str(filename or "").lower() suffix = Path(lower).suffix return suffix in {".xlsx", ".xlsm", ".xls", ".csv", ".tsv", ".txt"} and bool(re.search(r"(samples?_?info|sample_?metadata|metadata|sample_?group|样本|分组|信息)", lower)) def _classify_autofill_files(names: list[str]) -> dict: config_exts = { ".fa", ".fasta", ".faa", ".fna", ".ffn", ".pep", ".cds", ".gff", ".gff3", ".gtf", ".hmm", ".xlsx", ".xlsm", ".xls", } samples_ext_re = re.compile(r"\.(fastq|fq)(?:\.gz)?$", re.I) has_config = False has_samples = False for name in names: lower = str(name or "").lower() stemmed = lower[:-3] if lower.endswith(".gz") else lower suffix = Path(stemmed).suffix is_sample_info = _is_sample_info_upload(lower) if samples_ext_re.search(lower) or lower.endswith("samples.yaml") or lower.endswith("samples.yml") or is_sample_info: has_samples = True if (suffix in config_exts or lower.endswith("config.xlsx")) and not is_sample_info: has_config = True return { "config": has_config, "samples": has_samples, "types": [key for key, value in {"config": has_config, "samples": has_samples}.items() if value], } def _build_config_template_workbook(): from gfa_core.config_templates import build_config_template_workbook return build_config_template_workbook() try: from openpyxl import Workbook from openpyxl.styles import Font, PatternFill, Alignment from openpyxl.utils import get_column_letter except Exception as exc: raise RuntimeError(f"openpyxl is required to generate config.xlsx template: {exc}") wb = Workbook() ws = wb.active ws.title = "config" headers = ["Original_File_Name", "Species_Code", "Species_Name", "Gene_Family", "File_Type", "Ref", "Species_Role", "File_Scope", "Reference_Use"] rows = [ # Target species (whole-genome resources; Gene_Family can be empty) ["Os.genome.fasta", "Os", "Oryza sativa", "", "genome", "FALSE", "target", "whole_genome", ""], ["Os.gff3", "Os", "Oryza sativa", "", "gff", "FALSE", "target", "whole_genome", ""], ["Os.cds.fasta", "Os", "Oryza sativa", "", "cds", "FALSE", "target", "whole_genome", ""], ["Os.pep.fasta", "Os", "Oryza sativa", "", "pep", "FALSE", "target", "whole_genome", ""], # Reference species (whole-genome resources; downstream reference context) ["At.genome.fasta", "At", "Arabidopsis thaliana", "", "genome", "TRUE", "reference", "whole_genome", "downstream_primary"], ["At.gff3", "At", "Arabidopsis thaliana", "", "gff", "TRUE", "reference", "whole_genome", "downstream_primary"], ["At.cds.fasta", "At", "Arabidopsis thaliana", "", "cds", "TRUE", "reference", "whole_genome", "downstream_primary"], ["At.pep.fasta", "At", "Arabidopsis thaliana", "", "pep", "TRUE", "reference", "whole_genome", "downstream_primary"], # Comparative/outgroup species (whole-genome; used for multi-species context) ["Zm.gff3", "Zm", "Zea mays", "", "gff", "FALSE", "comparative", "whole_genome", ""], ["Zm.pep.fasta", "Zm", "Zea mays", "", "pep", "FALSE", "comparative", "whole_genome", ""], ["Vv.gff3", "Vv", "Vitis vinifera", "", "gff", "FALSE", "outgroup", "whole_genome", ""], ["Vv.pep.fasta", "Vv", "Vitis vinifera", "", "pep", "FALSE", "outgroup", "whole_genome", ""], # Seed reference (known family members; Gene_Family required for these rows) ["At.WRKY.pep.fasta", "At", "Arabidopsis thaliana", "WRKY", "family_pep", "TRUE", "seed_reference", "known_family", "identify_blast"], ] ws.append(headers) for row in rows: ws.append(row) header_fill = PatternFill("solid", fgColor="D9EAF7") for cell in ws[1]: cell.font = Font(bold=True) cell.fill = header_fill cell.alignment = Alignment(horizontal="center") widths = [28, 16, 28, 16, 18, 10, 18, 18, 22] for idx, width in enumerate(widths, start=1): ws.column_dimensions[get_column_letter(idx)].width = width ws.freeze_panes = "A2" readme = wb.create_sheet("README") readme_rows = [ ["config.xlsx template"], ["Download this file, fill the config sheet, then upload it together with genome/GFF/CDS/protein files."], ["The first sheet named config is read by the pipeline. Do not put notes or merged cells in that sheet."], [], ["Column", "Chinese meaning", "English description", "Required", "Example / Allowed values"], ["Original_File_Name", "uploaded original file name", "Original uploaded file name; must match the uploaded file", "Yes", "At.WRKY.pep.fa"], ["Species_Code", "species short code", "Species code; keep consistent for one species", "Yes", "At, Os, Lc"], ["Species_Name", "species scientific name", "Optional Latin/scientific name used only as an alias; filenames and Analysis species fields still use Species_Code", "Optional", "Arabidopsis thaliana"], ["Gene_Family", "gene family name", "Optional for whole-genome rows; required for known-family seed rows (e.g. At.WRKY.pep.fasta)", "Optional", "WRKY, NAC, MYB; leave blank for whole-genome resources"], ["File_Type", "file type", "File type", "Yes", "genome, gff, cds, pep, family_pep, family_cds (legacy: all_pep, protein)"], ["Ref", "legacy reference flag", "Backward compatible flag. Prefer Species_Role for new projects.", "Recommended", "TRUE or FALSE"], ["Species_Role", "species role", "Biological role in the analysis", "Recommended", "target, reference, comparative, outgroup, seed_reference"], ["File_Scope", "file scope", "Whether the file is whole-genome resource or known family seed", "Recommended", "whole_genome, known_family"], ["Reference_Use", "reference usage", "Advanced row-level override; usually leave empty and use Analysis sheet fields", "Optional", "identify_blast, downstream_primary, both, none"], [], ["File_Type notes"], ["genome", "Genome FASTA"], ["gff", "GFF or GFF3 annotation file"], ["cds", "CDS FASTA"], ["pep", "Whole proteome; used for identify and MCScanX background"], ["all_pep or protein", "Legacy whole-proteome value; treated as pep"], ["family_pep", "Known family proteins; standardized as Species.Family.pep.fasta"], ["family_cds", "Known family CDS; standardized as Species.Family.cds.fasta"], ["pep", "Compatible value: whole_genome rows become Species.pep.fasta; known_family rows become Species.Family.pep.fasta"], [], ["Naming rule"], ["whole_genome files are standardized as Species.pep.fasta, Species.cds.fasta, Species.genome.fasta, Species_final.gff3"], ["known_family seed files are standardized as Species.Family.pep.fasta or Species.Family.cds.fasta"], [], ["Upload rule"], ["You may upload any .xlsx file name. The latest uploaded Excel file will be normalized to config.xlsx."], ["If you upload another Excel file later, it replaces the current config.xlsx for this job."], [], ["Filling guide (how to think about this table)"], ["1) Genome Prepare = species-level normalization (no family binding)"], ["- For target/comparative/outgroup species, upload whole-genome resources and set File_Scope=whole_genome."], ["- For seed_reference species, you may upload known family member FASTA only (File_Scope=known_family, Gene_Family required)."], ["2) Two analysis paths are both supported"], ["- Path A (known family): you already know the family (e.g. WRKY). Use seed_reference (At.WRKY.pep.fasta) + whole-genome pep of target species; then run identify with prefix like Os.WRKY."], ["- Path B (unknown family): leave Gene_Family empty for whole-genome rows, run RNA-seq -> enrichment -> choose candidate family; then run identify using Species.pep.fasta."], ["3) Roles and what they mean"], ["- target: main species you study in the paper."], ["- comparative: additional species for cross-species comparison/synteny."], ["- outgroup: distant species for tree rooting or evolution context."], ["- seed_reference: curated known family members (usually Arabidopsis) used as seeds/validation."], ["4) Minimal rules to avoid mistakes"], ["- Do not put random notes/merged cells in the config sheet."], ["- Whole-genome rows: Gene_Family can be empty; file naming will be Species.*."], ["- Known-family rows: Gene_Family must be filled; naming will be Species.Family.*."], ["- Ref is legacy; Species_Role is recommended for new jobs."], ] for row in readme_rows: readme.append(row) readme["A1"].font = Font(bold=True, size=14) for cell in readme[5]: cell.font = Font(bold=True) cell.fill = PatternFill("solid", fgColor="E2F0D9") readme.column_dimensions["A"].width = 24 readme.column_dimensions["B"].width = 46 readme.column_dimensions["C"].width = 52 readme.column_dimensions["D"].width = 12 readme.column_dimensions["E"].width = 34 for row in readme.iter_rows(): for cell in row: cell.alignment = Alignment(wrap_text=True, vertical="top") return wb class ExecuteRequest(BaseModel): stage: str session_id: str = "" job_id: str = "" # 可选，指定job_id prefix: str = "Test" cpu: int = 8 gwasVcf: str = "" tasselPipeline: str = "run_pipeline.pl" tasselXmx: str = "10g" gwasUnsupportedMarkerPolicy: str = "mask" genotypeScopeId: str = "" phenotypeFile: str = "" phenotypeSampleColumn: str = "" phenotypeTraitColumn: str = "" phenotypeTraitName: str = "" gwasQFile: str = "" gwasFamFile: str = "" gwasQK: int = 2 gwasSortedVcf: str = "" gwasKinshipFile: str = "" gwasTraitFile: str = "" gwasCovariateFile: str = "" phenotypeScopeId: str = "" covariateScopeId: str = "" modelScopeId: str = "" sourceModelScopeId: str = "" analysisScopeId: str = "" populationVcf: str = "" populationSampleGroup: str = "" popPlink: str = "" popTassel: str = "" popFasttree: str = "" popAdmixture: str = "" popBcftools: str = "scripts/pysam_bcftools.py" popLddecay: str = "" popPlotOne: str = "scripts/plot_pop_ld_decay.py" popPlotMulti: str = "scripts/plot_pop_ld_decay.py" popRscript: str = "" popKMin: int = 2 popKMax: int = 10 popCvFolds: int = 5 popRandomSeed: int = 20260723 popGwasK: int = 0 popChromosomePolicy: str = "all_vcf_contigs" popChromosomes: str = "" gwasMlmPvalues: str = "" gwasCmlmPvalues: str = "" gwasMlmTestedHypotheses: Optional[int] = None gwasCmlmTestedHypotheses: Optional[int] = None gwasRscript: str = "" gwasPlink: str = "" gwasEmmax: str = "" gwasRequestedPcs: int = 3 gwasEmmaxPvalues: str = "" gwasEmmaxManifest: str = "" gwasEmmaxTestedHypotheses: Optional[int] = None gwasVcfManifest: str = "" gwasTraitManifest: str = "" gwasCovariateManifest: str = "" idEvalue: float = 0.001 idCoverage: float = 0.3 blastEvalue: float = 0.00001 blastIdentity: float = 30.0 requiredPfams: str = "" refSpecies: str = "" mafftParams: str = f"--auto --thread {cpu}" iqtreeParams: str = f"-m MFP -bb 1000 -nt {cpu}" treeLayout: str = "both" # MSA region plotting parameters msaStartCol: int = 0 msaEndCol: int = 0 msaAutoConserved: bool = False msaWindowLen: int = 60 msaMinNongap: float = 0.6 msaFile: str = "" msaSourceImage: str = "" msaReferenceId: str = "" msaReferenceStartResidue: int = 0 msaReferenceEndResidue: int = 0 treeFile: str = "" treeSourceImage: str = "" treeSourceStage: str = "" treeOutputLabel: str = "" treeLabelOffset: float = 0.4 treeTipLabelSize: float = 3.0 treeBranchColor: str = "black" treeBranchWidth: float = 0.5 treeShowBootstrap: bool = True treeBootstrapSize: float = 2.0 treeTipPointSize: float = 2.0 treeTargetShape: str = "triangle" treeReferenceShape: str = "circle" treeMarkerMode: str = "auto" treeSubfamilyArcEnabled: bool = True treeSubfamilyArcGap: float = 1.2 treeSubfamilyArcLabelGap: float = 0.6 treeSubfamilyArcLabelSize: float = 0.0 syntenySourceImage: str = "" syntenySourceStage: str = "synteny" syntenyOutputLabel: str = "synteny_replot_circos" syntenyGeneLabelSize: float = 38.0 syntenyGeneTrackR0: float = 1.03 syntenyGeneTrackR1: float = 1.30 syntenyChromLabelRadius: str = "1r - 60p" syntenyChromLabelSize: float = 40.0 syntenyIdeogramThickness: float = 90.0 promoterElements: str = "" promoterCategories: str = "" promoterMatchMode: str = "exact" promoterPlotModes: str = "position_distribution,heatmap" promoterColorBy: str = "auto" promoterTopN: int = 30 promoterOutputLabel: str = "" promoterFile: str = "" promoterTreeFile: str = "" promoterSubfamilyFile: str = "" tfbsUpstreamLength: int = 2000 tfbsMotifDbMode: str = "auto" tfbsFimoPvalue: float = 0.0001 tfbsFimoQvalue: float = 0.05 tfbsBackgroundMode: str = "none" tfbsMaxNearestRank: str = "order" tfbsTopMotifs: int = 30 tfbsScanBothStrands: bool = True comboPanels: List[str] = [] comboOutputLabel: str = "" referenceSubfamilyFile: str = "" subfamilyConfirmFile: str = "" referenceTargetPrefix: str = "" referencePrefix: str = "At" referenceOverwriteGeneSubfamily: bool = False nmotifs: int = 10 minw: int = 6 maxw: int = 200 domEvalue: float = 0.00001 domCoverage: float = 30.0 idTransform: str = "none" scopeFile: str = "" scopeMode: str = "" analysisRoute: str = "" memberSetId: str = "" contextRoute: str = "" subfamilyLabelSource: str = "" requiresConfirmedSubfamily: bool = False forceRerun: bool = False forceRerunStages: List[str] = [] analysisMode: str = "" speciesQueue: List[str] = [] confirmPrefixBySpecies: Dict[str, str] = {} geneFamily: str = "" targetSpecies: List[str] = [] referenceSpecies: List[str] = [] ppiKaksReferenceSpecies: List[str] = [] ppiReferenceSpecies: str = "" referenceBundleSpecies: str = "" ppiReferenceTaxonId: str = "" ppiReferenceStorageKey: str = "" ppiReferenceSelectionMode: str = "" ppiReferenceUserConfirmed: bool = False dependencyOwner: str = "" identifyBlastReferenceSpecies: List[str] = [] primaryReferenceSpecies: List[str] = [] refSpeciesList: List[str] = [] comparativeSpecies: List[str] = [] blastEngine: str = "diamond" interspeciesPlotMode: str = "both" geoId: str = "" # RNA-seq参数 rnaseqMode: str = "upload" speciesName: str = "" speciesCode: str = "" rnaseqPrefix: str = "" rnaseqOutputPrefix: str = "" # RNA-seq QC 参数 rnaseqDataMode: str = "upload" rnaseqDataPath: str = "" # RNA-seq Enrich参数 topPathways: int = 20 # 富集分析热图通路数量 enrichBackgroundSource: str = "auto" # auto, gene2go, orgdb enrichSpecies: str = "" # 内置OrgDb物种代码 enrichKeyType: str = "auto" # OrgDb keyType enrichType: str = "go" # go, kegg, both keggOrganism: str = "" # KEGG organism code sampleParallelJobs: int = 0 threadsPerSample: int = 0 rnaseqQcTool: str = "fastp" rnaseqFastpBin: str = "tools/fastp" rnaseqFastpDetectAdapterForPe: bool = True rnaseqFastpCutFront: bool = True rnaseqFastpCutTail: bool = True rnaseqFastpCutWindowSize: int = 4 rnaseqFastpCutMeanQuality: int = 20 rnaseqFastpQualifiedQualityPhred: int = 20 rnaseqFastpUnqualifiedPercentLimit: int = 30 rnaseqFastpNBaseLimit: int = 5 rnaseqFastpLengthRequired: int = 30 rnaseqSpliceSiteSource: str = "auto" rnaseqKnownSpliceSite: str = "" rnaseqQuantMode: str = "dual" rnaseqAbundanceEngine: str = "auto" rnaseqTranscriptomeFasta: str = "" rnaseqTx2gene: str = "" rnaseqSalmonLibtype: str = "A" rnaseqTximportCountsFromAbundance: str = "no" rnaseqKallistoBias: bool = False rnaseqDePadj: float = 0.05 rnaseqDeLog2fc: float = 1.0 enrichPvalueCutoff: float = 0.05 enrichQvalueCutoff: float = 0.05 goObo: str = "data/go/go-basic.obo" keggLevel: str = "pathway" keggFilterScope: str = "plant" keggHighlightMode: str = "auto" keggNoNetwork: bool = False keggSkipPathview: bool = False enrichPlotType: str = "r_plot" rnaseqForegroundMode: str = "auto" # auto, global, family confirmedAutoAnnotation: bool = False annotationApplications: list[str] = Field(default_factory=lambda: ["Pfam", "ProSiteProfiles"]) rnaseqSampleConfig: str = "" wgsSampleConfig: str = "" wgsReferenceFasta: str = "" wgsQcManifest: str = "" wgsIndexManifest: str = "" wgsSampleSetId: str = "" wgsReferenceId: str = "" wgsAnalysisId: str = "" wgsAlignManifest: str = "" wgsMarkdupManifest: str = "" wgsBamQcManifest: str = "" wgsHaplotypeManifest: str = "" wgsCombineManifest: str = "" wgsJointManifest: str = "" wgsSelectManifest: str = "" wgsFilterManifest: str = "" wgsImputeManifest: str = "" wgsSampleSetId: str = "" wgsReferenceId: str = "" wgsAnalysisId: str = "" wgsIndexAnalysisId: str = "" wgsMarkdupAnalysisId: str = "" wgsBamQcAnalysisId: str = "" wgsHaplotypeAnalysisId: str = "" wgsCombineAnalysisId: str = "" wgsJointAnalysisId: str = "" wgsSelectAnalysisId: str = "" wgsFilterAnalysisId: str = "" wgsImputeAnalysisId: str = "" wgsFastp: str = "" wgsSamtools: str = "" wgsBwaMem2: str = "" wgsGatk: str = "" wgsThreads: int = 1 wgsMaxJobs: int = 1 wgsBcftools: str = "" wgsMethod: str = "gatk" wgsJava: str = "" wgsBeagleJar: str = "" wgsMemory: str = "8g" wgsWindow: float = Field(default=40.0, gt=0) wgsOverlap: float = Field(default=2.0, gt=0) wgsNe: int = Field(default=100000, gt=0) wgsIterations: int = Field(default=12, gt=0) wgsPhaseStates: int = Field(default=280, gt=0) wgsImpStates: int = Field(default=1600, gt=0) wgsCluster: float = Field(default=0.005, gt=0) wgsMaf: Literal[0.05] = 0.05 wgsMaxMissing: Literal[0.5] = 0.5 wgsThreads: int = 1 wgsMaxJobs: int = 1 wgsNativePairHmmThreads: int = Field(default=1, gt=0) wgsMaxAlternateAlleles: Literal[6] = 6 wgsCallingThreshold: Literal[10] = 10 wgsIncludeNonVariantSites: bool = False wgsCoverageThreshold: Literal[1] = 1 wgsPlatform: str = "illumina" rnaseqCountMatrix: str = "" rnaseqExpressionMatrix: str = "" rnaseqSampleTable: str = "" fgseaComparison: str = "" fgseaMinSize: int = 15 fgseaMaxSize: int = 500 fgseaNperm: int = 10000 fgseaSeed: int = 1 wgcnaMinMean: float = 1.0 wgcnaMaxGenes: int = 8000 wgcnaMinModuleSize: int = 30 wgcnaMergeCutHeight: float = 0.25 wgcnaPowers: str = "1,2,3,4,5,6,7,8,9,10,12,14,16,18,20" candidateMaxDegProteins: int = 500 candidateRunInterproscan: str = "auto" # NCBI sequence fetch 参数 ncbiQuery: str = "" ncbiGeneIds: str = "" ncbiMode: str = "gene" ncbiIdType: str = "auto" ncbiTaxid: str = "" ncbiSpecies: str = "" ncbiSeqTypes: str = "both" ncbiTranscriptPolicy: str = "refseq_preferred" ncbiAllowFamilyDownload: bool = False ncbiEmail: str = "" # Local sequence extraction 参数 sequenceExtractFasta: str = "" sequenceExtractIdsFile: str = "" sequenceExtractIds: str = "" sequenceExtractSeqType: str = "auto" sequenceExtractRenameMode: str = "keep" sequenceExtractCleanHeaders: bool = False # PPI参数 ppiSpecies: str = "auto" # PPI物种代码；auto 时按 scope/prefix 推断 normalizationSpeciesLabel: str = "" ppiScore: int = 200 # PPI置信度阈值 # KaKs参数 kaksNeutralRate: float = 7.08e-9 # 中性替换率 # Genome Prepare参数 genomePrepareType: str = "auto" # auto, jgi, ensembl, ncbi class ConfirmedPlanStage(BaseModel): id: str dependencies: List[str] = Field(default_factory=list) params: Dict[str, Any] = Field(default_factory=dict) class ConfirmedPlanExecutionRequest(BaseModel): session_id: str job_id: str prefix: str = "" plan_id: str = "" stages: List[ConfirmedPlanStage] class MethodologyStageExecution(BaseModel): observation_id: str = "" literature_stage: str = "" bbav5_stage_id: str = "" selected_tool: str = "" execute: bool = True parameters: Dict[str, Any] = {} class MethodologyExecutionRequest(BaseModel): job_id: str = "" prefix: str = "" cpu: int = 8 stages: List[MethodologyStageExecution] = [] class MethodologyBindPlanRequest(BaseModel): job_id: str = "" plan: Dict[str, Any] = Field(default_factory=dict) plan_path: str = "" methodology_job_id: str = "" source: str = "" class PpiResetSubmitRequest(BaseModel): job_id: str prefix: str gene_ids: List[str] = Field(default_factory=list) instruction: str = "" ppiSpecies: str = "auto" ppiScore: int = 200 sourceGeneIdFile: str = "" class PpiSpeciesResolutionRequest(BaseModel): job_id: str prefix: str scopeFile: str = "" ppiSpecies: str = "auto" ppiScore: int = 200 targetSpecies: List[str] = Field(default_factory=list) referenceSpecies: List[str] = Field(default_factory=list) geneFamily: str = "" targetScientificName: str = "" ppiTargetTreeTaxonId: str = "" ppiReferenceTaxonId: str = "" ppiReferenceStorageKey: str = "" ppiReferenceSelectionMode: str = "" ppiReferenceUserConfirmed: bool = False dependencyOwner: str = "" ignoreExistingOrtholog: bool = False requestedStages: List[str] = Field(default_factory=list) def __init__(self, **data): aliases = { "jobId": "job_id", "scope_file": "scopeFile", "ppi_species": "ppiSpecies", "ppi_score": "ppiScore", "target_species": "targetSpecies", "reference_species": "referenceSpecies", "gene_family": "geneFamily", "target_scientific_name": "targetScientificName", "ppi_target_tree_taxon_id": "ppiTargetTreeTaxonId", "ppi_reference_taxon_id": "ppiReferenceTaxonId", "ppi_reference_selected_taxon_id": "ppiReferenceTaxonId", "ppi_reference_storage_key": "ppiReferenceStorageKey", "ppi_reference_selection_mode": "ppiReferenceSelectionMode", "ppi_reference_user_confirmed": "ppiReferenceUserConfirmed", "dependency_owner": "dependencyOwner", "ignore_existing_ortholog": "ignoreExistingOrtholog", "requested_stages": "requestedStages", } for source, target in aliases.items(): if source in data and target not in data: data[target] = data.pop(source) super().__init__(**data) class ReferenceBundleDownloadRequest(BaseModel): job_id: str speciesCode: str = "" speciesName: str = "" taxonId: str = "" accession: str = "" requestedStages: List[str] = Field(default_factory=list) includeRecommended: bool = False timeoutSec: int = 300 asyncMode: bool = False def __init__(self, **data): aliases = { "jobId": "job_id", "species_code": "speciesCode", "species_name": "speciesName", "taxon_id": "taxonId", "requested_stages": "requestedStages", "include_recommended": "includeRecommended", "timeout_sec": "timeoutSec", "async_mode": "asyncMode", "async": "asyncMode", } for source, target in aliases.items(): if source in data and target not in data: data[target] = data.pop(source) super().__init__(**data) class MethodologyReadinessRequest(BaseModel): job_id: str = "" plan: Dict[str, Any] = Field(default_factory=dict) plan_path: str = "" methodology_job_id: str = "" class MethodologySandboxBuildRequest(BaseModel): methodology_dir: str = "" sandbox_root: str = "/tmp/methodology_qa_sandbox_bbav5" run_id: str = "" include_figures: bool = True content_list_path: str = "" class MethodologySandboxAnswerRequest(BaseModel): question: str = "" registry_path: str = "" sandbox_dir: str = "" METHODOLOGY_STAGE_DEPENDENCIES = { "identify": [], "phylogeny": ["identify"], "reference_classification": ["identify", "phylogeny", "motif", "structure", "domain", "combo"], "subfamily_confirm": ["reference_classification"], "pep": ["identify"], "synteny": ["identify"], "interspecies_synteny": ["identify"], "karyotype": ["identify", "synteny"], "motif": ["identify", "phylogeny"], "domain": ["identify", "phylogeny"], "structure": ["identify", "phylogeny"], "combo": ["identify", "phylogeny", "motif", "domain", "structure"], "promoter": ["identify", "phylogeny"], "tfbs": ["genome_prepare", "identify"], "expression": ["identify", "phylogeny"], "rnaseq_qc": [], "rnaseq_align": ["rnaseq_qc"], "rnaseq_quant": ["rnaseq_align"], "rnaseq_sample_qc": ["rnaseq_quant"], "rnaseq_de": ["rnaseq_sample_qc"], "rnaseq_go_enrich": ["rnaseq_de"], "rnaseq_kegg_enrich": ["rnaseq_de"], "rnaseq_fgsea": ["rnaseq_de", "rnaseq_kegg_enrich"], "rnaseq_wgcna": ["rnaseq_sample_qc"], "ortholog": ["identify"], "kaks": ["identify", "ortholog"], "ppi": ["identify"], } METHODOLOGY_STAGE_ORDER = [ "genome_prepare", "identify", "phylogeny", "motif", "domain", "structure", "combo", "reference_classification", "subfamily_confirm", "pep", "synteny", "interspecies_synteny", "karyotype", "promoter", "tfbs", "expression", "rnaseq_qc", "rnaseq_align", "rnaseq_quant", "rnaseq_sample_qc", "rnaseq_de", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea", "rnaseq_wgcna", "ortholog", "kaks", "ppi", ] def _methodology_stage_sort_key(stage_id: str) -> tuple[int, str]: try: return (METHODOLOGY_STAGE_ORDER.index(stage_id), stage_id) except ValueError: return (len(METHODOLOGY_STAGE_ORDER), stage_id) def _methodology_selected_stages(req: MethodologyExecutionRequest) -> list[MethodologyStageExecution]: selected_by_stage: dict[str, MethodologyStageExecution] = {} skipped_stage_ids = {stage.bbav5_stage_id for stage in req.stages if stage.bbav5_stage_id and not stage.execute} source_by_stage = {stage.bbav5_stage_id: stage for stage in req.stages if stage.bbav5_stage_id} for stage in req.stages: stage_id = stage.bbav5_stage_id if not stage.execute or not stage_id or stage_id in selected_by_stage: continue selected_by_stage[stage_id] = stage def add_dependencies(stage_id: str) -> None: for dep_id in METHODOLOGY_STAGE_DEPENDENCIES.get(stage_id, []): if dep_id in skipped_stage_ids: continue if dep_id not in selected_by_stage: source = source_by_stage.get(dep_id) selected_by_stage[dep_id] = MethodologyStageExecution( observation_id=f"auto_dependency_for_{stage_id}_{dep_id}", literature_stage=f"dependency_for:{stage_id}", bbav5_stage_id=dep_id, selected_tool=source.selected_tool if source else "", execute=True, parameters=source.parameters if source else {}, ) add_dependencies(dep_id) for stage_id in list(selected_by_stage): add_dependencies(stage_id) return sorted(selected_by_stage.values(), key=lambda item: _methodology_stage_sort_key(item.bbav5_stage_id)) async def _wait_for_submitted_stage(task_id: str, stage_id: str, timeout_seconds: int = 7200) -> dict: deadline = time.time() + timeout_seconds last_status: dict[str, Any] = {} while time.time() < deadline: status = await get_task_status(task_id) last_status = dict(status) state = str(status.get("status", "")).lower() if state in {"success", "completed"}: return last_status if state in {"failed", "failure", "cancelled"}: raise RuntimeError(status.get("message") or f"{stage_id} 执行失败") await asyncio.sleep(5) raise TimeoutError(f"{stage_id} 等待超时，最后状态: {last_status.get('status', 'unknown')}") async def _run_methodology_execution(req: MethodologyExecutionRequest, job_id: str, status_path: Path) -> None: """Run confirmed methodology stages in dependency order without blocking the HTTP request.""" ordered_stages = _methodology_selected_stages(req) status_payload: dict[str, Any] = { "status": "running", "job_id": job_id, "execution_mode": "dependency_ordered_background", "execution_order": [stage.bbav5_stage_id for stage in ordered_stages], "submitted": [], "started_at": datetime.now().isoformat(), } def write_status() -> None: status_path.write_text(json.dumps(status_payload, ensure_ascii=False, indent=2), encoding="utf-8") write_status() try: for stage in ordered_stages: stage_id = stage.bbav5_stage_id payload = { "stage": stage_id, "job_id": job_id, "prefix": req.prefix, "cpu": req.cpu, } for key, value in (stage.parameters or {}).items(): if key in ExecuteRequest.__fields__: payload[key] = value result = await execute_stage(ExecuteRequest(**payload)) entry = { "observation_id": stage.observation_id, "literature_stage": stage.literature_stage, "stage": stage_id, "selected_tool": stage.selected_tool, "result": result, "submitted_at": datetime.now().isoformat(), } status_payload["submitted"].append(entry) write_status() if not result.get("success"): raise RuntimeError(result.get("message") or f"{stage_id} 提交失败") if result.get("task_id"): entry["final_status"] = await _wait_for_submitted_stage(result["task_id"], stage_id) write_status() status_payload["status"] = "completed" status_payload["completed_at"] = datetime.now().isoformat() write_status() except Exception as exc: status_payload["status"] = "failed" status_payload["error"] = str(exc) status_payload["failed_at"] = datetime.now().isoformat() write_status() def _validate_gwas_fmt_immediate_postflight( manifest: dict, trusted_genotype_scope_id: str, workspace_dir: Path, ) -> tuple[list[dict], str]: """Validate immediate-task outputs against the request-owned genotype identity.""" manifest_scope_id = str(manifest.get("genotype_scope_id") or "").strip() if not trusted_genotype_scope_id or manifest_scope_id != trusted_genotype_scope_id: return [], "genotype_scope_mismatch" records = _gwas_fmt_result_artifacts(manifest, trusted_genotype_scope_id, workspace_dir) roles = [str(record.get("role") or "") for record in records] if not {"sorted_vcf", "tassel_hapmap"}.issubset(roles): return [], "invalid_outputs" return records, "" def _latest_standard_stage_log(workspace_dir: Path, stage: str) -> Optional[Path]: workspace_root = workspace_dir.resolve() logs_dir = (workspace_root / "logs").resolve() try: logs_dir.relative_to(workspace_root) except ValueError: return None if not logs_dir.is_dir(): return None candidates: list[Path] = [] for candidate in logs_dir.glob(f"*_{stage}.log"): if candidate.is_symlink(): continue try: resolved = candidate.resolve(strict=True) resolved.relative_to(logs_dir) except (OSError, ValueError): continue if resolved.is_file(): candidates.append(resolved) if not candidates: return None return max(candidates, key=lambda path: path.stat().st_mtime_ns) def _standard_stage_log_snapshot(workspace_dir: Path, stage: str) -> dict[str, tuple[int, int, int, int]]: workspace_root = workspace_dir.resolve() logs_dir = (workspace_root / "logs").resolve() try: logs_dir.relative_to(workspace_root) except ValueError: return {} if not logs_dir.is_dir(): return {} snapshot = {} for candidate in logs_dir.glob(f"*_{stage}.log"): if candidate.is_symlink(): continue try: resolved = candidate.resolve(strict=True) resolved.relative_to(logs_dir) stat = resolved.stat() except (OSError, ValueError): continue if resolved.is_file(): snapshot[str(resolved)] = (stat.st_dev, stat.st_ino, stat.st_mtime_ns, stat.st_size) return snapshot def _latest_changed_standard_stage_log( workspace_dir: Path, stage: str, before: dict[str, tuple[int, int, int, int]], ) -> Optional[Path]: after = _standard_stage_log_snapshot(workspace_dir, stage) changed = [Path(path) for path, identity in after.items() if before.get(path) != identity] return max(changed, key=lambda path: path.stat().st_mtime_ns) if changed else None def run_stage_sync(task_id: str, stage: str, job_id: str, params: dict): """在后台线程中执行分析阶段""" def _param_value(*keys, default=None): for key in keys: if key in params and params.get(key) is not None and params.get(key) != "": return params.get(key) return default def _bool_param(*keys, default=False): value = _param_value(*keys, default=default) if isinstance(value, str): text = value.strip().lower() if text in {"1", "true", "yes", "y", "on"}: return True if text in {"0", "false", "no", "n", "off"}: return False return default return bool(value) trusted_genotype_scope_id = str(params.get("genotypeScopeId") or "").strip() resource_lease = None resource_acquired = False resource_outcome = "failed" try: try: from gfa_core.mas.resource_governor import ResourceLease from gfa_core.mas.stage_catalog import get_stage_queue, get_stage_resource_profile profile = get_stage_resource_profile(stage) resource_lease = ResourceLease( task_id=task_id, stage=stage, job_id=job_id, prefix=str(params.get("prefix") or ""), cpu=int(params.get("cpu") or profile.get("default_cpu") or 1), queue=get_stage_queue(stage), resource_class=str(profile.get("resource_class") or ""), concurrency_group=str(profile.get("concurrency_group") or ""), celery_task_id="threadpool", ) def _threadpool_waiting(wait_result: dict): tasks[task_id]["status"] = "waiting_resource" tasks[task_id]["message"] = wait_result.get("reason", "等待资源可用") tasks[task_id]["resource_usage"] = wait_result.get("usage", {}) tasks[task_id]["status"] = "waiting_resource" tasks[task_id]["message"] = "等待资源可用..." resource_lease.wait(update_state=_threadpool_waiting) resource_acquired = True except Exception as resource_exc: print(f"[WARN] Resource governor unavailable for threadpool stage {stage}: {resource_exc}") # 更新任务状态为运行中 tasks[task_id]["status"] = "running" tasks[task_id]["message"] = f"正在执行 {stage}..." tasks[task_id]["start_time"] = datetime.now().isoformat() # 导入pipeline_backend from apps.web.pipeline_backend import run_stage, JobInfo job_dir = INPUT_ROOT / job_id workspace_dir = WORKSPACE_ROOT / job_id effective_prefix = resolve_effective_prefix(job_id, params, stage) if effective_prefix != _nonempty_prefix(params.get("prefix")): print(f"[INFO] Resolved empty prefix for {stage}: {effective_prefix or ''}") params["prefix"] = effective_prefix tasks[task_id]["prefix"] = effective_prefix job = JobInfo( job_id=job_id, input_dir=str(job_dir), workspace_dir=str(workspace_dir), logs_dir=str(workspace_dir / "logs"), created_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S") ) os.makedirs(job.logs_dir, exist_ok=True) # 切换到项目根目录 try: original_cwd = os.getcwd() except FileNotFoundError: original_cwd = str(PROJECT_ROOT) os.chdir(PROJECT_ROOT) # 特殊状态提示 if stage == "promoter": tasks[task_id]["message"] = "正在提交序列到 PlantCARE..." elif stage == "synteny": tasks[task_id]["message"] = "正在进行共线性分析（MCScanX）..." elif stage == "rnaseq_qc": tasks[task_id]["message"] = "正在进行RNA-seq质控分析（fastp）... 这可能需要15-30分钟，请耐心等待..." backend_stage = { "rnaseq_go_enrich": "rnaseq_enrich", "rnaseq_kegg_enrich": "rnaseq_enrich", }.get(stage, stage) # 执行 run_cpu = int(params.get("cpu") or 8) gwas_log_snapshot = ( _standard_stage_log_snapshot(workspace_dir, stage) if stage in {"gwas_fmt", "gwas_plot", "gwas_emmax", "gwas_emplot", "gwas_glm"} else {} ) result_msg, manifest = run_stage( stage=backend_stage, job=job, prefix=params["prefix"], cpu=run_cpu, required_pfams=params.get("requiredPfams") or None, ref_species=(params.get("identifyBlastReferenceSpecies") if backend_stage == "identify" and params.get("identifyBlastReferenceSpecies") else params.get("refSpecies")) or None, id_evalue=params.get("idEvalue", 0.001), id_coverage=params.get("idCoverage", 0.3), blast_evalue=params.get("blastEvalue", 0.00001), blast_identity=params.get("blastIdentity", 30.0), phy_mafft=params.get("mafftParams", f"--auto --thread {run_cpu}"), phy_iqtree=params.get("iqtreeParams", f"-m MFP -bb 1000 -nt {run_cpu}"), phy_tree_layout=params.get("treeLayout", "both"), msa_start_col=params.get("msaStartCol", 0), msa_end_col=params.get("msaEndCol", 0), msa_auto_conserved=params.get("msaAutoConserved", False), msa_window_len=params.get("msaWindowLen", 60), msa_min_nongap=params.get("msaMinNongap", 0.6), msa_file=params.get("msaFile", ""), msa_source_image=params.get("msaSourceImage", ""), msa_reference_id=params.get("msaReferenceId", ""), msa_reference_start_residue=params.get("msaReferenceStartResidue", 0), msa_reference_end_residue=params.get("msaReferenceEndResidue", 0), tree_file=params.get("treeFile", ""), tree_source_image=params.get("treeSourceImage", ""), tree_source_stage=params.get("treeSourceStage", ""), tree_output_label=params.get("treeOutputLabel", ""), tree_label_offset=params.get("treeLabelOffset", 0.4), tree_tip_label_size=params.get("treeTipLabelSize", 3.0), tree_branch_color=params.get("treeBranchColor", "black"), tree_branch_width=params.get("treeBranchWidth", 0.5), tree_show_bootstrap=params.get("treeShowBootstrap", True), tree_bootstrap_size=params.get("treeBootstrapSize", 2.0), tree_tip_point_size=params.get("treeTipPointSize", 2.0), tree_target_shape=params.get("treeTargetShape", "triangle"), tree_reference_shape=params.get("treeReferenceShape", "circle"), tree_marker_mode=params.get("treeMarkerMode", "auto"), tree_subfamily_arc_enabled=params.get("treeSubfamilyArcEnabled", True), tree_subfamily_arc_gap=params.get("treeSubfamilyArcGap", 1.2), tree_subfamily_arc_label_gap=params.get("treeSubfamilyArcLabelGap", 0.6), tree_subfamily_arc_label_size=params.get("treeSubfamilyArcLabelSize", 0.0), synteny_source_image=params.get("syntenySourceImage", ""), synteny_source_stage=params.get("syntenySourceStage", "synteny"), synteny_output_label=params.get("syntenyOutputLabel", "synteny_replot_circos"), synteny_gene_label_size=params.get("syntenyGeneLabelSize", 38.0), synteny_gene_track_r0=params.get("syntenyGeneTrackR0", 1.03), synteny_gene_track_r1=params.get("syntenyGeneTrackR1", 1.30), synteny_chrom_label_radius=params.get("syntenyChromLabelRadius", "1r - 60p"), synteny_chrom_label_size=params.get("syntenyChromLabelSize", 40.0), synteny_ideogram_thickness=params.get("syntenyIdeogramThickness", 90.0), promoter_elements=params.get("promoterElements", ""), promoter_categories=params.get("promoterCategories", ""), promoter_match_mode=params.get("promoterMatchMode", "exact"), promoter_plot_modes=params.get("promoterPlotModes", "position_distribution,heatmap"), promoter_color_by=params.get("promoterColorBy", "auto"), promoter_top_n=params.get("promoterTopN", 30), promoter_output_label=params.get("promoterOutputLabel", ""), promoter_file=params.get("promoterFile", ""), promoter_tree_file=params.get("promoterTreeFile", ""), promoter_subfamily_file=params.get("promoterSubfamilyFile", ""), tfbs_upstream_length=params.get("tfbsUpstreamLength", 2000), tfbs_motif_db_mode=params.get("tfbsMotifDbMode", "auto"), tfbs_fimo_pvalue=params.get("tfbsFimoPvalue", 0.0001), tfbs_fimo_qvalue=params.get("tfbsFimoQvalue", 0.05), tfbs_background_mode=params.get("tfbsBackgroundMode", "none"), tfbs_max_nearest_rank=params.get("tfbsMaxNearestRank", "order"), tfbs_top_motifs=params.get("tfbsTopMotifs", 30), tfbs_scan_both_strands=bool(params.get("tfbsScanBothStrands", True)), combo_panels=params.get("comboPanels") or params.get("combo_panels") or [], combo_output_label=params.get("comboOutputLabel") or params.get("combo_output_label") or "", reference_subfamily_file=params.get("referenceSubfamilyFile", ""), subfamily_confirm_file=params.get("subfamilyConfirmFile", ""), reference_target_prefix=params.get("referenceTargetPrefix", ""), reference_prefix=params.get("referencePrefix", "At"), reference_overwrite_gene_subfamily=bool(params.get("referenceOverwriteGeneSubfamily", False)), motif_nmotifs=params.get("nmotifs", 10), motif_minw=params.get("minw", 6), motif_maxw=params.get("maxw", 200), dom_evalue=params.get("domEvalue", 0.00001), dom_coverage=params.get("domCoverage", 30.0), id_transform=params.get("idTransform", "none"), geo_id=params.get("geoId", ""), # RNA-seq参数 rnaseq_mode=params.get("rnaseqMode", "upload"), species_name=params.get("speciesName", ""), # RNA-seq Enrich参数 top_pathways=params.get("topPathways", 20), enrich_background_source=params.get("enrichBackgroundSource", "auto"), enrich_species=params.get("enrichSpecies", ""), enrich_key_type=params.get("enrichKeyType", "auto"), enrich_type=params.get("enrichType", "go"), kegg_organism=params.get("keggOrganism", ""), rnaseq_foreground_mode=params.get("rnaseqForegroundMode", "auto"), confirmedAutoAnnotation=bool(params.get("confirmedAutoAnnotation", False)), annotation_applications=params.get("annotationApplications", ["Pfam", "ProSiteProfiles"]), rnaseq_sample_config=params.get("rnaseqSampleConfig", ""), wgs_sample_config=params.get("wgsSampleConfig", ""), wgs_fastp=params.get("wgsFastp") or str(ASAW_BIN / "fastp"), wgs_threads=params.get("wgsThreads", 1), wgs_max_jobs=params.get("wgsMaxJobs", 1), wgs_reference_fasta=params.get("wgsReferenceFasta", ""), wgs_samtools=get_wgs_samtools(stage, params.get("wgsSamtools", "")), wgs_bwa_mem2=params.get("wgsBwaMem2") or str(ASAW_BIN / "bwa-mem2"), wgs_gatk=params.get("wgsGatk") or str(ASAW_BIN / "gatk"), wgs_qc_manifest=params.get("wgsQcManifest", ""), wgs_index_manifest=params.get("wgsIndexManifest", ""), wgs_align_manifest=params.get("wgsAlignManifest", ""), wgs_markdup_manifest=params.get("wgsMarkdupManifest", ""), wgs_bam_qc_manifest=params.get("wgsBamQcManifest", ""), wgs_haplotype_manifest=params.get("wgsHaplotypeManifest", ""), wgs_combine_manifest=params.get("wgsCombineManifest", ""), wgs_joint_manifest=params.get("wgsJointManifest", ""), wgs_select_manifest=params.get("wgsSelectManifest", ""), wgs_filter_manifest=params.get("wgsFilterManifest", ""), wgs_impute_manifest=params.get("wgsImputeManifest", ""), wgs_sample_set_id=params.get("wgsSampleSetId", ""), wgs_reference_id=params.get("wgsReferenceId", ""), wgs_analysis_id=params.get("wgsAnalysisId", ""), wgs_index_analysis_id=params.get("wgsIndexAnalysisId", ""), wgs_markdup_analysis_id=params.get("wgsMarkdupAnalysisId", ""), wgs_bam_qc_analysis_id=params.get("wgsBamQcAnalysisId", ""), wgs_haplotype_analysis_id=params.get("wgsHaplotypeAnalysisId", ""), wgs_combine_analysis_id=params.get("wgsCombineAnalysisId", ""), wgs_joint_analysis_id=params.get("wgsJointAnalysisId", ""), wgs_select_analysis_id=params.get("wgsSelectAnalysisId", ""), wgs_filter_analysis_id=params.get("wgsFilterAnalysisId", ""), wgs_impute_analysis_id=params.get("wgsImputeAnalysisId", ""), wgs_native_pair_hmm_threads=params.get("wgsNativePairHmmThreads", 1), wgs_max_alternate_alleles=params.get("wgsMaxAlternateAlleles", 6), wgs_calling_threshold=params.get("wgsCallingThreshold", 10), wgs_include_non_variant_sites=bool(params.get("wgsIncludeNonVariantSites", False)), wgs_method=params.get("wgsMethod", "gatk"), wgs_java=get_wgs_java(params.get("wgsJava", "")), wgs_beagle_jar=params.get("wgsBeagleJar") or WGS_BEAGLE_JAR, wgs_memory=params.get("wgsMemory", "8g"), wgs_window=params.get("wgsWindow", 40.0), wgs_overlap=params.get("wgsOverlap", 2.0), wgs_ne=params.get("wgsNe", 100000), wgs_iterations=params.get("wgsIterations", 12), wgs_phase_states=params.get("wgsPhaseStates", 280), wgs_imp_states=params.get("wgsImpStates", 1600), wgs_cluster=params.get("wgsCluster", 0.005), wgs_maf=params.get("wgsMaf", 0.05), wgs_max_missing=params.get("wgsMaxMissing", 0.5), wgs_bcftools=get_wgs_bcftools(params.get("wgsBcftools", "")), wgs_coverage_threshold=params.get("wgsCoverageThreshold", 1), wgs_platform=params.get("wgsPlatform", "illumina"), rnaseq_sample_parallel_jobs=params.get("sample_parallel_jobs") or params.get("sampleParallelJobs") or 0, rnaseq_threads_per_sample=params.get("threads_per_sample") or params.get("threadsPerSample") or 0, rnaseq_project_max_cpu=params.get("rnaseq_project_max_cpu") or params.get("rnaseqProjectMaxCpu") or 0, rnaseq_resource_decision=params.get("resource_decision") or params.get("resourceDecision") or None, rnaseq_qc_tool=_param_value("rnaseqQcTool", "qc_tool", default="fastp"), rnaseq_fastp_bin=_param_value("rnaseqFastpBin", "fastp_bin", default="tools/fastp"), rnaseq_fastp_detect_adapter_for_pe=_bool_param("rnaseqFastpDetectAdapterForPe", "detect_adapter_for_pe", default=True), rnaseq_fastp_cut_front=_bool_param("rnaseqFastpCutFront", "cut_front", default=True), rnaseq_fastp_cut_tail=_bool_param("rnaseqFastpCutTail", "cut_tail", default=True), rnaseq_fastp_cut_window_size=_param_value("rnaseqFastpCutWindowSize", "cut_window_size", default=4), rnaseq_fastp_cut_mean_quality=_param_value("rnaseqFastpCutMeanQuality", "cut_mean_quality", default=20), rnaseq_fastp_qualified_quality_phred=_param_value("rnaseqFastpQualifiedQualityPhred", "qualified_quality_phred", default=20), rnaseq_fastp_unqualified_percent_limit=_param_value("rnaseqFastpUnqualifiedPercentLimit", "unqualified_percent_limit", default=30), rnaseq_fastp_n_base_limit=_param_value("rnaseqFastpNBaseLimit", "n_base_limit", default=5), rnaseq_fastp_length_required=_param_value("rnaseqFastpLengthRequired", "length_required", default=30), rnaseq_splice_site_source=_param_value("rnaseqSpliceSiteSource", "splice_site_source", default="auto"), rnaseq_known_splice_site=_param_value("rnaseqKnownSpliceSite", "known_splice_site", default=""), rnaseq_quant_mode=_param_value("rnaseqQuantMode", "quant_mode", default="dual"), rnaseq_abundance_engine=_param_value("rnaseqAbundanceEngine", "abundance_engine", default="auto"), rnaseq_transcriptome_fasta=_param_value("rnaseqTranscriptomeFasta", "transcriptome_fasta", default=""), rnaseq_tx2gene=_param_value("rnaseqTx2gene", "tx2gene", default=""), rnaseq_salmon_libtype=_param_value("rnaseqSalmonLibtype", "salmon_libtype", default="A"), rnaseq_tximport_counts_from_abundance=_param_value("rnaseqTximportCountsFromAbundance", "tximport_counts_from_abundance", default="no"), rnaseq_kallisto_bias=bool(params.get("rnaseqKallistoBias") or params.get("kallisto_bias")), rnaseq_de_padj=_param_value("rnaseqDePadj", "padj", default=0.05), rnaseq_de_module=_param_value("rnaseqDeModule", "rnaseq_de_module", default=""), rnaseq_de_log2fc=_param_value("rnaseqDeLog2fc", "log2fc", default=1.0), enrich_pvalue_cutoff=_param_value("enrichPvalueCutoff", "pvalue_cutoff", default=0.05), enrich_qvalue_cutoff=_param_value("enrichQvalueCutoff", "qvalue_cutoff", default=0.05), go_obo=_param_value("goObo", "go_obo", default="data/go/go-basic.obo"), kegg_level=_param_value("keggLevel", "kegg_level", default="pathway"), kegg_filter_scope=_param_value("keggFilterScope", "kegg_filter_scope", default="plant"), kegg_highlight_mode=_param_value("keggHighlightMode", "kegg_highlight_mode", default="auto"), kegg_no_network=bool(params.get("keggNoNetwork") or params.get("no_kegg_network")), kegg_skip_pathview=bool(params.get("keggSkipPathview") or params.get("skip_pathview")), enrich_plot_type=params.get("enrichPlotType") or params.get("plot_type") or "r_plot", rnaseq_count_matrix=params.get("rnaseqCountMatrix", ""), gwas_vcf=params.get("gwasVcf", ""), tassel_pipeline=params.get("tasselPipeline", "run_pipeline.pl"), tassel_xmx=params.get("tasselXmx", "10g"), gwas_unsupported_marker_policy=params.get("gwasUnsupportedMarkerPolicy", "mask"), genotype_scope_id=trusted_genotype_scope_id, phenotype_file=params.get("phenotypeFile", ""), phenotype_sample_column=params.get("phenotypeSampleColumn", ""), phenotype_trait_column=params.get("phenotypeTraitColumn", ""), phenotype_trait_name=params.get("phenotypeTraitName", ""), gwas_q_file=params.get("gwasQFile", ""), gwas_fam_file=params.get("gwasFamFile", ""), gwas_q_k=params.get("gwasQK", 2), gwas_sorted_vcf=params.get("gwasSortedVcf", ""), gwas_kinship_file=params.get("gwasKinshipFile", ""), gwas_trait_file=params.get("gwasTraitFile", ""), gwas_covariate_file=params.get("gwasCovariateFile", ""), phenotype_scope_id=params.get("phenotypeScopeId", ""), covariate_scope_id=params.get("covariateScopeId", ""), gwas_mlm_pvalues=params.get("gwasMlmPvalues", ""), gwas_cmlm_pvalues=params.get("gwasCmlmPvalues", ""), gwas_mlm_tested_hypotheses=params.get("gwasMlmTestedHypotheses"), gwas_cmlm_tested_hypotheses=params.get("gwasCmlmTestedHypotheses"), gwas_rscript=params.get("gwasRscript") or str(ASAW_RSCRIPT), source_model_scope_id=params.get("sourceModelScopeId", ""), gwas_plink=params.get("gwasPlink") or str(ASAW_BIN / "plink"), gwas_emmax=params.get("gwasEmmax") or GWAS_EMMAX_BIN, gwas_requested_pcs=params.get("gwasRequestedPcs", 3), gwas_emmax_pvalues=params.get("gwasEmmaxPvalues", ""), gwas_emmax_manifest=params.get("gwasEmmaxManifest", ""), gwas_emmax_tested_hypotheses=params.get("gwasEmmaxTestedHypotheses"), gwas_vcf_manifest=params.get("gwasVcfManifest", ""), gwas_trait_manifest=params.get("gwasTraitManifest", ""), gwas_covariate_manifest=params.get("gwasCovariateManifest", ""), population_vcf=params.get("populationVcf", ""), population_sample_group=params.get("populationSampleGroup", ""), population_params={ "plink": params.get("popPlink") or str(ASAW_BIN / "plink"), "tassel": params.get("popTassel") or TASSEL_PIPELINE, "fasttree": params.get("popFasttree") or str(ASAW_BIN / "FastTree"), "admixture": params.get("popAdmixture") or str(ASAW_BIN / "admixture"), "bcftools": params.get("popBcftools", "scripts/pysam_bcftools.py"), "poplddecay": params.get("popLddecay") or str(ASAW_BIN / "PopLDdecay"), "plot_one": params.get("popPlotOne", "scripts/plot_pop_ld_decay.py"), "plot_multi": params.get("popPlotMulti", "scripts/plot_pop_ld_decay.py"), "rscript": params.get("popRscript") or str(ASAW_RSCRIPT), "k_min": params.get("popKMin", 2), "k_max": params.get("popKMax", 10), "cv_folds": params.get("popCvFolds", 5), "random_seed": params.get("popRandomSeed", 20260723), "gwas_k": params.get("popGwasK", 0), "chromosome_policy": params.get("popChromosomePolicy", "all_vcf_contigs"), "chromosomes": params.get("popChromosomes", ""), }, rnaseq_expression_matrix=params.get("rnaseqExpressionMatrix", ""), rnaseq_sample_table=params.get("rnaseqSampleTable", ""), fgsea_comparison=params.get("fgseaComparison", ""), fgsea_min_size=params.get("fgseaMinSize", 15), fgsea_max_size=params.get("fgseaMaxSize", 500), fgsea_nperm=params.get("fgseaNperm", 10000), fgsea_seed=params.get("fgseaSeed", 1), wgcna_min_mean=params.get("wgcnaMinMean", 1.0), wgcna_max_genes=params.get("wgcnaMaxGenes", 8000), wgcna_min_module_size=params.get("wgcnaMinModuleSize", 30), wgcna_merge_cut_height=params.get("wgcnaMergeCutHeight", 0.25), wgcna_powers=params.get("wgcnaPowers", "1,2,3,4,5,6,7,8,9,10,12,14,16,18,20"), candidate_max_deg_proteins=params.get("candidateMaxDegProteins", 500), candidate_run_interproscan=params.get("candidateRunInterproscan", "auto"), # PPI参数 ppi_species=params.get("ppiSpecies") or params.get("ppi_species") or "auto", normalization_species_label=params.get("normalizationSpeciesLabel") or params.get("normalization_species_label") or "", ppi_reference_taxon_id=params.get("ppiReferenceTaxonId") or params.get("ppi_reference_taxon_id") or "", ppi_reference_storage_key=params.get("ppiReferenceStorageKey") or params.get("ppi_reference_storage_key") or "", ppi_reference_selection_mode=params.get("ppiReferenceSelectionMode") or params.get("ppi_reference_selection_mode") or "", ppi_reference_user_confirmed=_bool_param("ppiReferenceUserConfirmed", "ppi_reference_user_confirmed", default=False), dependency_owner=params.get("dependencyOwner") or params.get("dependency_owner") or "", ppi_score=params.get("ppiScore", 200), # KaKs参数 kaks_neutral_rate=params.get("kaksNeutralRate", 7.08e-9), # Genome Prepare参数 genome_prepare_type=params.get("genomePrepareType", "auto"), scope_file=params.get("scopeFile", ""), analysis_route=params.get("analysisRoute", ""), member_set_id=params.get("memberSetId", ""), analysis_mode=params.get("analysisMode", ""), species_queue=params.get("speciesQueue", []), subfamily_label_source=params.get("subfamilyLabelSource", ""), target_species=params.get("targetSpecies") or None, interspecies_ref_species=params.get("refSpeciesList") or None, interspecies_blast_engine=params.get("blastEngine", "diamond") or "diamond", interspecies_plot_mode=params.get("interspeciesPlotMode", "both") or "both", ncbi_query=params.get("ncbiQuery", ""), ncbi_gene_ids=params.get("ncbiGeneIds", ""), ncbi_mode=params.get("ncbiMode", "gene"), ncbi_id_type=params.get("ncbiIdType", "auto"), ncbi_taxid=params.get("ncbiTaxid", ""), ncbi_species=params.get("ncbiSpecies", ""), ncbi_seq_types=params.get("ncbiSeqTypes", "both"), ncbi_transcript_policy=params.get("ncbiTranscriptPolicy", "refseq_preferred"), ncbi_allow_family_download=bool(params.get("ncbiAllowFamilyDownload", False)), ncbi_email=params.get("ncbiEmail", ""), sequence_extract_fasta=params.get("sequenceExtractFasta", ""), sequence_extract_ids_file=params.get("sequenceExtractIdsFile", ""), sequence_extract_ids=params.get("sequenceExtractIds", ""), sequence_extract_seq_type=params.get("sequenceExtractSeqType", "auto"), sequence_extract_rename_mode=params.get("sequenceExtractRenameMode", "keep"), sequence_extract_clean_headers=bool(params.get("sequenceExtractCleanHeaders", False)), ) # 恢复工作目录 try: os.chdir(original_cwd) except FileNotFoundError: os.chdir(PROJECT_ROOT) # 检查是否有特殊状态需要更新 if stage == "promoter" and "Waiting for PlantCARE" in result_msg: tasks[task_id]["message"] = "已提交 PlantCARE，等待邮件结果（可能需要10-60分钟）..." elif stage == "promoter" and "submitted" in result_msg.lower(): tasks[task_id]["message"] = "已提交 PlantCARE，正在等待邮件..." # 收集结果 - 按stage分类 stage_prefix_map = { "gwas_fmt": ["gwas_fmt"], "gwas_trait": ["gwas_trait"], "gwas_cov": ["gwas_cov"], "gwas_mlm": ["gwas_mlm"], "gwas_plot": ["gwas_plot"], "gwas_emmax": ["gwas_emmax"], "gwas_emplot": ["gwas_emplot"], "gwas_glm": ["gwas_glm"], "identify": ["identify", "final_members"], "identify_refine": ["identify_refine", "curated_members", "final_members.edited"], "phylogeny": ["phylogeny", "tree"], "reference_classification": ["reference_classification", "geneid.subfamily", "subfamily"], "motif": ["motif"], "domain": ["domain"], "structure": ["structure"], "promoter": ["promoter"], "promoter_plot_revise": ["promoter_custom", "promoter_plot_revise"], "tfbs": ["tfbs"], "synteny": ["synteny"], "karyotype": ["karyotype"], "interspecies_synteny": ["interspecies_synteny", "interspecies_tree"], "ppi": ["ppi"], "kaks": ["kaks"], "pep": ["pep"], "expression": ["exp", "expression"], "combo": ["combo"], "msa_region_plot": ["msa_region", "region.tsv", "conservation_profile"], "tree_plot_revise": ["tree_replot"], "synteny_plot_revise": ["synteny_replot"], "msa": ["msa"], "ortholog": ["ortholog"], "rnaseq_qc": ["rnaseq_qc"], "rnaseq_align": ["rnaseq_align"], "rnaseq_quant": ["rnaseq_quant"], "rnaseq_de": ["rnaseq_de"], "annotation": ["annotation", "gene2go", "gene2ko", "eggnog"], "rnaseq_go_enrich": ["rnaseq_go_enrich", "go_enrich", "_go_"], "rnaseq_kegg_enrich": ["rnaseq_kegg_enrich", "kegg_enrich", "_kegg_", "kegg_pathway", "kegg_pathview"], "rnaseq_candidate_family": ["rnaseq_candidate_family", "candidate_family", "候选家族"], "rnaseq_enrich": ["rnaseq_enrich"], "genome_prepare": ["genome_prepare"], "prepare_organize": ["prepare_organize"], "prepare_normalize_gff": ["prepare_normalize_gff", "normalized.gff"], "prepare_longest_transcript": ["prepare_longest_transcript", "longest"], "prepare_extract_pep": ["prepare_extract_pep"], "prepare_extract_cds": ["prepare_extract_cds"], "prepare_filter_fasta": ["prepare_filter_fasta", "filtered"], "prepare_simplify_ncbi": ["prepare_simplify_ncbi", "simplified", "ncbi"], "sequence_extract_by_id": ["sequence_extract", "sequence_extract_by_id"], "ncbi_fetch_sequences": ["ncbi_fetch", "ncbi_fetch_sequences"] } # 反转映射 prefix_to_stage = {} for s, prefixes in stage_prefix_map.items(): for p in prefixes: prefix_to_stage[p] = s # 初始化各stage文件 stage_files = {s: {"images": [], "files": []} for s in stage_prefix_map} final_dir = workspace_dir / "final" if final_dir.exists(): for ext in ['.png', '.jpg', '.jpeg', '.svg']: for f in final_dir.glob(f"*{ext}"): fname_lower = f.name.lower() if re.fullmatch(r"ko\d{5}\.(png|jpg|jpeg|svg)", fname_lower): continue matched = "ncbi_fetch_sequences" if "ncbi_fetch" in fname_lower else stage if matched != "ncbi_fetch_sequences": for stage_name, prefixes in stage_prefix_map.items(): if any(p in fname_lower for p in prefixes): matched = stage_name break stage_files[matched]["images"].append(f"/workspace/{job_id}/final/{f.name}") for f in final_dir.iterdir(): if f.is_file() and f.suffix.lower() not in {".pdf", ".png", ".jpg", ".jpeg", ".svg"}: fname_lower = f.name.lower() matched = "ncbi_fetch_sequences" if "ncbi_fetch" in fname_lower else stage if matched != "ncbi_fetch_sequences": for stage_name, prefixes in stage_prefix_map.items(): if any(p in fname_lower for p in prefixes): matched = stage_name break stage_files[matched]["files"].append({ "name": f.name, "size": f.stat().st_size, "path": f"/workspace/{job_id}/final/{f.name}", "role": ( "sorted_vcf" if matched == "gwas_fmt" and f.name.endswith(".sorted.vcf.gz") else "tassel_hapmap" if matched == "gwas_fmt" and f.name.endswith(".hapmap.hmp.txt") else "manifest" if matched == "gwas_fmt" and f.name.endswith(".manifest.json") else "supporting_evidence" ), }) # 汇总所有文件（保留兼容） all_images = [] all_files = [] for s_data in stage_files.values(): all_images.extend(s_data["images"]) all_files.extend(s_data["files"]) # 收集PPI和KaKs结果文件 if final_dir.exists(): # PPI结果文件 for ppi_file in final_dir.glob("*.ppi_*"): if ppi_file.is_file(): ppi_url = f"/workspace/{job_id}/final/{ppi_file.name}" if ppi_file.suffix.lower() in {".png", ".jpg", ".jpeg", ".svg"}: if ppi_url not in stage_files["ppi"]["images"]: stage_files["ppi"]["images"].append(ppi_url) if ppi_url not in all_images: all_images.append(ppi_url) else: file_info = { "name": ppi_file.name, "size": ppi_file.stat().st_size, "path": ppi_url } stage_files["ppi"]["files"].append(file_info) all_files.append(file_info) # KaKs结果文件 for kaks_file in final_dir.glob("*.kaks_*"): if kaks_file.is_file(): file_info = { "name": kaks_file.name, "size": kaks_file.stat().st_size, "path": f"/workspace/{job_id}/final/{kaks_file.name}" } stage_files["kaks"]["files"].append(file_info) all_files.append(file_info) # ?? manifest ????? final ?????????????? manifest/FASTA/tree ??? # ????????? manifest ???????? success/partial_success manifest_stage_name = "ncbi_fetch" if stage == "ncbi_fetch_sequences" else ("sequence_extract" if stage == "sequence_extract_by_id" else stage) expected_manifest = Path(manifest) if manifest else final_dir / f"{params['prefix']}.{manifest_stage_name}.manifest.json" paused_manifest = final_dir / f"{params['prefix']}.{stage}.paused.json" manifest_valid = False manifest_status = None manifest_data = {} import time time.sleep(0.5) manifest_to_check = paused_manifest if paused_manifest.exists() else expected_manifest if manifest_to_check.exists(): try: with open(manifest_to_check, 'r', encoding='utf-8') as f: manifest_data = json.load(f) manifest_status = manifest_data.get("status") actual_manifest_stage = manifest_data.get("stage") or manifest_stage_name actual_manifest_prefix = manifest_data.get("prefix") or params["prefix"] direct_replot_stage = actual_manifest_stage in {"tree_plot_revise", "synteny_plot_revise", "promoter_plot_revise", "msa_region_plot"} if direct_replot_stage and manifest_status in {"success", "partial_success"} and "❌" not in str(result_msg or ""): manifest_valid = True else: try: from gfa_core.mas.tools.stage_identity import ( StageIdentity, get_stage_status_for_identity, merge_manifest_identity_params, resolve_stage_identity, ) task_identity = tasks.get(task_id, {}).get("stage_identity") if isinstance(manifest_data, dict) and manifest_data: identity_params = merge_manifest_identity_params( {**params, "prefix": actual_manifest_prefix}, manifest_data, ) strict_postflight_identity = bool( identity_params.get("scopeId") or identity_params.get("scope_id") or identity_params.get("analysisRoute") or identity_params.get("analysis_route") or identity_params.get("memberSetId") or identity_params.get("member_set_id") ) identity = resolve_stage_identity( stage=actual_manifest_stage, prefix=actual_manifest_prefix, params=identity_params, job_id=job_id, workspace_root=WORKSPACE_ROOT, ) if not strict_postflight_identity: identity.scope_id = "" identity.analysis_route = "" identity.member_set_id = "" elif isinstance(task_identity, dict) and task_identity: identity = StageIdentity(**{ key: value for key, value in task_identity.items() if key in StageIdentity.__dataclass_fields__ }) else: identity = resolve_stage_identity( stage=actual_manifest_stage, prefix=actual_manifest_prefix, params={**params, "prefix": actual_manifest_prefix}, job_id=job_id, workspace_root=WORKSPACE_ROOT, ) if 'strict_postflight_identity' not in locals(): strict_postflight_identity = bool(identity.scope_id or identity.analysis_route or identity.member_set_id) unified_status = get_stage_status_for_identity( job_id=job_id, identity=identity, workspace_root=WORKSPACE_ROOT, strict_identity=strict_postflight_identity, ) except Exception: from gfa_core.mas.tools.stage_status import get_stage_status as get_unified_stage_status unified_status = get_unified_stage_status( job_id=job_id, prefix=actual_manifest_prefix, stage=actual_manifest_stage, scope_id=str(manifest_data.get("scope_id") or ""), analysis_route=str(manifest_data.get("analysis_route") or params.get("analysisRoute") or ""), member_set_id=str(manifest_data.get("member_set_id") or params.get("memberSetId") or ""), strict_identity=bool(manifest_data.get("scope_id") or manifest_data.get("analysis_route") or manifest_data.get("member_set_id")), ) result_msg_text = str(result_msg or "") explicit_failure = ( "❌" in result_msg_text or "失败" in result_msg_text or "failed" in result_msg_text.lower() or "error" in result_msg_text.lower() ) manifest_valid = unified_status.completed and not explicit_failure if not manifest_valid: print(f"[WARN] Unified manifest validation failed: {unified_status.to_dict()}") except Exception as e: print(f"[WARN] Failed to read manifest {manifest_to_check}: {e}") gwas_postflight_records: list[dict] = [] gwas_postflight_error = "" if manifest_valid and stage == "gwas_fmt": gwas_postflight_records, gwas_postflight_error = _validate_gwas_fmt_immediate_postflight( manifest_data, trusted_genotype_scope_id, workspace_dir ) manifest_valid = not gwas_postflight_error elif manifest_valid and stage == "gwas_mlm": expected_mlm_scopes = { key: str(manifest_data.get(key) or "").strip() for key in ( "genotype_scope_id", "phenotype_scope_id", "covariate_scope_id", "model_scope_id", "analysis_scope_id", ) } if not all(expected_mlm_scopes.values()): gwas_postflight_error = "incomplete_gwas_mlm_identity" else: gwas_postflight_records = _validated_gwas_result_artifacts( manifest_data, workspace=workspace_dir, expected_scopes=expected_mlm_scopes, ) required_roles = { "mlm_full", "cmlm_full", "mlm_pvalues", "cmlm_pvalues", "report", "environment", "log", } if {str(record.get("role") or "") for record in gwas_postflight_records} != required_roles: gwas_postflight_error = "invalid_gwas_mlm_outputs" manifest_valid = not gwas_postflight_error elif manifest_valid and stage == "gwas_plot": expected_plot_scopes = { key: str(manifest_data.get(key) or "").strip() for key in ( "genotype_scope_id", "phenotype_scope_id", "covariate_scope_id", "source_model_scope_id", "analysis_scope_id", ) } if not all(expected_plot_scopes.values()): gwas_postflight_error = "incomplete_gwas_plot_identity" else: gwas_postflight_records = _validated_gwas_result_artifacts( manifest_data, workspace=workspace_dir, expected_scopes=expected_plot_scopes, ) required_roles = { "mlm_manhattan", "mlm_qq", "cmlm_manhattan", "cmlm_qq", "report", "environment", } if {str(record.get("role") or "") for record in gwas_postflight_records} != required_roles: gwas_postflight_error = "invalid_gwas_plot_outputs" manifest_valid = not gwas_postflight_error elif manifest_valid and stage == "gwas_emmax": expected_emmax_scopes = { key: str(manifest_data.get(key) or "").strip() for key in ( "genotype_scope_id", "phenotype_scope_id", "model_scope_id", "analysis_scope_id", ) } if not all(expected_emmax_scopes.values()): gwas_postflight_error = "incomplete_gwas_emmax_identity" else: gwas_postflight_records = _validated_gwas_result_artifacts( manifest_data, workspace=workspace_dir, expected_scopes=expected_emmax_scopes, ) required_roles = {"emmax_full", "emmax_pvalues", "report", "environment"} if {str(record.get("role") or "") for record in gwas_postflight_records} != required_roles: gwas_postflight_error = "invalid_gwas_emmax_outputs" manifest_valid = not gwas_postflight_error elif manifest_valid and stage == "gwas_emplot": expected_emplot_scopes = { key: str(manifest_data.get(key) or "").strip() for key in ( "genotype_scope_id", "phenotype_scope_id", "source_model_scope_id", "analysis_scope_id", ) } if not all(expected_emplot_scopes.values()): gwas_postflight_error = "incomplete_gwas_emplot_identity" else: gwas_postflight_records = _validated_gwas_result_artifacts( manifest_data, workspace=workspace_dir, expected_scopes=expected_emplot_scopes, ) required_roles = {"emmax_manhattan", "emmax_qq", "report", "environment"} if {str(record.get("role") or "") for record in gwas_postflight_records} != required_roles: gwas_postflight_error = "invalid_gwas_emplot_outputs" manifest_valid = not gwas_postflight_error elif manifest_valid and stage == "gwas_glm": expected_glm_scopes = { key: str(manifest_data.get(key) or "").strip() for key in ( "genotype_scope_id", "phenotype_scope_id", "covariate_scope_id", "model_scope_id", "analysis_scope_id", ) } if not all(expected_glm_scopes.values()): gwas_postflight_error = "incomplete_gwas_glm_identity" else: gwas_postflight_records = _validated_gwas_result_artifacts( manifest_data, workspace=workspace_dir, expected_scopes=expected_glm_scopes, ) required_roles = { "glm_full", "glm_structure_full", "glm_pvalues", "glm_structure_pvalues", "log", "report", "environment", } if {str(record.get("role") or "") for record in gwas_postflight_records} != required_roles: gwas_postflight_error = "invalid_gwas_glm_outputs" manifest_valid = not gwas_postflight_error gwas_failure_log = ( _latest_changed_standard_stage_log(workspace_dir, "gwas_fmt", gwas_log_snapshot) if stage == "gwas_fmt" else ( _latest_changed_standard_stage_log(workspace_dir, "gwas_plot", gwas_log_snapshot) if stage == "gwas_plot" else ( _latest_changed_standard_stage_log(workspace_dir, "gwas_emmax", gwas_log_snapshot) if stage == "gwas_emmax" else ( _latest_changed_standard_stage_log(workspace_dir, "gwas_emplot", gwas_log_snapshot) if stage == "gwas_emplot" else ( _latest_changed_standard_stage_log(workspace_dir, "gwas_glm", gwas_log_snapshot) if stage == "gwas_glm" else None ) ) ) ) ) gwas_failure_log_path = str(gwas_failure_log) if gwas_failure_log else "" if gwas_postflight_error and gwas_failure_log and stage not in {"gwas_plot", "gwas_emmax", "gwas_emplot", "gwas_glm"}: with gwas_failure_log.open("a", encoding="utf-8") as log_handle: log_handle.write(f"ERROR: {gwas_postflight_error}\n") if not manifest_valid: # manifest 验证失败 if stage == "gwas_fmt": failure_error = str(gwas_postflight_error or manifest_data.get("error") or result_msg or "Manifest validation failed") failure_message = "gwas_fmt 结果未通过 manifest 校验。" if gwas_failure_log_path: failure_message += f" Log: {gwas_failure_log_path}" elif stage == "gwas_plot" and gwas_postflight_error: failure_error = gwas_postflight_error failure_message = f"❌ gwas_plot 失败：{gwas_postflight_error}" if gwas_failure_log_path: failure_message += f". Log: {gwas_failure_log_path}" elif isinstance(result_msg, str) and ("❌" in result_msg or "failed" in result_msg.lower()): failure_message = result_msg failure_error = result_msg elif gwas_postflight_error: failure_message = f"❌ {stage} 失败：{gwas_postflight_error}" failure_error = gwas_postflight_error else: failure_message = f"❌ {stage} 失败：manifest 状态为 {manifest_status}。{result_msg}" failure_error = f"Manifest validation failed: expected {expected_manifest}, status={manifest_status}" diagnosis = _run_failure_diagnosis_for_task(job_id, stage, params["prefix"], failure_error, params) tasks[task_id]["status"] = "failed" tasks[task_id]["message"] = failure_message tasks[task_id]["diagnosis"] = diagnosis tasks[task_id]["end_time"] = datetime.now().isoformat() tasks[task_id]["result"] = { "success": False, "error": failure_error, "stage": stage, "diagnosis": diagnosis, "manifest": manifest_data, "manifest_path": ( str(manifest_to_check) if manifest_to_check.exists() else "" ), } if manifest_data.get("fallback_method") == "bcftools": tasks[task_id]["result"]["fallback_method"] = "bcftools" if stage in {"gwas_fmt", "gwas_plot", "gwas_emmax", "gwas_emplot", "gwas_glm"}: tasks[task_id]["result"]["log_path"] = gwas_failure_log_path else: # 更新任务状态为完成 if stage == "gwas_fmt": canonical_files = [] for artifact in gwas_postflight_records: artifact_path = Path(str(artifact.get("path") or "")) local_path = artifact_path if artifact_path.is_absolute() else workspace_dir / artifact_path if not local_path.is_file(): continue canonical_files.append({ "name": local_path.name, "size": local_path.stat().st_size, "path": f"/workspace/{job_id}/{artifact_path}" if not artifact_path.is_absolute() else f"/workspace/{job_id}/final/{local_path.name}", "role": str(artifact.get("role") or artifact.get("artifact") or "supporting_evidence"), }) all_images, all_files = _canonicalize_gwas_task_results( stage_files, all_images, all_files, canonical_files ) elif stage == "gwas_mlm": canonical_files = [] for artifact in gwas_postflight_records: artifact_path = Path(str(artifact.get("path") or "")) local_path = workspace_dir / artifact_path canonical_files.append({ "name": local_path.name, "size": local_path.stat().st_size, "path": f"/workspace/{job_id}/{artifact_path}", "role": str(artifact.get("role") or "supporting_evidence"), }) canonical_files.append({ "name": manifest_to_check.name, "size": manifest_to_check.stat().st_size, "path": f"/workspace/{job_id}/final/{manifest_to_check.name}", "role": "manifest", }) previous = stage_files["gwas_mlm"] stale_paths = {item.get("path") for item in previous.get("files") or []} all_files = [item for item in all_files if item.get("path") not in stale_paths] all_files.extend(canonical_files) stage_files["gwas_mlm"] = { "images": [], "files": canonical_files, "elapsed_time": round(float(manifest_data.get("elapsed_sec") or 0), 1), } if "identity" in locals(): tasks[task_id]["stage_identity"] = identity.to_dict() tasks[task_id]["stage_status"] = unified_status.to_dict() elif stage == "gwas_plot": image_roles = {"mlm_manhattan", "mlm_qq", "cmlm_manhattan", "cmlm_qq"} canonical_images = [] canonical_files = [] for artifact in gwas_postflight_records: artifact_path = Path(str(artifact.get("path") or "")) local_path = workspace_dir / artifact_path url = f"/workspace/{job_id}/{artifact_path}" role = str(artifact.get("role") or "supporting_evidence") if role in image_roles: canonical_images.append(url) else: canonical_files.append({ "name": local_path.name, "size": local_path.stat().st_size, "path": url, "role": role, }) canonical_files.append({ "name": manifest_to_check.name, "size": manifest_to_check.stat().st_size, "path": f"/workspace/{job_id}/final/{manifest_to_check.name}", "role": "manifest", }) previous = stage_files["gwas_plot"] stale_images = set(previous.get("images") or []) stale_paths = {item.get("path") for item in previous.get("files") or []} all_images = [item for item in all_images if item not in stale_images] all_files = [item for item in all_files if item.get("path") not in stale_paths] all_images.extend(item for item in canonical_images if item not in all_images) all_files.extend(canonical_files) stage_files["gwas_plot"] = { "images": canonical_images, "files": canonical_files, "elapsed_time": round(float(manifest_data.get("elapsed_sec") or 0), 1), } if "identity" in locals(): tasks[task_id]["stage_identity"] = identity.to_dict() tasks[task_id]["stage_status"] = unified_status.to_dict() elif stage == "gwas_emmax": canonical_files = [] for artifact in gwas_postflight_records: artifact_path = Path(str(artifact.get("path") or "")) local_path = workspace_dir / artifact_path canonical_files.append({ "name": local_path.name, "size": local_path.stat().st_size, "path": f"/workspace/{job_id}/{artifact_path}", "role": str(artifact.get("role") or "supporting_evidence"), }) canonical_files.append({ "name": manifest_to_check.name, "size": manifest_to_check.stat().st_size, "path": f"/workspace/{job_id}/final/{manifest_to_check.name}", "role": "manifest", }) previous = stage_files["gwas_emmax"] stale_paths = {item.get("path") for item in previous.get("files") or []} all_files = [item for item in all_files if item.get("path") not in stale_paths] all_files.extend(canonical_files) stage_files["gwas_emmax"] = { "images": [], "files": canonical_files, "elapsed_time": round(float(manifest_data.get("elapsed_sec") or 0), 1), } if "identity" in locals(): tasks[task_id]["stage_identity"] = identity.to_dict() tasks[task_id]["stage_status"] = unified_status.to_dict() elif stage == "gwas_emplot": image_roles = {"emmax_manhattan", "emmax_qq"} canonical_images = [] canonical_files = [] for artifact in gwas_postflight_records: artifact_path = Path(str(artifact.get("path") or "")) local_path = workspace_dir / artifact_path url = f"/workspace/{job_id}/{artifact_path}" role = str(artifact.get("role") or "supporting_evidence") if role in image_roles: canonical_images.append(url) else: canonical_files.append({ "name": local_path.name, "size": local_path.stat().st_size, "path": url, "role": role, }) canonical_files.append({ "name": manifest_to_check.name, "size": manifest_to_check.stat().st_size, "path": f"/workspace/{job_id}/final/{manifest_to_check.name}", "role": "manifest", }) previous = stage_files["gwas_emplot"] stale_images = set(previous.get("images") or []) stale_paths = {item.get("path") for item in previous.get("files") or []} all_images = [item for item in all_images if item not in stale_images] all_files = [item for item in all_files if item.get("path") not in stale_paths] all_images.extend(item for item in canonical_images if item not in all_images) all_files.extend(canonical_files) stage_files["gwas_emplot"] = { "images": canonical_images, "files": canonical_files, "elapsed_time": round(float(manifest_data.get("elapsed_sec") or 0), 1), } if "identity" in locals(): tasks[task_id]["stage_identity"] = identity.to_dict() tasks[task_id]["stage_status"] = unified_status.to_dict() elif stage == "gwas_glm": canonical_files = [] for artifact in gwas_postflight_records: artifact_path = Path(str(artifact.get("path") or "")) local_path = workspace_dir / artifact_path canonical_files.append({ "name": local_path.name, "size": local_path.stat().st_size, "path": f"/workspace/{job_id}/{artifact_path}", "role": str(artifact.get("role") or "supporting_evidence"), }) canonical_files.append({ "name": manifest_to_check.name, "size": manifest_to_check.stat().st_size, "path": f"/workspace/{job_id}/final/{manifest_to_check.name}", "role": "manifest", }) previous = stage_files["gwas_glm"] stale_paths = {item.get("path") for item in previous.get("files") or []} all_files = [item for item in all_files if item.get("path") not in stale_paths] all_files.extend(canonical_files) stage_files["gwas_glm"] = { "images": [], "files": canonical_files, "elapsed_time": round(float(manifest_data.get("elapsed_sec") or 0), 1), } if "identity" in locals(): tasks[task_id]["stage_identity"] = identity.to_dict() tasks[task_id]["stage_status"] = unified_status.to_dict() tasks[task_id]["status"] = "success" tasks[task_id]["message"] = result_msg tasks[task_id]["end_time"] = datetime.now().isoformat() resource_outcome = "success" tasks[task_id]["result"] = { "success": True, "message": result_msg, "images": all_images, "files": all_files, "stage_files": stage_files, # 按stage分类的文件 "stage": stage, "manifest_path": str(manifest_to_check) if manifest_to_check.exists() else str(expected_manifest), "stage_status": unified_status.to_dict() if "unified_status" in locals() else {}, "stage_identity": tasks[task_id].get("stage_identity", {}), } # ============ 生成累积解读报告（新增） ============ try: if MAS_AVAILABLE and LLMManager: from gfa_core.mas.agents.result_parser import create_result_parser_agent # 获取同一 identity 下已完成的阶段列表，避免跨 scope 混入缓存 completed_stages = [] requested_scope_id = str(params.get("scope_id") or params.get("scopeId") or (params.get("stage_context") or {}).get("scope_id") or "").strip() requested_rnaseq_scope_id = str(params.get("rnaseq_scope_id") or params.get("rnaseqScopeId") or "").strip() requested_analysis_route = str(params.get("analysis_route") or params.get("analysisRoute") or (params.get("stage_context") or {}).get("analysis_route") or "").strip() requested_member_set_id = str(params.get("member_set_id") or params.get("memberSetId") or (params.get("stage_context") or {}).get("member_set_id") or "").strip() if final_dir.exists(): for manifest_file in final_dir.glob(f"{params['prefix']}.*.manifest.json"): try: with open(manifest_file, 'r', encoding='utf-8') as f: m = json.load(f) stage_context_for_manifest = m.get("stage_context") or m.get("stageContext") or {} manifest_params = m.get("parameters") or {} manifest_scope = str(m.get("scope_id") or manifest_params.get("scope_id") or stage_context_for_manifest.get("scope_id") or "").strip() manifest_rnaseq_scope = str(m.get("rnaseq_scope_id") or manifest_params.get("rnaseq_scope_id") or stage_context_for_manifest.get("rnaseq_scope_id") or "").strip() manifest_route = str(m.get("analysis_route") or manifest_params.get("analysis_route") or stage_context_for_manifest.get("analysis_route") or "").strip() manifest_member = str(m.get("member_set_id") or manifest_params.get("member_set_id") or stage_context_for_manifest.get("member_set_id") or "").strip() if requested_scope_id and manifest_scope and manifest_scope != requested_scope_id: continue if requested_rnaseq_scope_id and manifest_rnaseq_scope != requested_rnaseq_scope_id: continue if requested_analysis_route and manifest_route and manifest_route != requested_analysis_route: continue if requested_member_set_id and manifest_member and manifest_member != requested_member_set_id: continue if m.get("status") == "success": stage_name = m.get("stage") if stage_name and stage_name not in completed_stages: completed_stages.append(stage_name) except Exception: continue # 按顺序排序 stage_order = ["identify", "phylogeny", "structure", "motif", "domain", "promoter", "tfbs", "synteny", "expression", "kaks", "ppi"] completed_stages = [s for s in stage_order if s in completed_stages] + [s for s in completed_stages if s not in stage_order] print(f"[CumulativeInterpretation] 生成累积解读，已完成阶段: {completed_stages}") # 创建 LLMManager 和 ResultParserAgent llm_manager = LLMManager(config=llm_config) agent = create_result_parser_agent(llm_manager) # 生成累积解读报告 report = asyncio.run(agent.generate_cumulative_report( job_id=job_id, prefix=params["prefix"], completed_stages=completed_stages, scope_id=requested_scope_id, )) # 保存 scope-aware 解读报告 save_success = agent.save_interpretation( job_id, params["prefix"], report, scope_id=requested_scope_id, rnaseq_scope_id=requested_rnaseq_scope_id, ) if save_success: print(f"[CumulativeInterpretation] scope-aware 解读报告已保存") else: print(f"[CumulativeInterpretation] 保存解读报告失败") except Exception as e: # 解读失败不阻塞主流程 print(f"[CumulativeInterpretation] 生成解读报告失败（非阻塞）: {e}") import traceback traceback.print_exc() # 返回任务结果，供 Celery 使用 return tasks[task_id]["result"] except Exception as e: diagnosis = _run_failure_diagnosis_for_task(job_id, stage, params.get("prefix", ""), str(e), params) tasks[task_id]["status"] = "failed" tasks[task_id]["message"] = f"执行失败: {str(e)}" tasks[task_id]["diagnosis"] = diagnosis tasks[task_id]["end_time"] = datetime.now().isoformat() tasks[task_id]["result"] = { "success": False, "error": str(e), "stage": stage, "diagnosis": diagnosis, } # 返回任务结果，供 Celery 使用 return tasks[task_id]["result"] finally: if resource_acquired and resource_lease is not None: try: resource_lease.release(status=resource_outcome, message=f"{stage} {resource_outcome}") except Exception as release_exc: print(f"[WARN] Failed to release threadpool resource lease for {task_id}: {release_exc}") @app.get("/") async def root(): """返回Vue3前端页面""" return FileResponse( Path(__file__).parent / "index.html", headers={ "Cache-Control": "no-cache, no-store, must-revalidate", "Pragma": "no-cache", "Expires": "0" } ) @app.post("/api/new_job") async def create_new_job(): """创建新工作目录""" global current_job_id import time import uuid current_job_id = f"job_{int(time.time()*1000)}_{uuid.uuid4().hex[:6]}" job_dir = INPUT_ROOT / current_job_id workspace_dir = WORKSPACE_ROOT / current_job_id job_dir.mkdir(parents=True, exist_ok=True) workspace_dir.mkdir(parents=True, exist_ok=True) (workspace_dir / "logs").mkdir(exist_ok=True) (workspace_dir / "final").mkdir(exist_ok=True) return { "success": True, "job_id": current_job_id, "input_dir": str(job_dir), "workspace_dir": str(workspace_dir) } @app.get("/api/list_jobs") async def list_existing_jobs(): """列出所有已有的工作目录""" jobs = [] skipped = [] try: # 扫描 workspace 目录。跳过坏软链接/被删除的目录，避免前端收到 text/plain 500 后 JSON.parse 失败。 if WORKSPACE_ROOT.exists(): job_entries = [] for job_dir in WORKSPACE_ROOT.iterdir(): try: stat_result = job_dir.stat() except FileNotFoundError: skipped.append({"job_id": job_dir.name, "reason": "missing_workspace_entry"}) continue except OSError as exc: skipped.append({"job_id": job_dir.name, "reason": str(exc)}) continue job_entries.append((job_dir, stat_result.st_mtime)) for job_dir, st_mtime in sorted(job_entries, key=lambda item: item[1], reverse=True): if job_dir.is_dir() and job_dir.name.startswith("job_"): # 检查目录内容 final_dir = job_dir / "final" # 获取已完成的阶段 completed_stages = [] if final_dir.exists(): for manifest in final_dir.glob("*.manifest.json"): stage_name = manifest.name.replace(".manifest.json", "").split(".")[-1] completed_stages.append(stage_name) # 获取最后修改时间 mtime = datetime.fromtimestamp(st_mtime) jobs.append({ "job_id": job_dir.name, "created_at": mtime.strftime("%Y-%m-%d %H:%M:%S"), "completed_stages": completed_stages, "has_input": (INPUT_ROOT / job_dir.name).exists(), "workspace_dir": str(job_dir) }) return { "success": True, "jobs": jobs[:20], # 只返回最近20个 "skipped": skipped, } except Exception as exc: return JSONResponse( { "success": False, "error": f"Failed to list working directories: {exc}", "jobs": [], "skipped": skipped, }, status_code=500, ) @app.post("/api/select_job") async def select_existing_job(request: Request): """选择已有工作目录""" global current_job_id data = await request.json() job_id = data.get("job_id") if not job_id: return JSONResponse({"success": False, "error": "No job_id provided"}, status_code=400) # 验证工作目录是否存在 workspace_dir = WORKSPACE_ROOT / job_id input_dir = INPUT_ROOT / job_id if not workspace_dir.exists(): return JSONResponse({"success": False, "error": f"Job {job_id} not found"}, status_code=404) # 设置为当前工作目录 current_job_id = job_id return { "success": True, "job_id": job_id, "input_dir": str(input_dir), "workspace_dir": str(workspace_dir), "message": f"已切换到工作目录: {job_id}" } @app.get("/api/examples/config-xlsx") async def download_config_xlsx_template(): """Download a real config.xlsx template with a machine-readable config sheet and a README sheet.""" try: wb = _build_config_template_workbook() buffer = BytesIO() wb.save(buffer) buffer.seek(0) headers = {"Content-Disposition": 'attachment; filename="config.xlsx"'} return Response( content=buffer.getvalue(), media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", headers=headers, ) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @app.post("/api/upload") async def upload_file(files: List[UploadFile] = File(...), job_id: str = Form("")): """Upload files; Excel config uploads are normalized to config.xlsx.""" global current_job_id if job_id: current_job_id = job_id elif not current_job_id: current_job_id = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:18] job_id = current_job_id job_dir = INPUT_ROOT / job_id job_dir.mkdir(exist_ok=True) config_archive_dir = job_dir / "uploaded_configs" saved_files = [] config_uploads = [] warnings = [] messages = [] for f in files: original_name = _safe_upload_name(f.filename) content = await f.read() if _is_excel_config_upload(original_name) and not _is_sample_info_upload(original_name): config_archive_dir.mkdir(exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f") archive_name = f"{timestamp}_{original_name}" archive_path = config_archive_dir / archive_name with open(archive_path, "wb") as fp: fp.write(content) config_path = job_dir / "config.xlsx" previous_archive = None if config_path.exists(): previous_archive = config_archive_dir / f"{timestamp}_previous_config.xlsx" shutil.copy2(config_path, previous_archive) with open(config_path, "wb") as fp: fp.write(content) config_info = { "original_name": original_name, "archive_path": str(archive_path), "normalized_name": "config.xlsx", "normalized_path": str(config_path), "previous_config_archived": bool(previous_archive), "previous_archive_path": str(previous_archive) if previous_archive else "", } config_uploads.append(config_info) saved_files.append("config.xlsx") messages.append(f"Excel config '{original_name}' normalized to config.xlsx; latest uploaded config will be used.") if previous_archive: messages.append("Previous config.xlsx was archived before replacement.") else: file_path = job_dir / original_name with open(file_path, "wb") as fp: fp.write(content) saved_files.append(original_name) if len(config_uploads) > 1: warnings.append("Multiple Excel files uploaded; the last one was used as config.xlsx.") return { "success": True, "job_id": job_id, "files": saved_files, "config_normalized": bool(config_uploads), "config_uploads": config_uploads, "warnings": warnings, "messages": messages, "autofill_suggestions": _classify_autofill_files(saved_files), } class ExtensionStageRunRequest(BaseModel): extension_id: str job_id: str = "" input_paths: List[str] = Field(default_factory=list) parameters: Dict[str, Any] = Field(default_factory=dict) class ExtensionStageRepairRequest(BaseModel): max_attempts: int = 20 auto_validate: bool = True auto_publish: bool = True class ExtensionStagePromoteRepairRequest(BaseModel): publish: bool = False class ExtensionStageFrontendGateRecordRequest(BaseModel): gate_record: Dict[str, Any] = Field(default_factory=dict) class ExtensionStageFrontendE2EGateRequest(BaseModel): message: str job_id: str = "" input_paths: List[str] = Field(default_factory=list) parameters: Dict[str, Any] = Field(default_factory=dict) execute: bool = True base_url: str = "" timeout_sec: int = 600 intent_policy: str = "strict" gate_phase: str = "pre_publish" class ExtensionStageNLRuntimeGateRequest(BaseModel): message: str job_id: str = "" run_job_id: str = "" input_source_job_id: str = "" input_paths: List[str] = Field(default_factory=list) parameters: Dict[str, Any] = Field(default_factory=dict) execute: bool = True intent_policy: str = "strict" class ExtensionStageNLRuntimeGateRepairRequest(ExtensionStageNLRuntimeGateRequest): max_attempts: int = 20 auto_publish: bool = True class ExtensionStageConfirmRequest(BaseModel): parameters: Dict[str, Any] = Field(default_factory=dict) outputs: List[Dict[str, Any]] = Field(default_factory=list) input_bindings: Dict[str, Any] = Field(default_factory=dict) display_name: str = "" description: str = "" nl_triggers: List[str] = Field(default_factory=list) class ExtensionStageParameterPreviewRequest(BaseModel): parameters: Dict[str, Any] = Field(default_factory=dict) job_id: str = "" input_source_job_id: str = "" input_paths: List[str] = Field(default_factory=list) class ExtensionStageResolveIntentRequest(BaseModel): message: str job_id: str = "" @app.post("/api/extension-stages/build") async def build_extension_stage( display_name: str = Form(...), command: str = Form(""), runtime_env: str = Form("test4py311"), description: str = Form(""), code_files: List[UploadFile] = File(...), data_files: Optional[List[UploadFile]] = File(None), ): """Build a lightweight extension analysis stage from uploaded runnable code.""" try: code_payload = [(f.filename or "code", await f.read()) for f in (code_files or [])] data_payload = [(f.filename or "data", await f.read()) for f in (data_files or [])] return extension_stage_agent.create_from_uploads( display_name=display_name, command=command, runtime_env=runtime_env, code_files=code_payload, data_files=data_payload, description=description, ) except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段构建失败: {exc}") @app.post("/api/extension-stages/{extension_id}/trial-run") async def trial_run_extension_stage(extension_id: str): try: return extension_stage_agent.trial_run(extension_id) except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段试运行失败: {exc}") @app.post("/api/extension-stages/{extension_id}/confirm") async def confirm_extension_stage(extension_id: str, req: ExtensionStageConfirmRequest): try: return extension_stage_agent.confirm( extension_id, parameters=req.parameters, outputs=req.outputs, input_bindings=req.input_bindings, display_name=req.display_name, description=req.description, nl_triggers=req.nl_triggers, ) except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段确认失败: {exc}") @app.get("/api/extension-stages/{extension_id}/preview") async def preview_extension_stage(extension_id: str): try: return extension_stage_agent.preview(extension_id) except ExtensionStageError as exc: raise HTTPException(status_code=404, detail=str(exc)) @app.post("/api/extension-stages/{extension_id}/validate-parameters") async def validate_extension_stage_parameters(extension_id: str, req: ExtensionStageParameterPreviewRequest): try: return extension_stage_agent.validate_parameters(extension_id, req.parameters) except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段参数检查失败: {exc}") @app.post("/api/extension-stages/resolve-intent") async def resolve_extension_stage_intent(req: ExtensionStageResolveIntentRequest): intent = _match_extension_stage_intent(req.message, req.job_id) return {"success": bool(intent), "intent": intent} def _validate_extension_stage_intent_policy(intent_policy: str) -> str: policy = str(intent_policy or "strict").strip().lower() if policy not in {"strict", "path_target"}: raise HTTPException(status_code=400, detail="intent_policy must be one of: strict, path_target") return policy def _path_target_extension_intent(extension_id: str, message: str, job_id: str = "", raw_intent: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: raw = raw_intent if isinstance(raw_intent, dict) else {} raw_params = raw.get("params") if isinstance(raw.get("params"), dict) else {} try: ext = extension_stage_agent.load(extension_id) except Exception: ext = {"extension_id": extension_id} display_name = ext.get("display_name") or ext.get("name") or ext.get("extension_id") or extension_id raw_resolved_extension_id = str(raw_params.get("extension_id") or "") run_job_id = str(job_id or raw_params.get("run_job_id") or raw_params.get("job_id") or "").strip() input_source_job_id = str(raw_params.get("input_source_job_id") or raw_params.get("job_id") or run_job_id or "").strip() params = { "extension_id": extension_id, "extensionName": display_name, "archetype_id": ext.get("archetype_id", ""), "template_gate_status": ext.get("template_gate_status", ""), "input_paths": raw_params.get("input_paths") or [], "parameters": raw_params.get("parameters") or {}, "targetSpecies": raw_params.get("targetSpecies") or [], "referenceSpecies": raw_params.get("referenceSpecies") or [], "job_id": run_job_id, "run_job_id": run_job_id, "input_source_job_id": input_source_job_id or run_job_id, "file_roles": raw_params.get("file_roles") or {}, "intent_slots": raw_params.get("intent_slots") or {}, "intent_policy": "path_target", "raw_resolved_extension_id": raw_resolved_extension_id, } return { "stage": "extension_stage", "stages": ["extension_stage"], "primary_stage": "extension_stage", "params": params, "prefix": "", "gene_family": "", "species": "", "task_kind": "extension_stage", "taskType": "extension_stage", "message": f"按路径目标执行拓展分析阶段：{display_name}。自然语言原始匹配仅作为诊断记录。", "confidence": 1.0, "job_id": run_job_id, "workspace": str(WORKSPACE_ROOT / run_job_id) if run_job_id else "", "raw_intent": raw, } def _extension_stage_intent_resolver(extension_id: str, intent_policy: str): policy = _validate_extension_stage_intent_policy(intent_policy) if policy == "strict": return _match_extension_stage_intent def resolve_path_target(message: str, job_id: str = "") -> Dict[str, Any]: raw_intent = _match_extension_stage_intent(message, job_id) return _path_target_extension_intent(extension_id, message, job_id, raw_intent) return resolve_path_target @app.post("/api/extension-stages/{extension_id}/validate") async def validate_extension_stage(extension_id: str): try: return extension_stage_agent.validate(extension_id) except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段验证失败: {exc}") @app.post("/api/extension-stages/{extension_id}/repair") async def repair_extension_stage(extension_id: str, req: ExtensionStageRepairRequest = ExtensionStageRepairRequest()): try: return extension_stage_agent.repair( extension_id, max_attempts=req.max_attempts, auto_validate=req.auto_validate, auto_publish=req.auto_publish, ) except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段修订失败: {exc}") @app.get("/api/extension-stages/{extension_id}/repair-candidates") async def list_extension_stage_repair_candidates(extension_id: str): try: return {"success": True, "candidates": extension_stage_agent.list_repair_candidates(extension_id)} except ExtensionStageError as exc: raise HTTPException(status_code=404, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段修订候选读取失败: {exc}") @app.post("/api/extension-stages/{extension_id}/repair-candidates/{candidate_id}/promote") async def promote_extension_stage_repair_candidate(extension_id: str, candidate_id: str, req: ExtensionStagePromoteRepairRequest = ExtensionStagePromoteRepairRequest()): try: return extension_stage_agent.promote_repair_candidate(extension_id, candidate_id, publish=req.publish) except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段修订候选发布失败: {exc}") @app.post("/api/extension-stages/{extension_id}/publish") async def publish_extension_stage(extension_id: str): try: return extension_stage_agent.publish(extension_id) except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段发布失败: {exc}") @app.post("/api/extension-stages/{extension_id}/frontend-gate-record") async def record_extension_stage_frontend_gate(extension_id: str, req: ExtensionStageFrontendGateRecordRequest): try: return extension_stage_agent.record_frontend_gate(extension_id, req.gate_record) except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段前端门控记录失败: {exc}") @app.post("/api/extension-stages/{extension_id}/frontend-e2e-gate") async def run_extension_stage_frontend_e2e_gate(extension_id: str, req: ExtensionStageFrontendE2EGateRequest, request: Request): global current_job_id job_id = req.job_id or current_job_id if req.execute and not job_id: raise HTTPException(status_code=400, detail="job_id is required for running an extension frontend E2E gate") intent_policy = _validate_extension_stage_intent_policy(req.intent_policy) gate_phase = str(req.gate_phase or "pre_publish").strip().lower() if gate_phase not in {"pre_publish", "post_publish"}: raise HTTPException(status_code=400, detail="gate_phase must be one of: pre_publish, post_publish") base_url = (req.base_url or str(request.base_url)).rstrip("/") gate_run_id = f"ui_gate_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}" extension_token = re.sub(r"[^A-Za-z0-9._-]+", "_", extension_id).strip("._") or "extension" output_dir = extension_stage_manager.workspace_root / "_extension_stage_gates" / extension_token / gate_run_id try: gate_record = await asyncio.to_thread( run_playwright_frontend_gate, base_url=base_url, extension_id=extension_id, job_id=job_id, prompt=req.message, input_paths=req.input_paths, parameters=req.parameters, output_dir=output_dir, execute=req.execute, timeout_sec=req.timeout_sec, intent_policy=intent_policy, gate_phase=gate_phase, ) gate_record["gate_run_id"] = gate_run_id gate_record["output_dir"] = str(output_dir) result = extension_stage_agent.record_frontend_gate(extension_id, gate_record) if job_id: current_job_id = job_id return { "success": (result.get("ui_display_gate") or {}).get("status") == "passed", "gate_run_id": gate_run_id, "ui_display_gate": result.get("ui_display_gate") or {}, "frontend_gate": result.get("frontend_gate") or {}, "gate_record": gate_record, "extension": result.get("extension") or {}, } except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段前端展示门控失败: {exc}") @app.post("/api/extension-stages/{extension_id}/nl-runtime-gate") async def run_extension_stage_nl_runtime_gate(extension_id: str, req: ExtensionStageNLRuntimeGateRequest): global current_job_id run_job_id = req.run_job_id or req.job_id or current_job_id input_source_job_id = req.input_source_job_id or req.job_id or run_job_id if req.execute and not run_job_id: raise HTTPException(status_code=400, detail="job_id is required for running an extension NL runtime gate") intent_policy = _validate_extension_stage_intent_policy(req.intent_policy) try: result = extension_stage_agent.run_nl_runtime_gate( extension_id, message=req.message, job_id=req.job_id or run_job_id, run_job_id=run_job_id, input_source_job_id=input_source_job_id, input_paths=req.input_paths, parameters=req.parameters, execute=req.execute, resolve_intent=_extension_stage_intent_resolver(extension_id, intent_policy), ) if run_job_id: current_job_id = run_job_id return result except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段自然语言运行门控失败: {exc}") @app.post("/api/extension-stages/{extension_id}/nl-runtime-gate-repair") async def repair_extension_stage_nl_runtime_gate(extension_id: str, req: ExtensionStageNLRuntimeGateRepairRequest): global current_job_id run_job_id = req.run_job_id or req.job_id or current_job_id input_source_job_id = req.input_source_job_id or req.job_id or run_job_id if req.execute and not run_job_id: raise HTTPException(status_code=400, detail="job_id is required for repairing an extension NL runtime gate") intent_policy = _validate_extension_stage_intent_policy(req.intent_policy) try: gate = extension_stage_agent.run_nl_runtime_gate( extension_id, message=req.message, job_id=req.job_id or run_job_id, run_job_id=run_job_id, input_source_job_id=input_source_job_id, input_paths=req.input_paths, parameters=req.parameters, execute=req.execute, resolve_intent=_extension_stage_intent_resolver(extension_id, intent_policy), ) if (gate.get("nl_runtime_gate") or {}).get("status") == "passed": if run_job_id: current_job_id = run_job_id return { "success": True, "phase": "nl_runtime_gate", "attempts": 0, "extension": gate.get("extension") or {}, "nl_runtime_gate": gate.get("nl_runtime_gate") or {}, "gate_record": gate.get("gate_record") or {}, "published": False, } run_context = { "repair_phase": "nl_runtime_gate", "status": "failed" if (gate.get("nl_runtime_gate") or {}).get("status") != "passed" else "passed", "error": (gate.get("nl_runtime_gate") or {}).get("failure_type", ""), "nl_runtime_gate": gate.get("gate_record") or {}, "nl_runtime_gate_summary": gate.get("nl_runtime_gate") or {}, "nl_runtime_gate_request": { "message": req.message, "job_id": req.job_id or run_job_id, "run_job_id": run_job_id, "input_source_job_id": input_source_job_id, "input_paths": req.input_paths, "parameters": req.parameters, "execute": req.execute, "intent_policy": intent_policy, }, } result = extension_stage_agent.repair_from_context( extension_id, phase="nl_runtime_gate", run_context=run_context, max_attempts=req.max_attempts, auto_publish=req.auto_publish, resolve_intent=_extension_stage_intent_resolver(extension_id, intent_policy), ) if run_job_id: current_job_id = run_job_id return result except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段自然语言运行门控修订失败: {exc}") @app.get("/api/extension-stages") async def list_extension_stages(include_drafts: bool = False): return {"success": True, "extensions": extension_stage_agent.list_extensions(include_drafts=include_drafts)} @app.get("/api/extension-stages/{extension_id}") async def get_extension_stage(extension_id: str): try: return {"success": True, "extension": extension_stage_agent.load(extension_id)} except ExtensionStageError as exc: raise HTTPException(status_code=404, detail=str(exc)) @app.post("/api/extension-stages/run") async def run_extension_stage(req: ExtensionStageRunRequest): global current_job_id job_id = req.job_id or current_job_id if not job_id: raise HTTPException(status_code=400, detail="job_id is required for running an extension analysis stage") try: result = extension_stage_agent.run( extension_id=req.extension_id, job_id=job_id, input_paths=req.input_paths, parameters=req.parameters, ) current_job_id = job_id return result except ExtensionStageError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=f"拓展分析阶段执行失败: {exc}") @app.get("/api/extension-stage-runs") async def list_extension_stage_runs(job_id: str = ""): return {"success": True, "runs": extension_stage_agent.list_runs(job_id=job_id)} @app.get("/api/extension-stage-file") async def get_extension_stage_file(job_id: str, run_id: str, path: str): try: file_path = extension_stage_agent.resolve_run_file(job_id, run_id, path) return FileResponse(str(file_path), filename=file_path.name) except ExtensionStageError as exc: raise HTTPException(status_code=404, detail=str(exc)) class AutofillDraftRequest(BaseModel): job_id: str = "" source_dir: str = "" description: str = "" species_code: str = "" species_name: str = "" kegg_organism: str = "" analysis_family: str = "" primary_reference_species: str = "" identify_blast_reference_species: str = "" gene_family: str = "" design: str = "" control_group: str = "" de_module: str = "" comparison_axis: str = "" reference_condition: str = "" reference_genotype: str = "" reference_tissue: str = "" sample_info_path: str = "" draft_path: str = "" rows: list[dict] = Field(default_factory=list) class IdentifyDomainReviewConfirmRequest(BaseModel): job_id: str prefix: str selected_ids: List[str] = Field(default_factory=list) source: str = "user_filter" class SubfamilyReviewConfirmRequest(BaseModel): job_id: str prefix: str rows: List[Dict[str, Any]] = Field(default_factory=list) source: str = "user_confirm" def _autofill_job_dirs(job_id: str) -> tuple[str, Path, Path]: global current_job_id resolved = job_id or current_job_id if not resolved: resolved = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:18] current_job_id = resolved input_dir = INPUT_ROOT / resolved workspace_dir = WORKSPACE_ROOT / resolved input_dir.mkdir(parents=True, exist_ok=True) (workspace_dir / "final").mkdir(parents=True, exist_ok=True) return resolved, input_dir, workspace_dir def _extract_path_from_text(text: str, suffixes: tuple[str, ...]) -> str: for match in re.finditer(r"(?:~|/)[^\s，,;；]+", str(text or "")): value = match.group(0).strip().strip("\"'") if value.lower().endswith(suffixes): return value return "" def _find_sample_info_file(*roots: Path) -> Path | None: candidates = [] for root in roots: if not root or not root.exists(): continue iterator = root.iterdir() if root.is_dir() else [root] for path in iterator: if path.is_file() and _is_sample_info_upload(path.name): candidates.append(path) return sorted(candidates, key=lambda p: (len(p.name), p.name))[0] if candidates else None def _run_autofill_cli(cmd: list[str]) -> dict: result = subprocess.run(cmd, cwd=str(PROJECT_ROOT), capture_output=True, text=True, timeout=120) if result.returncode != 0: raise HTTPException(status_code=500, detail=(result.stderr or result.stdout or "autofill failed")) try: return json.loads(result.stdout) except Exception: return {"stdout": result.stdout, "stderr": result.stderr} def _read_config_draft(path: Path) -> dict: try: from openpyxl import load_workbook except Exception as exc: raise HTTPException(status_code=500, detail=f"openpyxl unavailable: {exc}") from gfa_core.config_templates import config_template_examples wb = load_workbook(path) analysis = {} if "Analysis" in wb.sheetnames: analysis_ws = wb["Analysis"] for values in analysis_ws.iter_rows(min_row=2, values_only=True): key = str(values[0] if values and values[0] is not None else "").strip() value = str(values[1] if len(values) > 1 and values[1] is not None else "").strip() if key: analysis[key] = value ws = wb["config"] if "config" in wb.sheetnames else wb.active headers = [str(cell.value or "") for cell in ws[1]] rows = [] for values in ws.iter_rows(min_row=2, values_only=True): row = {headers[i]: (values[i] if i < len(values) and values[i] is not None else "") for i in range(len(headers))} if row.get("Original_File_Name"): rows.append(row) if "_autofill_review" in wb.sheetnames: review = wb["_autofill_review"] review_headers = [str(cell.value or "") for cell in review[1]] review_by_name = {} for values in review.iter_rows(min_row=2, values_only=True): item = {review_headers[i]: (values[i] if i < len(values) and values[i] is not None else "") for i in range(len(review_headers))} if item.get("Original_File_Name"): review_by_name[item["Original_File_Name"]] = item for row in rows: item = review_by_name.get(row.get("Original_File_Name"), {}) row["confidence"] = item.get("confidence", "") row["notes"] = item.get("notes", "") examples = [] example_sheet = "official_template" if "official_template" in wb.sheetnames else "examples" if "examples" in wb.sheetnames else "" if example_sheet: ex = wb[example_sheet] ex_headers = [str(cell.value or "") for cell in ex[1]] for values in ex.iter_rows(min_row=2, values_only=True): row = {ex_headers[i]: (values[i] if i < len(values) and values[i] is not None else "") for i in range(len(ex_headers))} if row.get("Original_File_Name"): examples.append(row) if not examples: examples = config_template_examples() return {"headers": headers, "rows": rows, "examples": examples, "analysis": analysis} def _read_config_analysis_policy(path: Path) -> dict[str, str]: policy = {"family": "", "primary_reference_species": "", "identify_blast_reference_species": ""} try: from openpyxl import load_workbook wb = load_workbook(path, read_only=True, data_only=True) if "Analysis" not in wb.sheetnames: return policy ws = wb["Analysis"] for values in ws.iter_rows(min_row=2, values_only=True): key = str(values[0] if values and values[0] is not None else "").strip().lower() value = str(values[1] if len(values) > 1 and values[1] is not None else "").strip() if not value or value.lower() in {"nan", "none"}: continue if key in {"family", "gene_family", "analysis_family", "project_family"}: policy["family"] = value elif key == "primary_reference_species": policy["primary_reference_species"] = value elif key == "identify_blast_reference_species": policy["identify_blast_reference_species"] = value elif key == "reference_species" and not policy["primary_reference_species"]: policy["primary_reference_species"] = value except Exception: return policy return policy def _read_config_analysis_family(path: Path) -> str: return _read_config_analysis_policy(path).get("family", "") def _read_yaml_draft(path: Path) -> dict: try: import yaml except Exception as exc: raise HTTPException(status_code=500, detail=f"PyYAML unavailable: {exc}") data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} autofill = data.get("_autofill") if isinstance(data.get("_autofill"), dict) else {} rows = autofill.get("rows", []) if not rows and isinstance(data.get("samples"), dict): rows = [] for sample_id, meta in data.get("samples", {}).items(): item = meta if isinstance(meta, dict) else {} rows.append({ "run": item.get("run", ""), "sample_id": str(sample_id), "fastq_1": item.get("r1", item.get("fastq_1", "")), "fastq_2": item.get("r2", item.get("fastq_2", "")), "genome_version": item.get("genome_version", item.get("genome", "")), "condition": item.get("condition", ""), "genotype": item.get("genotype", ""), "tissue": item.get("tissue", ""), "time": item.get("time", ""), "dose": item.get("dose", ""), "batch": item.get("batch", ""), "subject_id": item.get("subject_id", item.get("subject", item.get("plant_id", ""))), "replicate": item.get("replicate", ""), "phenotype": item.get("phenotype", ""), "library_strandness": item.get("library_strandness", data.get("rnaseq", {}).get("library_strandness", "")) if isinstance(data.get("rnaseq"), dict) else item.get("library_strandness", ""), "group_role": item.get("group_role", ""), "include": item.get("include", True), "confidence": "confirmed", "note": item.get("note", item.get("notes", "")), "notes": item.get("notes", ""), }) return { "rows": rows, "warnings": autofill.get("warnings", []), "design_review": autofill.get("design_review", {}), "examples": autofill.get("examples", {}), "yaml": data, } def _write_config_draft_rows(path: Path, rows: list[dict], analysis_family: str = "", primary_reference_species: str = "", identify_blast_reference_species: str = "") -> None: from openpyxl import Workbook from openpyxl.styles import Alignment, Font, PatternFill from gfa_core.config_templates import CONFIG_HEADERS, config_template_examples headers = CONFIG_HEADERS review_headers = [*CONFIG_HEADERS, "confidence", "notes"] wb = Workbook() ws = wb.active ws.title = "config" ws.append(headers) for row in rows: ws.append([row.get(col, "") for col in headers]) fill = PatternFill("solid", fgColor="D9EAF7") for cell in ws[1]: cell.font = Font(bold=True) cell.fill = fill cell.alignment = Alignment(horizontal="center") examples = wb.create_sheet("official_template") examples.append(headers) for row in config_template_examples(): examples.append([row.get(col, "") for col in headers]) for cell in examples[1]: cell.font = Font(bold=True) cell.fill = fill analysis = wb.create_sheet("Analysis") analysis.append(["Key", "Value", "Description"]) primary_reference_species = str(primary_reference_species or "").strip() identify_blast_reference_species = str(identify_blast_reference_species or "").strip() analysis.append(["Family", analysis_family, "Gene family for this job. This is the preferred source when the user does not name a family."]) analysis.append(["Target_Species", "", "Optional comma-separated target species codes."]) analysis.append(["Reference_Species", primary_reference_species, "Legacy/compatible downstream reference species. Mirrored from Primary_Reference_Species for old code paths."]) analysis.append(["Primary_Reference_Species", primary_reference_species, "Main downstream reference species. Usually one species."]) analysis.append(["Identify_Blast_Reference_Species", identify_blast_reference_species, "Reference family protein species used only as identify BLAST query evidence against the target whole-proteome database; can be multiple."]) analysis.append(["Comparative_Species", "", "Optional comma-separated comparative/outgroup species codes."]) analysis.append(["Required_Pfams", "", "Optional Pfam accession(s) for identify."]) for cell in analysis[1]: cell.font = Font(bold=True) cell.fill = PatternFill("solid", fgColor="E2F0D9") cell.alignment = Alignment(horizontal="center") review = wb.create_sheet("_autofill_review") review.append(review_headers) for row in rows: review.append([row.get(col, "") for col in review_headers]) for cell in review[1]: cell.font = Font(bold=True) cell.fill = fill wb.save(path) def _normalize_sample_group_role(value: str) -> str: if not str(value or "").strip(): return "" try: return normalize_group_role(value) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) def _sample_de_group_label(item: dict, final_design: str) -> str: condition = str(item.get("condition") or "").strip() time_value = str(item.get("time") or "").strip() if final_design == "time_series" and time_value: return time_value return condition or time_value def _sample_metadata_from_rows(rows: list[dict]) -> dict[str, dict]: samples: dict[str, dict] = {} for row in rows: sample_id = str(row.get("sample_id") or "").strip() if not sample_id: continue role = _normalize_sample_group_role(row.get("group_role", "")) include = parse_bool(row.get("include"), default=True) and role != "ignore" samples[sample_id] = { "condition": row.get("condition", ""), "genotype": row.get("genotype", ""), "tissue": row.get("tissue", ""), "time": row.get("time", ""), "dose": row.get("dose", ""), "batch": row.get("batch", ""), "subject_id": row.get("subject_id", ""), "replicate": row.get("replicate", ""), "group_role": role, "include": include, } return samples def _plan_samples_design( samples: dict[str, dict], data: dict, *, final_design: str = "", requested_control: str = "", de_module: str = "", comparison_axis: str = "", reference_condition: str = "", reference_genotype: str = "", reference_tissue: str = "", ) -> tuple[dict, dict]: experiment_options = data.get("experiment") if isinstance(data.get("experiment"), dict) else {} de_options = data.get("de") if isinstance(data.get("de"), dict) else {} options = {**experiment_options, **de_options} if comparison_axis: options["comparison_axis"] = comparison_axis if reference_condition or requested_control: options["reference_condition"] = reference_condition or requested_control if reference_genotype: options["reference_genotype"] = reference_genotype if reference_tissue: options["reference_tissue"] = reference_tissue plan = plan_rnaseq_de_design( samples, matrix_samples=list(samples), configured_compare=None, user_module=de_module, design_options=options, ) if plan.get("module") == "M1": if final_design and final_design != plan.get("design"): raise ValueError( f"表格分组角色推断为 {plan.get('design')}，与上方 design={final_design} 不一致；" "请清空 design 或修正样本表。" ) if requested_control and requested_control != plan.get("control_group"): raise ValueError( f"上方 control_group={requested_control} 与表格标记的 " f"control={plan.get('control_group')} 不一致，请统一后再运行 DE。" ) return plan, build_de_design_review(plan) def _write_samples_draft_rows( path: Path, rows: list[dict], design: str = "", control_group: str = "", species_code: str = "", species_name: str = "", kegg_organism: str = "", de_module: str = "", comparison_axis: str = "", reference_condition: str = "", reference_genotype: str = "", reference_tissue: str = "", ) -> dict: import yaml from apps.cli.run_samples_autofill import clean_confirmed_autofill_warnings data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} samples = {} raw_items: list[tuple[str, dict]] = [] row_time_values = {str(row.get("time") or "").strip() for row in rows if str(row.get("time") or "").strip()} row_conditions = {str(row.get("condition") or "").strip() for row in rows if str(row.get("condition") or "").strip()} final_design = (design or data.get("experiment", {}).get("design") or "").strip() if final_design.lower() == "manual": final_design = "" for row in rows: sample_id = str(row.get("sample_id") or "").strip() if not sample_id: continue role = _normalize_sample_group_role(row.get("group_role", "")) include = parse_bool(row.get("include"), default=True) and role != "ignore" item = { "run": row.get("run", ""), "r1": row.get("fastq_1", ""), "r2": row.get("fastq_2", ""), "genome_version": row.get("genome_version", ""), "condition": row.get("condition", ""), "genotype": row.get("genotype", ""), "tissue": row.get("tissue", ""), "time": row.get("time", ""), "dose": row.get("dose", ""), "batch": row.get("batch", ""), "subject_id": row.get("subject_id", ""), "replicate": row.get("replicate", ""), "library_strandness": row.get("library_strandness", ""), "group_role": role, "include": include, } phenotype_text = str(row.get("phenotype") or "").strip() if phenotype_text: if "=" in phenotype_text: key, value = phenotype_text.split("=", 1) key = key.strip() if key: item[key if key.startswith("phenotype_") else f"phenotype_{key}"] = value.strip() else: item["phenotype"] = phenotype_text note = str(row.get("note") or row.get("notes") or "").strip() if note: item["note"] = note samples[sample_id] = item raw_items.append((sample_id, item)) data["samples"] = samples code = str(species_code or data.get("speciesCode") or data.get("species", {}).get("code") or "").strip() latin = str(species_name or data.get("speciesName") or data.get("species", {}).get("speciesName") or data.get("species", {}).get("latin_name") or "").strip() kegg = str(kegg_organism or data.get("keggOrganism") or data.get("species", {}).get("keggOrganism") or data.get("species", {}).get("kegg_organism") or "").strip().lower() if code or latin or kegg: data["species"] = {"code": code, "speciesName": latin, "keggOrganism": kegg} data["speciesCode"] = code data["speciesName"] = latin data["keggOrganism"] = kegg requested_control = str(control_group or "").strip() de_warning = "" design_review: dict = {} try: design_plan, design_review = _plan_samples_design( samples, data, final_design=final_design, requested_control=requested_control, de_module=de_module, comparison_axis=comparison_axis, reference_condition=reference_condition, reference_genotype=reference_genotype, reference_tissue=reference_tissue, ) except ValueError as exc: design_plan = {"groups": {}, "experiment": {}, "compare": [], "de": {}} de_warning = f"DE design pending user input: {exc}" design_review = { "status": "needs_input", "error": str(exc), } data["groups"] = design_plan.get("groups", {}) data["experiment"] = design_plan.get("experiment") or { "design": design_plan.get("design", ""), "advanced_factor": design_plan.get("advanced_factor", ""), } data["compare"] = design_plan.get("compare", []) de_config = dict(design_plan.get("de") or {}) if de_module: de_config["module"] = de_module if comparison_axis: de_config["comparison_axis"] = comparison_axis for key, value in { "reference_condition": reference_condition, "reference_genotype": reference_genotype, "reference_tissue": reference_tissue, }.items(): if value: de_config[key] = value data["de"] = de_config autofill = data.setdefault("_autofill", {}) autofill["rows"] = rows autofill["design_review"] = design_review warnings = clean_confirmed_autofill_warnings(autofill.get("warnings", [])) if de_warning: warnings.append(de_warning) autofill["warnings"] = warnings path.write_text(yaml.safe_dump(data, allow_unicode=True, sort_keys=False), encoding="utf-8") return design_review def _draft_design_review(draft_data: dict, req: AutofillDraftRequest | None = None) -> dict: req = req or AutofillDraftRequest() data = draft_data.get("yaml") if isinstance(draft_data.get("yaml"), dict) else {} rows = req.rows or draft_data.get("rows") or [] samples = _sample_metadata_from_rows(rows) if rows else data.get("samples", {}) if not samples: return {"status": "needs_input", "error": "样品表中没有可用于设计审查的样本。"} try: _plan, review = _plan_samples_design( samples, data, final_design=str(req.design or "").strip(), requested_control=str(req.control_group or "").strip(), de_module=str(req.de_module or "").strip(), comparison_axis=str(req.comparison_axis or "").strip(), reference_condition=str(req.reference_condition or "").strip(), reference_genotype=str(req.reference_genotype or "").strip(), reference_tissue=str(req.reference_tissue or "").strip(), ) return review except (ValueError, HTTPException) as exc: detail = exc.detail if isinstance(exc, HTTPException) else str(exc) return { "status": "needs_input", "error": str(detail), } @app.get("/api/autofill/config/template") async def get_config_autofill_template(): from gfa_core.config_templates import CONFIG_HEADERS, config_template_examples return {"success": True, "headers": CONFIG_HEADERS, "examples": config_template_examples()} @app.post("/api/autofill/config/draft") async def draft_config_autofill(req: AutofillDraftRequest): job_id, input_dir, workspace_dir = _autofill_job_dirs(req.job_id) cmd = [ "/home/xzq/miniconda3/envs/test4py311/bin/python", "apps/cli/run_config_autofill.py", "--input-dir", str(input_dir), "--workspace", str(workspace_dir), ] if req.source_dir: cmd.extend(["--source-dir", req.source_dir]) if req.description: cmd.extend(["--description", req.description]) if req.species_code: cmd.extend(["--species-code", req.species_code]) family = (req.analysis_family or req.gene_family or "").strip() if family: cmd.extend(["--gene-family", family]) if req.primary_reference_species: cmd.extend(["--primary-reference-species", req.primary_reference_species]) if req.identify_blast_reference_species: cmd.extend(["--identify-blast-reference-species", req.identify_blast_reference_species]) manifest = _run_autofill_cli(cmd) draft_path = Path(manifest.get("artifacts", {}).get("draft_config", {}).get("path", workspace_dir / "final" / "config.autofill.draft.xlsx")) draft_abs = draft_path if draft_path.is_absolute() else PROJECT_ROOT / draft_path draft_data = _read_config_draft(draft_abs) return {"success": True, "job_id": job_id, "manifest": manifest, "draft": draft_data, "draft_path": str(draft_path)} @app.post("/api/autofill/samples/draft") async def draft_samples_autofill(req: AutofillDraftRequest): job_id, input_dir, workspace_dir = _autofill_job_dirs(req.job_id) cmd = [ "/home/xzq/miniconda3/envs/test4py311/bin/python", "apps/cli/run_samples_autofill.py", "--input-dir", str(input_dir), "--workspace", str(workspace_dir), ] if req.source_dir: cmd.extend(["--source-dir", req.source_dir]) sample_info = (req.sample_info_path or _extract_path_from_text(req.description, (".xlsx", ".xlsm", ".xls", ".csv", ".tsv", ".txt"))).strip() if not sample_info: source_path = Path(req.source_dir).expanduser() if req.source_dir else input_dir found_sample_info = _find_sample_info_file(input_dir, source_path, source_path.parent if source_path.exists() else Path()) sample_info = str(found_sample_info) if found_sample_info else "" if sample_info: cmd.extend(["--sample-info", sample_info]) if req.description: cmd.extend(["--description", req.description]) if req.design: cmd.extend(["--design", req.design]) if req.control_group: cmd.extend(["--control-group", req.control_group]) if req.species_code: cmd.extend(["--species-code", req.species_code]) if req.species_name: cmd.extend(["--species-name", req.species_name]) if req.kegg_organism: cmd.extend(["--kegg-organism", req.kegg_organism]) manifest = _run_autofill_cli(cmd) draft_path = Path(manifest.get("artifacts", {}).get("draft_samples", {}).get("path", workspace_dir / "final" / "samples.autofill.draft.yaml")) draft_data = _read_yaml_draft(draft_path if draft_path.is_absolute() else PROJECT_ROOT / draft_path) draft_data["design_review"] = _draft_design_review(draft_data, req) rows = draft_data.get("rows") if isinstance(draft_data, dict) else [] if not rows: return { "success": False, "job_id": job_id, "message": "未检测到可配对的 FASTQ 样本文件，未生成可确认的 samples.yaml 草稿。当前路径更像是基因组/注释/蛋白/CDS资源目录，应使用 config.xlsx 自动填充。", "manifest": manifest, "draft": draft_data, "draft_path": str(draft_path), } return {"success": True, "job_id": job_id, "manifest": manifest, "draft": draft_data, "draft_path": str(draft_path)} @app.post("/api/autofill/config/open") async def open_config_autofill(req: AutofillDraftRequest): job_id, input_dir, workspace_dir = _autofill_job_dirs(req.job_id) candidates = [ workspace_dir / "final" / "config.autofill.draft.xlsx", input_dir / "config.xlsx", ] for path in candidates: if path.exists(): draft_data = _read_config_draft(path) return { "success": True, "job_id": job_id, "draft": draft_data, "draft_path": str(path), "source": "draft" if path.name == "config.autofill.draft.xlsx" else "confirmed_config", } return {"success": False, "job_id": job_id, "message": "未找到 config.autofill.draft.xlsx 或 config.xlsx"} @app.post("/api/autofill/samples/open") async def open_samples_autofill(req: AutofillDraftRequest): job_id, input_dir, workspace_dir = _autofill_job_dirs(req.job_id) candidates = [ workspace_dir / "final" / "samples.autofill.draft.yaml", input_dir / "samples.yaml", input_dir / "samples.yml", ] for path in candidates: if path.exists(): draft_data = _read_yaml_draft(path) if not draft_data.get("design_review"): draft_data["design_review"] = _draft_design_review(draft_data, req) return { "success": True, "job_id": job_id, "draft": draft_data, "draft_path": str(path), "source": "draft" if path.name == "samples.autofill.draft.yaml" else "confirmed_samples", } return {"success": False, "job_id": job_id, "message": "未找到 samples.autofill.draft.yaml 或 samples.yaml"} @app.post("/api/autofill/samples/design-review") async def review_samples_autofill_design(req: AutofillDraftRequest): job_id, input_dir, workspace_dir = _autofill_job_dirs(req.job_id) draft_data: dict = {"rows": req.rows, "yaml": {}} candidates: list[Path] = [] if req.draft_path: candidate = Path(req.draft_path).expanduser() candidates.append(candidate if candidate.is_absolute() else PROJECT_ROOT / candidate) candidates.extend([ input_dir / "samples.yaml", input_dir / "samples.yml", workspace_dir / "final" / "samples.autofill.draft.yaml", workspace_dir / "final" / "samples.yaml", ]) for path in candidates: if path.exists(): loaded = _read_yaml_draft(path) draft_data["yaml"] = loaded.get("yaml", {}) if not req.rows: draft_data["rows"] = loaded.get("rows", []) break review = _draft_design_review(draft_data, req) return { "success": review.get("status") == "ready", "job_id": job_id, "design_review": review, "message": review.get("error", ""), } @app.post("/api/autofill/config/confirm") async def confirm_config_autofill(req: AutofillDraftRequest): job_id, input_dir, workspace_dir = _autofill_job_dirs(req.job_id) draft_to_use = req.draft_path if req.rows: draft_path = Path(req.draft_path).expanduser() if req.draft_path else workspace_dir / "final" / "config.autofill.draft.xlsx" if not draft_path.is_absolute(): draft_path = PROJECT_ROOT / draft_path existing_policy = _read_config_analysis_policy(draft_path) family = (req.analysis_family or req.gene_family or existing_policy.get("family", "")).strip() primary_reference_species = (req.primary_reference_species or existing_policy.get("primary_reference_species", "")).strip() identify_blast_reference_species = (req.identify_blast_reference_species or existing_policy.get("identify_blast_reference_species", "")).strip() _write_config_draft_rows(draft_path, req.rows, family, primary_reference_species, identify_blast_reference_species) draft_to_use = str(draft_path) cmd = [ "/home/xzq/miniconda3/envs/test4py311/bin/python", "apps/cli/run_config_autofill.py", "--input-dir", str(input_dir), "--workspace", str(workspace_dir), "--confirm", ] if draft_to_use: cmd.extend(["--draft", draft_to_use]) family = (req.analysis_family or req.gene_family or "").strip() if family: cmd.extend(["--gene-family", family]) if req.primary_reference_species: cmd.extend(["--primary-reference-species", req.primary_reference_species]) if req.identify_blast_reference_species: cmd.extend(["--identify-blast-reference-species", req.identify_blast_reference_species]) manifest = _run_autofill_cli(cmd) return {"success": True, "job_id": job_id, "manifest": manifest} @app.post("/api/autofill/samples/confirm") async def confirm_samples_autofill(req: AutofillDraftRequest): job_id, input_dir, workspace_dir = _autofill_job_dirs(req.job_id) draft_to_use = req.draft_path if req.rows: draft_path = Path(req.draft_path).expanduser() if req.draft_path else workspace_dir / "final" / "samples.autofill.draft.yaml" if not draft_path.is_absolute(): draft_path = PROJECT_ROOT / draft_path design_review = _write_samples_draft_rows( draft_path, req.rows, req.design, req.control_group, req.species_code, req.species_name, req.kegg_organism, req.de_module, req.comparison_axis, req.reference_condition, req.reference_genotype, req.reference_tissue, ) draft_to_use = str(draft_path) else: design_review = _draft_design_review( _read_yaml_draft(Path(draft_to_use)) if draft_to_use and Path(draft_to_use).exists() else {"rows": [], "yaml": {}}, req, ) cmd = [ "/home/xzq/miniconda3/envs/test4py311/bin/python", "apps/cli/run_samples_autofill.py", "--input-dir", str(input_dir), "--workspace", str(workspace_dir), "--confirm", ] if draft_to_use: cmd.extend(["--draft", draft_to_use]) manifest = _run_autofill_cli(cmd) return {"success": True, "job_id": job_id, "manifest": manifest, "design_review": design_review} def _canonicalize_species_family_prefix(job_id: str, prefix: str) -> str: parts = [p for p in str(prefix or "").split(".") if p] if len(parts) == 2 and parts[0] != "Target": species = _canonicalize_species_codes_for_job(job_id, [parts[0]]) if species: return f"{species[0]}.{parts[1]}" if len(parts) == 3 and parts[0] == "Target": species = _canonicalize_species_codes_for_job(job_id, [parts[1]]) if species: return f"Target.{species[0]}.{parts[2]}" return str(prefix or "") def _domain_review_path(job_id: str, prefix: str) -> Path: prefix = _nonempty_prefix(prefix) if not prefix: raise HTTPException(status_code=400, detail="prefix is required") return WORKSPACE_ROOT / job_id / "final" / f"{prefix}.domain_review.tsv" def _single_target_prefix(prefix: str) -> str: parts = [p for p in str(prefix or "").split(".") if p] if len(parts) == 2 and parts[0] != "Target": return f"Target.{parts[0]}.{parts[1]}" if len(parts) == 3 and parts[0] == "Target" and parts[1] != "SPL": return prefix return "" def _domain_review_paths(job_id: str, prefix: str) -> list[Path]: final_dir = WORKSPACE_ROOT / job_id / "final" prefix = _nonempty_prefix(prefix) if not prefix: raise HTTPException(status_code=400, detail="prefix is required") candidates: list[Path] = [] target_prefix = _single_target_prefix(prefix) if target_prefix: target_path = _domain_review_path(job_id, target_prefix) if target_path.exists() and target_path.stat().st_size > 0: candidates.append(target_path) elif target_prefix != prefix: candidates.append(_domain_review_path(job_id, prefix)) else: exact = _domain_review_path(job_id, prefix) candidates.append(exact) parts = prefix.split(".") family = parts[-1] if parts else "" if family: candidates.extend(sorted(final_dir.glob(f"*.{family}.domain_review.tsv"))) candidates.extend(sorted(final_dir.glob("*domain_review.tsv"))) seen: set[Path] = set() out: list[Path] = [] for path in candidates: try: resolved = path.resolve() except Exception: resolved = path if resolved in seen or not path.exists() or path.stat().st_size == 0: continue seen.add(resolved) out.append(path) return out def _read_domain_review_rows(path: Path, limit: int = 2000) -> list[dict]: if not path.exists(): return [] rows: list[dict] = [] with path.open("r", encoding="utf-8", errors="ignore") as handle: header: list[str] = [] for line in handle: if not line.strip(): continue cols = line.rstrip("\n").split("\t") if not header: header = cols continue row = {header[idx]: cols[idx] if idx < len(cols) else "" for idx in range(len(header))} for key in ("domain_coverage", "score_or_evalue"): try: row[f"{key}_num"] = float(row.get(key, "")) except Exception: row[f"{key}_num"] = None row["required_domain_bool"] = str(row.get("required_domain", "")).upper() == "TRUE" row["default_keep_bool"] = str(row.get("default_keep", "")).upper() == "TRUE" row["source_file"] = path.name row.setdefault("species", _species_from_domain_review_name(path.name)) rows.append(row) if len(rows) >= limit: break return rows def _identify_species_family_from_prefix(prefix: str) -> tuple[str, str]: parts = [part for part in str(prefix or "").split(".") if part] if len(parts) >= 3 and parts[0] == "Target": return parts[1], parts[2] if len(parts) >= 2: return parts[0], parts[1] return "", parts[-1] if parts else "" def _identify_final_member_id_candidates(final_dir: Path, prefix: str) -> list[Path]: species, family = _identify_species_family_from_prefix(prefix) candidates: list[Path] = [] for stem in ("identified_members", "final_members"): for name in [ f"{prefix}.{stem}.id", f"Target.{species}.{family}.{stem}.id" if species and family else "", f"{species}.{family}.{stem}.id" if species and family else "", ]: if name: candidates.append(final_dir / name) if family: candidates.extend(sorted(final_dir.glob(f"*.{family}.{stem}.id"))) candidates.extend(sorted(final_dir.glob(f"*.{stem}.id"))) seen: set[Path] = set() out: list[Path] = [] for path in candidates: try: resolved = path.resolve() except Exception: resolved = path if resolved in seen or not path.exists() or path.stat().st_size == 0: continue seen.add(resolved) out.append(path) return out def _read_first_existing_id_file(paths: list[Path]) -> tuple[list[str], str]: for path in paths: ids = sorted(_read_id_file(path)) if ids: return ids, str(path.resolve()) return [], "" def _read_blast_source_matrix(path: Path) -> dict[str, dict[str, str]]: if not path.exists() or path.stat().st_size == 0: return {} rows: dict[str, dict[str, str]] = {} with path.open("r", encoding="utf-8", errors="ignore", newline="") as handle: reader = csv.DictReader(handle, delimiter="\t") for row in reader: gene_id = str(row.get("gene_id") or "").strip() if gene_id: rows[gene_id] = {str(k): str(v or "") for k, v in row.items()} return rows def _identify_member_review_fallback(job_id: str, prefix: str, limit: int = 5000) -> dict[str, Any]: final_dir = WORKSPACE_ROOT / job_id / "final" workspace_dir = WORKSPACE_ROOT / job_id species, family = _identify_species_family_from_prefix(prefix) member_ids, member_path = _read_first_existing_id_file(_identify_final_member_id_candidates(final_dir, prefix)) if not member_ids: return {"rows": [], "default_ids": [], "paths": [], "species_groups": {}, "message": "未找到 identified_members.id，无法生成成员确认表。"} analysis_dir = workspace_dir / "identify" / species / "analysis" if species else workspace_dir / "identify" hmm_ids: set[str] = set() blast_ids: set[str] = set() matrix_rows: dict[str, dict[str, str]] = {} if analysis_dir.exists(): for path in sorted(analysis_dir.glob("*.hmm.ids")): hmm_ids.update(_read_id_file(path)) for path in sorted(analysis_dir.glob("*.blast.ids")): blast_ids.update(_read_id_file(path)) matrix_candidates = sorted(analysis_dir.glob("*.blast.source_matrix.tsv")) if matrix_candidates: matrix_rows = _read_blast_source_matrix(matrix_candidates[0]) rows: list[dict[str, Any]] = [] for gene_id in member_ids[:limit]: matrix = matrix_rows.get(gene_id, {}) hmm_hit = gene_id in hmm_ids blast_hit = gene_id in blast_ids evidence = [] if hmm_hit: evidence.append("HMM") if blast_hit: evidence.append("BLAST") rows.append({ "gene_id": gene_id, "species": species, "analysis": "+".join(evidence) if evidence else "identified_members", "signature_accession": "member_evidence", "signature_description": "HMM/BLAST final member evidence fallback", "interpro_accession": "", "interpro_description": "", "start": "", "end": "", "protein_length": "", "domain_coverage": "", "score_or_evalue": matrix.get("best_evalue", ""), "domain_coverage_num": None, "score_or_evalue_num": None, "required_domain_bool": False, "default_keep_bool": True, "hmm_hit": hmm_hit, "blast_hit": blast_hit, "blast_reference_sources": matrix.get("blast_reference_sources", ""), "blast_reference_count": matrix.get("blast_reference_count", ""), "blast_query_ids": matrix.get("blast_query_ids", ""), "best_bitscore": matrix.get("best_bitscore", ""), "review_source": "member_evidence_fallback", "source_file": Path(member_path).name if member_path else "", }) return { "rows": rows, "default_ids": member_ids, "paths": [member_path] if member_path else [], "species_groups": {species or "merged": len(rows)}, "message": "未找到 InterProScan domain_review.tsv，已使用 identified_members + HMM/BLAST 证据生成成员确认表。", } def _species_from_domain_review_name(name: str) -> str: parts = str(name or "").split(".") if len(parts) >= 5 and parts[0] == "Target" and parts[-3] != "": # Target.Dl.SPL.domain_review.tsv -> Dl return parts[1] if len(parts) >= 4 and parts[0] not in {"Target", "Multi"}: # Dl.SPL.domain_review.tsv -> Dl return parts[0] return "" @app.get("/api/identify/domain_review") async def get_identify_domain_review(job_id: str = Query(""), prefix: str = Query("")): if not job_id: return {"success": False, "message": "job_id is required", "rows": []} effective_prefix = _canonicalize_species_family_prefix(job_id, resolve_effective_prefix(job_id, {"prefix": prefix}, "identify")) review_paths = _domain_review_paths(job_id, effective_prefix) rows: list[dict] = [] species_groups: dict[str, int] = {} for review_path in review_paths: file_rows = _read_domain_review_rows(review_path) rows.extend(file_rows) species = _species_from_domain_review_name(review_path.name) or "merged" species_groups[species] = species_groups.get(species, 0) + len(file_rows) review_source = "interproscan_domain_review" if rows else "member_evidence_fallback" fallback: dict[str, Any] = {} if not rows: fallback = _identify_member_review_fallback(job_id, effective_prefix) rows = fallback.get("rows") or [] species_groups = fallback.get("species_groups") or {} default_ids = sorted({row.get("gene_id", "") for row in rows if row.get("default_keep_bool") and row.get("gene_id")}) if not default_ids and fallback.get("default_ids"): default_ids = list(fallback.get("default_ids") or []) response_paths = [str(path.resolve()) for path in review_paths] if review_paths else list(fallback.get("paths") or []) response_path = response_paths[0] if response_paths else str(_domain_review_path(job_id, effective_prefix)) return { "success": bool(rows), "job_id": job_id, "prefix": effective_prefix, "path": response_path, "paths": response_paths, "review_source": review_source, "rows": rows, "row_count": len(rows), "species_groups": species_groups, "default_ids": default_ids, "default_count": len(default_ids), "message": fallback.get("message") or "默认不筛选时，仅保留命中本基因家族目标结构域的成员。", } @app.post("/api/identify/domain_review/confirm") async def confirm_identify_domain_review(req: IdentifyDomainReviewConfirmRequest): job_id = req.job_id if not job_id: raise HTTPException(status_code=400, detail="job_id is required") prefix = _canonicalize_species_family_prefix(job_id, resolve_effective_prefix(job_id, {"prefix": req.prefix}, "identify")) final_dir = WORKSPACE_ROOT / job_id / "final" input_dir = INPUT_ROOT / job_id workspace_dir = WORKSPACE_ROOT / job_id selected: list[str] = [] seen: set[str] = set() for raw in req.selected_ids or []: seq_id = str(raw or "").strip().split()[0] if seq_id and seq_id not in seen: selected.append(seq_id) seen.add(seq_id) review_paths = _domain_review_paths(job_id, prefix) review_rows: list[dict] = [] for review_path in review_paths: review_rows.extend(_read_domain_review_rows(review_path)) if not selected: selected = sorted({row.get("gene_id", "") for row in review_rows if row.get("default_keep_bool") and row.get("gene_id")}) if not selected: raise HTTPException(status_code=400, detail="No selected IDs; cannot refine identify members") target_prefix = _single_target_prefix(prefix) if target_prefix: allowed_members = {str(row.get("gene_id") or "").strip() for row in review_rows if row.get("gene_id")} if not allowed_members: allowed_ids, _allowed_path = _read_first_existing_id_file(_identify_final_member_id_candidates(final_dir, target_prefix)) allowed_members = set(allowed_ids) foreign_ids = [seq_id for seq_id in selected if allowed_members and seq_id not in allowed_members] if foreign_ids: raise HTTPException( status_code=400, detail=( f"当前是 {target_prefix} 单物种成员确认，但检测到其他物种或非当前成员 ID: " + ", ".join(foreign_ids[:20]) + (" ..." if len(foreign_ids) > 20 else "") ), ) curated_prefix = target_prefix or prefix curated_file = final_dir / f"{curated_prefix}.curated_members.id" curated_file.parent.mkdir(parents=True, exist_ok=True) curated_file.write_text("\n".join(selected) + "\n", encoding="utf-8") cmd = [ "/home/xzq/miniconda3/envs/test4py311/bin/python", "apps/cli/run_identify_refine.py", "--input-dir", str(input_dir), "--workspace", str(workspace_dir), "--prefix", prefix, "--id-file", str(curated_file), "--allow-missing", "--curated-only", ] proc = subprocess.run(cmd, cwd=str(PROJECT_ROOT), capture_output=True, text=True) if proc.returncode != 0: raise HTTPException(status_code=500, detail=(proc.stderr or proc.stdout or "identify_refine failed")[-4000:]) return { "success": True, "job_id": job_id, "prefix": prefix, "selected_count": len(selected), "curated_file": str(curated_file.resolve()), "curated_pep_file": str((final_dir / f"{curated_prefix}.curated_members.pep.fa").resolve()), "requires_downstream_rerun": True, "invalidates_stages": ["phylogeny", "reference_classification", "motif", "domain", "structure", "combo"], "message": "Domain筛选已生成 curated_members；亚家族证据 phylogeny、motif、domain、structure、combo 必须基于筛选后的 curated 成员重新运行。final_members 将在联合确认后更新。", "stdout": proc.stdout[-4000:], } def _read_simple_tsv(path: Path, limit: int = 5000) -> list[dict]: if not path.exists() or path.stat().st_size == 0: return [] lines = [line.rstrip("\n") for line in path.read_text(encoding="utf-8", errors="ignore").splitlines() if line.strip()] if not lines: return [] first = lines[0].split("\t") header_tokens = {x.strip().lower() for x in first} has_header = bool(header_tokens & {"gene_id", "gene", "id", "subfamily", "predicted_subfamily", "confirmed_subfamily"}) rows: list[dict] = [] if has_header: header = [x.strip() for x in first] data_lines = lines[1:] for line in data_lines: cols = line.split("\t") rows.append({header[idx]: cols[idx] if idx < len(cols) else "" for idx in range(len(header))}) if len(rows) >= limit: break return rows for line in lines: cols = line.split("\t") if len(cols) >= 2: rows.append({"gene_id": cols[0].strip(), "confirmed_subfamily": cols[1].strip(), "predicted_subfamily": cols[1].strip()}) if len(rows) >= limit: break return rows def _manifest_declares_classifiable_species(manifest_path: Path, species: str) -> bool: if not species or not manifest_path.exists(): return False try: data = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: return False declared: list[str] = [] for key in ("classifiable_species", "target_species", "comparative_species"): for item in data.get(key) or []: item = str(item or "").strip() if item and item not in declared: declared.append(item) metrics = data.get("metrics") if isinstance(data.get("metrics"), dict) else {} for item in (metrics.get("scope_target_member_counts") or {}).keys(): item = str(item or "").strip() if item and item not in declared: declared.append(item) return species in declared def _joint_subfamily_prefixes_for_species(job_id: str, species: str, family: str) -> list[str]: final_dir = WORKSPACE_ROOT / job_id / "final" if not species or not family or not final_dir.exists(): return [] prefixes: list[str] = [] for manifest_path in sorted( final_dir.glob(f"Target.{family}*.reference_classification.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True, ): if not _manifest_declares_classifiable_species(manifest_path, species): continue output_prefix = manifest_path.name[: -len(".reference_classification.manifest.json")] if output_prefix not in prefixes: prefixes.append(output_prefix) base_prefix = output_prefix.replace(".reference_classification", "") if base_prefix not in prefixes: prefixes.append(base_prefix) return prefixes def _subfamily_source_path(job_id: str, prefix: str) -> tuple[Path, str]: final_dir = WORKSPACE_ROOT / job_id / "final" input_dir = INPUT_ROOT / job_id species = prefix.split(".", 1)[0] if "." in prefix else "" family = prefix.split(".", 1)[1] if "." in prefix else "" candidates = [ (final_dir / f"{prefix}.target_plus_reference.confirmed_subfamily.review.tsv", "confirmed_review"), (final_dir / f"{prefix}.target_plus_reference.confirmed_subfamily.tsv", "confirmed"), (final_dir / f"{prefix}.confirmed_subfamily.review.tsv", "confirmed_review"), (final_dir / f"{prefix}.confirmed_subfamily.tsv", "confirmed"), (input_dir / f"{prefix}.confirmed_subfamily.tsv", "input_confirmed"), ] if species and family: for pattern, source in ( (f"Target.{family}.target-{species}*.subfamily_prediction.confirmed_subfamily.review.tsv", "confirmed_review"), (f"Target.{family}.target-{species}*.subfamily_prediction.confirmed_subfamily.tsv", "confirmed"), (f"*target-{species}*.subfamily_prediction.confirmed_subfamily.review.tsv", "confirmed_review"), (f"*target-{species}*.subfamily_prediction.confirmed_subfamily.tsv", "confirmed"), ): for path in sorted(final_dir.glob(pattern), key=lambda p: p.stat().st_mtime, reverse=True): candidates.append((path, source)) for joint_prefix in _joint_subfamily_prefixes_for_species(job_id, species, family): candidates.extend([ (final_dir / f"{joint_prefix}.confirmed_subfamily.review.tsv", "confirmed_review"), (final_dir / f"{joint_prefix}.confirmed_subfamily.tsv", "confirmed"), ]) candidates.extend([ (final_dir / f"{prefix}.target_plus_reference.reference_classification.subfamily.tsv", "reference_classification"), (final_dir / f"{prefix}.reference_classification.subfamily.tsv", "reference_classification"), (final_dir / f"{prefix}.target_plus_reference.geneID.subfamily", "geneID_subfamily"), (final_dir / f"{prefix}.geneID.subfamily", "geneID_subfamily"), (input_dir / f"{prefix}.geneID.subfamily", "input_geneID_subfamily"), ]) if species and family: for joint_prefix in _joint_subfamily_prefixes_for_species(job_id, species, family): candidates.extend([ (final_dir / f"{joint_prefix}.reference_classification.subfamily.tsv", "reference_classification"), (final_dir / f"{joint_prefix}.geneID.subfamily", "geneID_subfamily"), ]) for path, source in candidates: if path.exists() and path.stat().st_size > 0: return path, source return final_dir / f"{prefix}.reference_classification.subfamily.tsv", "missing" def _latest_stage_manifest(final_dir: Path, prefix: str, stage: str) -> dict: candidates = sorted( final_dir.glob(f"{prefix}*.{stage}.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True, ) for path in candidates: try: data = json.loads(path.read_text(encoding="utf-8")) except Exception: continue if data.get("stage") == stage and data.get("status") in {"success", "partial_success"}: data["_manifest_path"] = str(path) return data return {} def _resolve_subfamily_review_prefix(job_id: str, prefix: str) -> str: final_dir = WORKSPACE_ROOT / job_id / "final" prefix = _nonempty_prefix(prefix) source_path, source_type = _subfamily_source_path(job_id, prefix) if source_type != "missing" and source_path.exists(): return prefix manifest = _latest_stage_manifest(final_dir, prefix, "reference_classification") output_prefix = str( manifest.get("output_prefix") or (manifest.get("parameters") or {}).get("output_prefix") or "" ).strip() return output_prefix or prefix def _source_prediction_manifest_path(job_id: str, prefix: str) -> Path | None: final_dir = WORKSPACE_ROOT / job_id / "final" prefix = _nonempty_prefix(prefix) if not prefix: return None candidate_prefixes = [prefix] if prefix.endswith(".confirmed_subfamily"): candidate_prefixes.append(prefix[: -len(".confirmed_subfamily")]) for candidate_prefix in dict.fromkeys(candidate_prefixes): direct = final_dir / f"{candidate_prefix}.reference_classification.manifest.json" if direct.exists(): return direct manifest = _latest_stage_manifest(final_dir, candidate_prefix, "reference_classification") manifest_path = str(manifest.get("_manifest_path") or "").strip() if manifest else "" if manifest_path: path = Path(manifest_path) if path.exists(): return path parts = [part for part in prefix.split(".") if part] if len(parts) >= 2 and parts[0] not in {"Target", "Multi"}: for joint_prefix in _joint_subfamily_prefixes_for_species(job_id, parts[0], parts[1]): direct = final_dir / f"{joint_prefix}.reference_classification.manifest.json" if direct.exists(): return direct return None def _normalize_subfamily_rows(rows: list[dict]) -> list[dict]: out = [] for row in rows: lowered = {str(k).lower(): str(v or "").strip() for k, v in row.items()} gene_id = lowered.get("gene_id") or lowered.get("gene") or lowered.get("id") or lowered.get("target_gene") or lowered.get("label") confirmed = lowered.get("confirmed_subfamily") or lowered.get("subfamily") or lowered.get("assigned_subfamily") or lowered.get("prediction") predicted = lowered.get("predicted_subfamily") or lowered.get("assigned_subfamily") or lowered.get("subfamily") or confirmed if not gene_id or not confirmed: continue out.append({ "gene_id": gene_id, "species": lowered.get("species") or lowered.get("species_code") or lowered.get("target_species") or "", "predicted_subfamily": predicted or confirmed, "confirmed_subfamily": confirmed, "confidence": lowered.get("confidence") or lowered.get("score") or "", "note": lowered.get("note") or lowered.get("status") or lowered.get("comment") or "", }) return out def _subfamily_assignment_candidates(job_id: str, prefix: str) -> list[Path]: final_dir = WORKSPACE_ROOT / job_id / "final" candidates: list[Path] = [] names = [] if prefix: names.extend([ f"{prefix}.target_plus_reference.reference_classification.assignment.tsv", f"{prefix}.reference_classification.assignment.tsv", ]) for name in names: path = final_dir / name if path not in candidates: candidates.append(path) if prefix: for path in sorted(final_dir.glob(f"{prefix}*.reference_classification.assignment.tsv")): if path not in candidates: candidates.append(path) return candidates def _read_subfamily_assignment_map(job_id: str, prefix: str) -> dict[str, dict]: for path in _subfamily_assignment_candidates(job_id, prefix): if not path.exists() or path.stat().st_size <= 0: continue try: with path.open(encoding="utf-8", errors="ignore", newline="") as handle: sample = handle.readline() delimiter = " " if " " in sample else "," handle.seek(0) reader = csv.DictReader(handle, delimiter=delimiter) out: dict[str, dict] = {} for row in reader: lowered = {str(k).lower(): str(v or "").strip() for k, v in row.items()} gene_id = lowered.get("target_gene") or lowered.get("gene_id") or lowered.get("gene") if not gene_id: continue out[gene_id] = lowered except Exception: out = {} if out: return out return {} def _merge_subfamily_assignment_metadata(rows: list[dict], assignments: dict[str, dict]) -> list[dict]: if not assignments: return rows merged: list[dict] = [] for row in rows: next_row = dict(row) meta = assignments.get(str(next_row.get("gene_id") or "")) if meta: assigned = meta.get("assigned_subfamily") or "" next_row["predicted_subfamily"] = assigned or next_row.get("predicted_subfamily", "") next_row["confidence"] = meta.get("confidence") or next_row.get("confidence", "") next_row["note"] = meta.get("classification_note") or meta.get("note") or next_row.get("note", "") for key in [ "nearest_reference", "nearest_reference_subfamily", "nearest_reference_distance", "nearest3_references", "minimal_mixed_clade_ref_subfamilies", "minimal_mixed_clade_bootstrap", "topology_reference_subfamily", "topology_bootstrap", "nearest3_majority_subfamily", "nearest3_majority_count", "classification_conflict", ]: if meta.get(key): next_row[key] = meta[key] merged.append(next_row) return merged def _read_id_file(path: Path) -> set[str]: if not path.exists(): return set() ids: set[str] = set() for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): token = line.strip().split() if token: ids.add(token[0]) return ids def _family_from_prefix_or_job(job_id: str, prefix: str) -> str: parts = [part for part in str(prefix or "").split(".") if part] if len(parts) >= 2 and parts[0] in {"Target", "Multi"}: return parts[1] if len(parts) >= 2: return parts[1] return _infer_family_from_config_or_final(job_id) def _declared_target_species(job_id: str) -> list[str]: selection = _load_identify_species_selection(job_id) or {} targets = [] for item in (selection.get("groups") or {}).get("target") or []: species = str(item.get("species") or "").strip() if species and species not in targets: targets.append(species) return targets def _declared_classifiable_species(job_id: str) -> list[str]: selection = _load_identify_species_selection(job_id) or {} species_list: list[str] = [] groups = selection.get("groups") or {} for group_name in ("target", "comparative", "other"): for item in groups.get(group_name) or []: species = str((item or {}).get("species") or "").strip() if species and species not in species_list: species_list.append(species) return species_list def _latest_gene_family_scope_defaults(job_id: str, family: str = "", prefix: str = "") -> dict[str, list[str]]: final_dir = WORKSPACE_ROOT / job_id / "final" if job_id else None if not final_dir or not final_dir.exists(): return {"target_species": [], "reference_species": [], "comparative_species": []} family_norm = str(family or "").strip() prefix_norm = str(prefix or "").strip() candidates = sorted(final_dir.glob("*.scope.json"), key=lambda path: path.stat().st_mtime if path.exists() else 0, reverse=True) fallback: dict[str, list[str]] = {"target_species": [], "reference_species": [], "comparative_species": []} for path in candidates: try: data = read_scope(path) except Exception: continue scope_family = str(data.get("family") or "").strip() scope_prefix = str(data.get("prefix") or "").strip() if family_norm and scope_family and scope_family != family_norm: continue targets = [str(x).strip() for x in (data.get("target_species") or []) if str(x).strip()] refs = [str(x).strip() for x in (data.get("reference_species") or []) if str(x).strip()] comps = [str(x).strip() for x in (data.get("comparative_species") or []) if str(x).strip()] current = {"target_species": targets, "reference_species": refs, "comparative_species": comps} if prefix_norm and scope_prefix == prefix_norm: return current if targets and not fallback.get("target_species"): fallback = current return fallback def _target_species_fallback_from_prefix(prefix: str) -> list[str]: prefix_species, _prefix_family = _prefix_species_family(prefix) if prefix_species and prefix_species not in {"Target", "Multi"}: return [prefix_species] return [] def _subfamily_species_member_maps(job_id: str, family: str) -> tuple[dict[str, set[str]], dict[str, str]]: final_dir = WORKSPACE_ROOT / job_id / "final" species_to_ids: dict[str, set[str]] = {} id_to_species: dict[str, str] = {} if not final_dir.exists() or not family: return species_to_ids, id_to_species candidates: list[tuple[str, Path]] = [] for species in _declared_classifiable_species(job_id): candidates.extend([ (species, final_dir / f"{species}.{family}.final_members.id"), (species, final_dir / f"Target.{species}.{family}.final_members.id"), (species, final_dir / f"{species}.{family}.curated_members.id"), (species, final_dir / f"Target.{species}.{family}.curated_members.id"), ]) for path in sorted(final_dir.glob(f"*.{family}.final_members.id")) + sorted(final_dir.glob(f"*.{family}.curated_members.id")): parts = path.name.split(".") if len(parts) < 3: continue species = parts[1] if parts[0] == "Target" and len(parts) >= 4 else parts[0] if species in {"Target", "Multi", family}: continue candidates.append((species, path)) seen: set[tuple[str, str]] = set() for species, path in candidates: key = (species, str(path)) if key in seen: continue seen.add(key) ids = _read_id_file(path) if not ids: continue species_to_ids.setdefault(species, set()).update(ids) for gene_id in ids: id_to_species.setdefault(gene_id, species) return species_to_ids, id_to_species def _annotate_subfamily_rows_with_species(job_id: str, prefix: str, rows: list[dict]) -> list[dict]: family = _family_from_prefix_or_job(job_id, prefix) _species_to_ids, id_to_species = _subfamily_species_member_maps(job_id, family) annotated = [] for row in rows: next_row = dict(row) species = str(next_row.get("species") or "").strip() if not species: species = id_to_species.get(str(next_row.get("gene_id") or ""), "") next_row["species"] = species annotated.append(next_row) return annotated def _review_classifiable_species_for_prefix(job_id: str, prefix: str) -> list[str]: parts = [part for part in str(prefix or "").split(".") if part] if parts and parts[0] not in {"Target", "Multi"}: return [parts[0]] return _declared_classifiable_species(job_id) def _filter_subfamily_review_rows(job_id: str, prefix: str, rows: list[dict]) -> list[dict]: allowed = set(_review_classifiable_species_for_prefix(job_id, prefix)) if not allowed: return rows annotated_count = sum(1 for row in rows if str(row.get("species") or "").strip()) if annotated_count == 0: return rows filtered = [row for row in rows if str(row.get("species") or "").strip() in allowed] return filtered or rows def _should_split_joint_subfamily_prefix(prefix: str) -> bool: parts = [part for part in str(prefix or "").split(".") if part] return bool(parts and parts[0] in {"Target", "Multi"}) def _split_joint_subfamily_confirmation(job_id: str, prefix: str, rows: list[dict]) -> list[dict]: if not _should_split_joint_subfamily_prefix(prefix): return [] family = _family_from_prefix_or_job(job_id, prefix) species_to_ids, id_to_species = _subfamily_species_member_maps(job_id, family) if not family or not species_to_ids: return [] final_dir = WORKSPACE_ROOT / job_id / "final" input_dir = INPUT_ROOT / job_id workspace_dir = WORKSPACE_ROOT / job_id grouped: dict[str, list[dict]] = {species: [] for species in species_to_ids} for row in _annotate_subfamily_rows_with_species(job_id, prefix, rows): gene_id = str(row.get("gene_id") or "").strip() species = str(row.get("species") or "").strip() or id_to_species.get(gene_id, "") if not species or gene_id not in species_to_ids.get(species, set()): continue grouped.setdefault(species, []).append(row) joint_source_manifest = _source_prediction_manifest_path(job_id, prefix) results: list[dict] = [] for species in sorted(grouped): species_rows = grouped.get(species) or [] if not species_rows: continue species_prefix = f"{species}.{family}" edited_file = final_dir / f"{species_prefix}.confirmed_subfamily.edited.tsv" with edited_file.open("w", encoding="utf-8") as handle: handle.write("gene_id\tspecies\tpredicted_subfamily\tconfirmed_subfamily\tconfidence\tnote\n") for row in species_rows: handle.write( f"{row['gene_id']}\t{species}\t{row.get('predicted_subfamily','')}\t{row['confirmed_subfamily']}\t" f"{row.get('confidence','')}\t{row.get('note','')}\n" ) cmd = [ "/home/xzq/miniconda3/envs/test4py311/bin/python", "apps/cli/run_subfamily_confirm.py", "--input-dir", str(input_dir), "--workspace", str(workspace_dir), "--prefix", species_prefix, "--edited-subfamily", str(edited_file), ] source_manifest = joint_source_manifest or _source_prediction_manifest_path(job_id, species_prefix) if source_manifest: cmd.extend(["--source-prediction-manifest", str(source_manifest)]) proc = subprocess.run(cmd, cwd=str(PROJECT_ROOT), capture_output=True, text=True) manifest_path = final_dir / f"{species_prefix}.subfamily_confirm.manifest.json" if proc.returncode == 0 and manifest_path.exists(): try: manifest_data = json.loads(manifest_path.read_text(encoding="utf-8")) projection = { "type": "per_species_projection_from_joint_subfamily_confirmation", "species": species, "family": family, "source_joint_prefix": prefix, "source_prediction_manifest": str(source_manifest.resolve()) if source_manifest else "", "source_rows": len(rows), "projected_rows": len(species_rows), } manifest_data["projection"] = projection manifest_data.setdefault("parameters", {})["projection"] = projection manifest_data.setdefault("metrics", {})["projected_from_joint_rows"] = len(species_rows) manifest_data.setdefault("artifacts", {})["projection_manifest"] = { "path": f"final/{manifest_path.name}", "full_path": str(manifest_path), "type": "json", } manifest_path.write_text(json.dumps(manifest_data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") except Exception as exc: print(f"[WARN] Failed to annotate subfamily projection manifest {manifest_path}: {exc}") results.append({ "species": species, "prefix": species_prefix, "confirmed_count": len(species_rows), "edited_file": str(edited_file.resolve()), "manifest": str(manifest_path.resolve()), "source_joint_prefix": prefix, "source_prediction_manifest": str(source_manifest.resolve()) if source_manifest else "", "success": proc.returncode == 0, "stdout": proc.stdout[-2000:], "stderr": proc.stderr[-2000:], }) return results def _manifest_scope_id(manifest: dict) -> str: if not isinstance(manifest, dict): return "" stage_context = manifest.get("stage_context") if isinstance(manifest.get("stage_context"), dict) else {} params = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {} return str(manifest.get("scope_id") or stage_context.get("scope_id") or params.get("scope_id") or "").strip() def _final_artifact_path(final_dir: Path, artifact_value: object) -> Path | None: raw = "" if isinstance(artifact_value, dict): raw = str(artifact_value.get("path") or artifact_value.get("full_path") or artifact_value.get("file") or "").strip() else: raw = str(artifact_value or "").strip() if not raw: return None path = Path(raw) if path.is_absolute(): return path if path.parts and path.parts[0] == "final": return final_dir.parent / path return final_dir / path def _iter_scope_matched_manifests(final_dir: Path, scope_id: str, stages: set[str]) -> list[dict]: if not scope_id: return [] matched: list[dict] = [] for path in sorted(final_dir.glob("*.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True): try: data = json.loads(path.read_text(encoding="utf-8")) except Exception: continue if stages and str(data.get("stage") or "") not in stages: continue if str(data.get("status") or "") not in {"success", "partial_success", "needs_user_confirmation"}: continue if _manifest_scope_id(data) != scope_id: continue data["_manifest_path"] = str(path) matched.append(data) return matched def _subfamily_evidence_assets(job_id: str, prefix: str) -> tuple[list[dict], list[str]]: final_dir = WORKSPACE_ROOT / job_id / "final" species = "" family = "" prefix_parts = [part for part in str(prefix or "").split(".") if part] if len(prefix_parts) >= 2: species, family = prefix_parts[0], prefix_parts[1] scoped_target = f"Target.{family}.target-{species}*" if species and family else "" groups = { "rect_combo": { "label": "矩形树 + 亚家族 + Motif/Structure/Domain", "patterns": [ f"{prefix}*reference_classification*evidence_combo*.png", f"{prefix}*combo_tree_motif_structure_domain*.png", f"{prefix}*combo*.png", f"{scoped_target}.subfamily_evidence*combo*.png" if scoped_target else "", ], }, "circular_tree": { "label": "圈图 + 亚家族分类", "patterns": [ f"{prefix}*reference_classification.tree.circular_tree*.png", f"{prefix}*subfamily*circular_tree*.png", f"{scoped_target}.subfamily_prediction.reference_classification.tree.circular_tree*.png" if scoped_target else "", f"{scoped_target}.subfamily_evidence.tree.circular_tree*.png" if scoped_target else "", ], }, "tree": { "label": "亚家族矩形树", "patterns": [ f"{prefix}*reference_classification.tree.png", f"{scoped_target}.subfamily_prediction.reference_classification.tree.png" if scoped_target else "", f"{scoped_target}.subfamily_evidence.tree.png" if scoped_target else "", ], }, "structure": { "label": "Structure", "patterns": [ f"{prefix}*structure*.png", f"{prefix}*gene_structure*.png", f"{scoped_target}.subfamily_evidence.structure*.png" if scoped_target else "", ], }, "motif": { "label": "Motif", "patterns": [ f"{prefix}*motif*.png", f"{scoped_target}.subfamily_evidence.motif*.png" if scoped_target else "", ], }, "domain": { "label": "Domain", "patterns": [ f"{prefix}*domain*.png", f"{scoped_target}.subfamily_evidence.domain*.png" if scoped_target else "", ], }, } assets: list[dict] = [] missing: list[str] = [] emitted_names: set[str] = set() def emit_asset(path: Path | None, group: str, label: str, key_hint: str = "manifest") -> None: if not path or not path.exists() or path.name in emitted_names: return suffix = path.suffix.lower() if suffix not in {".png", ".jpg", ".jpeg", ".svg", ".pdf", ".nwk"}: return ftype = "image" if suffix in {".png", ".jpg", ".jpeg", ".svg"} else "file" assets.append({ "key": f"{group}:{key_hint}:{path.name}", "group": group, "label": label, "name": path.name, "url": f"/workspace/{job_id}/final/{path.name}", "type": ftype, }) emitted_names.add(path.name) def emit_manifest_evidence(manifest_data: dict, key_hint: str) -> None: for key, artifact in (manifest_data.get("artifacts") or {}).items(): if not isinstance(artifact, dict): continue path = _final_artifact_path(final_dir, artifact) lowered = f"{key} {path.name if path else ''}".lower() if "combo" in lowered: emit_asset(path, "rect_combo", "矩形树 + 亚家族 + Motif/Structure/Domain", key_hint) elif "circular_tree" in lowered: emit_asset(path, "circular_tree", "圈图 + 亚家族分类", key_hint) elif "tree" in lowered and "reference_classification" in lowered: emit_asset(path, "tree", "亚家族矩形树", key_hint) elif path and path.suffix.lower() == ".nwk" and "tree" in lowered: emit_asset(path, "tree_file", "分物种亚家族树文件", key_hint) for idx, item in enumerate(manifest_data.get("per_species_results") or []): if not isinstance(item, dict): continue emit_asset(_final_artifact_path(final_dir, item.get("combo")), "rect_combo", "矩形树 + 亚家族 + Motif/Structure/Domain", f"{key_hint}:per_species:{idx}") emit_asset(_final_artifact_path(final_dir, item.get("circular_tree")), "circular_tree", "圈图 + 亚家族分类", f"{key_hint}:per_species:{idx}") emit_asset(_final_artifact_path(final_dir, item.get("tree")), "tree", "亚家族矩形树", f"{key_hint}:per_species:{idx}") manifest = _latest_stage_manifest(final_dir, prefix, "reference_classification") emit_manifest_evidence(manifest, "reference_manifest") scope_id = _manifest_scope_id(manifest) for scoped_manifest in _iter_scope_matched_manifests(final_dir, scope_id, {"reference_classification", "combo"}): emit_manifest_evidence(scoped_manifest, "scope_manifest") for key, spec in groups.items(): if key == "rect_combo" and any(a.get("group") == "rect_combo" for a in assets): continue if key in {"structure", "motif", "domain"} and any(a.get("group") == "rect_combo" for a in assets): continue if key == "circular_tree" and any(a.get("group") == "circular_tree" for a in assets): continue if key == "tree" and any(a.get("group") == "tree" for a in assets): continue matches: list[Path] = [] for pattern in spec["patterns"]: if not pattern: continue matches.extend(sorted(final_dir.glob(pattern))) if not matches: missing.append(key) continue for path in matches: if path.name in emitted_names: continue assets.append({ "key": f"{key}:{path.name}", "group": key, "label": spec["label"], "name": path.name, "url": f"/workspace/{job_id}/final/{path.name}", "type": "image" if path.suffix.lower() in {".png", ".jpg", ".jpeg", ".svg"} else "file", }) emitted_names.add(path.name) if sum(1 for a in assets if a.get("group") == key) >= 2: break for pattern, label, group in [ (f"{prefix}.reference_classification_report.md", "Reference report", "reference"), (f"{prefix}.confirmed_subfamily_report.md", "Confirmed report", "confirmed"), (f"{prefix}.reference_classification.confidence.tsv", "Confidence table", "reference"), ]: path = final_dir / pattern if path.exists(): assets.append({ "key": f"{group}:{path.name}", "group": group, "label": label, "name": path.name, "url": f"/workspace/{job_id}/final/{path.name}", "type": "file", }) priority = {"rect_combo": 0, "circular_tree": 1, "tree": 2, "tree_file": 3, "reference": 4, "confirmed": 5} def asset_sort_key(item: dict) -> tuple: group = str(item.get("group") or "") name = str(item.get("name") or "") lowered_name = name.lower() key = str(item.get("key") or "").lower() is_rect_combo = group == "rect_combo" # In subfamily review, prefer the per-species/reference-classification combo # that actually carries subfamily labels over legacy scoped copies. canonical_combo = is_rect_combo and lowered_name == f"{prefix.lower()}.target_plus_reference.combo.png" legacy_scoped_combo = is_rect_combo and (".target-" in lowered_name or ".ref-" in lowered_name or ".comp-" in lowered_name) image_rank = 0 if item.get("type") == "image" else 1 source_rank = 0 if "reference_manifest:per_species" in key or "reference_manifest" in key else 1 return ( priority.get(group, 99), source_rank, 0 if canonical_combo else 1, 1 if legacy_scoped_combo else 0, image_rank, lowered_name, ) assets.sort(key=asset_sort_key) return assets, missing @app.get("/api/subfamily/review") async def get_subfamily_review(job_id: str = Query(""), prefix: str = Query("")): if not job_id: return {"success": False, "message": "job_id is required", "rows": []} base_prefix = _nonempty_prefix(prefix) or resolve_effective_prefix(job_id, {"prefix": prefix}, "reference_classification") effective_prefix = _resolve_subfamily_review_prefix(job_id, base_prefix) source_path, source_type = _subfamily_source_path(job_id, effective_prefix) rows = _normalize_subfamily_rows(_read_simple_tsv(source_path)) rows = _merge_subfamily_assignment_metadata(rows, _read_subfamily_assignment_map(job_id, effective_prefix)) rows = _annotate_subfamily_rows_with_species(job_id, effective_prefix, rows) rows = _filter_subfamily_review_rows(job_id, effective_prefix, rows) assets, missing = _subfamily_evidence_assets(job_id, effective_prefix) return { "success": bool(rows), "job_id": job_id, "prefix": effective_prefix, "source": str(source_path.resolve()) if source_path.exists() else str(source_path), "source_type": source_type, "rows": rows, "row_count": len(rows), "evidence_assets": assets, "missing_evidence": missing, "message": "确认后会生成 confirmed_subfamily.tsv；后续树图、structure、motif、domain、combo、promoter 会优先使用确认后的亚家族标签。", } @app.post("/api/subfamily/review/confirm") async def confirm_subfamily_review(req: SubfamilyReviewConfirmRequest): if not req.job_id: raise HTTPException(status_code=400, detail="job_id is required") prefix = _nonempty_prefix(req.prefix) or resolve_effective_prefix(req.job_id, {"prefix": req.prefix}, "subfamily_confirm") final_dir = WORKSPACE_ROOT / req.job_id / "final" input_dir = INPUT_ROOT / req.job_id workspace_dir = WORKSPACE_ROOT / req.job_id rows = _normalize_subfamily_rows(req.rows) if not rows: source_path, _source_type = _subfamily_source_path(req.job_id, prefix) rows = _normalize_subfamily_rows(_read_simple_tsv(source_path)) rows = _annotate_subfamily_rows_with_species(req.job_id, prefix, rows) rows = _filter_subfamily_review_rows(req.job_id, prefix, rows) if not rows: raise HTTPException(status_code=400, detail="No subfamily rows to confirm") final_dir.mkdir(parents=True, exist_ok=True) edited_file = final_dir / f"{prefix}.confirmed_subfamily.edited.tsv" with edited_file.open("w", encoding="utf-8") as handle: handle.write("gene_id\tspecies\tpredicted_subfamily\tconfirmed_subfamily\tconfidence\tnote\n") for row in rows: handle.write( f"{row['gene_id']}\t{row.get('species','')}\t{row.get('predicted_subfamily','')}\t{row['confirmed_subfamily']}\t" f"{row.get('confidence','')}\t{row.get('note','')}\n" ) current_ids = ( _read_id_file(final_dir / f"{prefix}.final_members.id") or _read_id_file(final_dir / f"{prefix}.curated_members.id") ) confirmed_ids = {row["gene_id"] for row in rows} # Reference classification tables commonly include reference genes plus # target genes. This is a label-confirmation step, so a superset is not a # member-set change; missing current members are. member_set_changed = bool(current_ids and confirmed_ids and not current_ids.issubset(confirmed_ids)) cmd = [ "/home/xzq/miniconda3/envs/test4py311/bin/python", "apps/cli/run_subfamily_confirm.py", "--input-dir", str(input_dir), "--workspace", str(workspace_dir), "--prefix", prefix, "--edited-subfamily", str(edited_file), ] source_manifest = _source_prediction_manifest_path(req.job_id, prefix) if source_manifest: cmd.extend(["--source-prediction-manifest", str(source_manifest)]) proc = subprocess.run(cmd, cwd=str(PROJECT_ROOT), capture_output=True, text=True) if proc.returncode != 0: raise HTTPException(status_code=500, detail=(proc.stderr or proc.stdout or "subfamily_confirm failed")[-4000:]) split_confirmed = _split_joint_subfamily_confirmation(req.job_id, prefix, rows) failed_splits = [item for item in split_confirmed if not item.get("success")] if failed_splits: detail = "\n".join(f"{item.get('prefix')}: {item.get('stderr') or item.get('stdout')}" for item in failed_splits) raise HTTPException(status_code=500, detail=(detail or "per-species subfamily_confirm split failed")[-4000:]) manifest_path = final_dir / f"{prefix}.subfamily_confirm.manifest.json" manifest = {} if manifest_path.exists(): try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: manifest = {} stage_context = manifest.get("stage_context") if isinstance(manifest.get("stage_context"), dict) else {} scope_id = str(manifest.get("scope_id") or stage_context.get("scope_id") or "") analysis_route = "confirmed_subfamily" member_set_id = str(manifest.get("member_set_id") or stage_context.get("member_set_id") or "") if scope_id and not member_set_id: member_set_id = f"{scope_id}:confirmed_subfamily:all_members" from gfa_core.mas.tools.stage_status import get_stage_status as get_manifest_stage_status stage_report = get_manifest_stage_status( job_id=req.job_id, prefix=prefix, stage="subfamily_confirm", scope_id=scope_id, analysis_route=analysis_route, member_set_id=member_set_id, strict_identity=bool(scope_id), ) stage_status = stage_report.to_dict() identity = { "stage": "subfamily_confirm", "prefix": prefix, "scope_id": scope_id, "analysis_route": analysis_route, "member_set_id": member_set_id, } return { "success": True, "job_id": req.job_id, "prefix": prefix, "confirmed_count": len(rows), "edited_file": str(edited_file.resolve()), "manifest": str(manifest_path.resolve()), "manifest_data": manifest, "identity": identity, "stage_status": stage_status, "split_confirmed": split_confirmed, "member_set_changed": member_set_changed, "requires_phylogeny_rerun": member_set_changed, "message": "亚家族标签已确认。联合确认会同步拆分生成各目标物种的 confirmed_subfamily.tsv；成员集合变化时需要重跑 phylogeny。", "stdout": proc.stdout[-4000:], } @app.get("/api/interspecies/species-options") async def get_interspecies_species_options(job_id: str = Query("")): if not job_id: return {"success": False, "message": "job_id is required", "targets": [], "refs": [], "default_targets": [], "default_refs": []} options = _ordered_species_options(job_id) return {"success": True, "job_id": job_id, **options} @app.get("/api/gene-family/scope-options") async def get_gene_family_scope_options(job_id: str = Query(""), prefix: str = Query("")): if not job_id: return {"success": False, "message": "job_id is required"} input_dir = INPUT_ROOT / job_id workspace = WORKSPACE_ROOT / job_id if not input_dir.exists(): return {"success": False, "message": f"input directory not found: {job_id}"} try: scope_prefix = prefix if "." in str(prefix or "") else "" scope = build_gene_family_scope(input_dir=input_dir, workspace=workspace, prefix=scope_prefix) return { "success": True, "job_id": job_id, "prefix": scope.prefix, "family": scope.family, "target_species": scope.target_species, "reference_species": scope.reference_species, "primary_reference_species": getattr(scope, "primary_reference_species", scope.reference_species), "identify_blast_reference_species": getattr(scope, "identify_blast_reference_species", []), "comparative_species": scope.comparative_species, "default_scope_mode": _subfamily_scope_mode_for_roles(scope.reference_species, scope.comparative_species), "required_pfams": str(scope.analysis.get("Required_Pfams") or "").strip(), "warnings": scope.warnings, } except Exception as exc: return {"success": False, "message": str(exc)} def _read_fasta_headers_for_options(path: Path, limit: int = 5000) -> List[Dict[str, str]]: options: List[Dict[str, str]] = [] with path.open("r", encoding="utf-8", errors="ignore") as handle: for raw in handle: if not raw.startswith(">"): continue header = raw[1:].strip() if not header: continue seq_id = header.split()[0] options.append({"id": seq_id, "header": header}) if len(options) >= limit: break return options @app.get("/api/msa/sequence-options") async def msa_sequence_options( job_id: str = Query(...), prefix: str = Query(""), msa_file: str = Query(""), source_image: str = Query(""), ): try: from apps.web.pipeline_backend import _resolve_msa_file_for_region_plot workspace = WORKSPACE_ROOT / job_id if not workspace.exists(): raise HTTPException(status_code=404, detail=f"Workspace not found: {job_id}") resolved = _resolve_msa_file_for_region_plot( workspace=workspace, prefix=prefix, msa_file=msa_file, source_image=source_image, ) if not resolved or not resolved.exists(): raise HTTPException(status_code=404, detail="MSA file not found") return { "success": True, "msa_file": str(resolved), "options": _read_fasta_headers_for_options(resolved), } except HTTPException: raise except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @app.get("/api/msa/conserved-region-recommendation") async def msa_conserved_region_recommendation( job_id: str = Query(...), prefix: str = Query(""), msa_file: str = Query(""), source_image: str = Query(""), window_len: int = Query(60), min_nongap: float = Query(0.6), ): try: from apps.web.pipeline_backend import _resolve_msa_file_for_region_plot from apps.cli.run_msa_region_plot import read_fasta, validate_alignment, conservation_profile, best_window workspace = WORKSPACE_ROOT / job_id if not workspace.exists(): raise HTTPException(status_code=404, detail=f"Workspace not found: {job_id}") resolved = _resolve_msa_file_for_region_plot( workspace=workspace, prefix=prefix, msa_file=msa_file, source_image=source_image, ) if not resolved or not resolved.exists(): raise HTTPException(status_code=404, detail="MSA file not found") records = read_fasta(resolved) aln_len = validate_alignment(records) scores, nongaps = conservation_profile(records) start_col, end_col, mean_score, mean_nongap = best_window( scores=scores, nongaps=nongaps, window_len=window_len, min_nongap=min_nongap, ) return { "success": True, "msa_file": str(resolved), "alignment_length": aln_len, "sequence_count": len(records), "start_col": start_col, "end_col": end_col, "window_len": end_col - start_col + 1, "requested_window_len": window_len, "min_nongap": min_nongap, "mean_score": mean_score, "mean_nongap": mean_nongap, } except HTTPException: raise except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @app.post("/api/execute") async def execute_stage(req: ExecuteRequest): """执行分析阶段 - 支持异步模式""" print(f"[API DEBUG] ============================================") print(f"[API DEBUG] /api/execute called") print(f"[API DEBUG] stage={req.stage}, job_id={req.job_id}") print(f"[API DEBUG] prefix={req.prefix}, cpu={req.cpu}") print(f"[API DEBUG] rnaseqMode={getattr(req, 'rnaseqMode', 'NOT_SET')}, speciesName={getattr(req, 'speciesName', 'NOT_SET')}") print(f"[API DEBUG] kaksNeutralRate={getattr(req, 'kaksNeutralRate', 'NOT_SET')}") print(f"[API DEBUG] ============================================") global current_job_id # 如果请求中指定了 job_id，使用它；否则使用 current_job_id if req.job_id: job_id = req.job_id elif not current_job_id: current_job_id = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:18] job_id = current_job_id else: job_id = current_job_id # ===== 检查是否已完成：以统一 manifest/artifacts 校验为准 ===== effective_prefix = resolve_effective_prefix(job_id, {"prefix": req.prefix, "speciesCode": getattr(req, "speciesCode", ""), "rnaseqPrefix": getattr(req, "rnaseqPrefix", ""), "rnaseqOutputPrefix": getattr(req, "rnaseqOutputPrefix", "")}, req.stage) wgs_prefix_error = "" try: effective_prefix = _resolve_wgs_execute_prefix( job_id=job_id, stage=req.stage, prefix=effective_prefix, params={"wgsMethod": getattr(req, "wgsMethod", "gatk")}, ) except ValueError as exc: wgs_prefix_error = str(exc) if effective_prefix != _nonempty_prefix(req.prefix): print(f"[API DEBUG] Resolved request prefix for {req.stage}: {effective_prefix or ''}") early_gwas_preflight = None if req.stage in {"gwas_fmt", "gwas_trait", "gwas_cov", "gwas_mlm", "gwas_plot", "gwas_emmax", "gwas_emplot", "gwas_glm"}: early_gwas_params = { "gwasVcf": getattr(req, "gwasVcf", ""), "tasselPipeline": getattr(req, "tasselPipeline", "run_pipeline.pl"), "tasselXmx": getattr(req, "tasselXmx", "10g"), "phenotypeFile": getattr(req, "phenotypeFile", ""), "phenotypeSampleColumn": getattr(req, "phenotypeSampleColumn", ""), "phenotypeTraitColumn": getattr(req, "phenotypeTraitColumn", ""), "phenotypeTraitName": getattr(req, "phenotypeTraitName", ""), "gwasQFile": getattr(req, "gwasQFile", ""), "gwasFamFile": getattr(req, "gwasFamFile", ""), "gwasQK": getattr(req, "gwasQK", 2), "genotypeScopeId": getattr(req, "genotypeScopeId", ""), "phenotypeScopeId": getattr(req, "phenotypeScopeId", ""), "covariateScopeId": getattr(req, "covariateScopeId", ""), "sourceModelScopeId": getattr(req, "sourceModelScopeId", ""), "gwasMlmPvalues": getattr(req, "gwasMlmPvalues", ""), "gwasCmlmPvalues": getattr(req, "gwasCmlmPvalues", ""), "gwasMlmTestedHypotheses": getattr(req, "gwasMlmTestedHypotheses", None), "gwasCmlmTestedHypotheses": getattr(req, "gwasCmlmTestedHypotheses", None), "gwasRscript": getattr(req, "gwasRscript", "") or str(ASAW_RSCRIPT), "gwasPlink": getattr(req, "gwasPlink", "") or str(ASAW_BIN / "plink"), "gwasEmmax": getattr(req, "gwasEmmax", "") or GWAS_EMMAX_BIN, "gwasRequestedPcs": getattr(req, "gwasRequestedPcs", 3), "gwasEmmaxPvalues": getattr(req, "gwasEmmaxPvalues", ""), "gwasEmmaxManifest": getattr(req, "gwasEmmaxManifest", ""), "gwasEmmaxTestedHypotheses": getattr(req, "gwasEmmaxTestedHypotheses", None), } early_gwas_preflight = _api_semantic_preflight(job_id, req.stage, effective_prefix, early_gwas_params) for key, value in (early_gwas_preflight.get("materialized_params") or {}).items(): if hasattr(req, key): setattr(req, key, value) early_gwas_decision = (early_gwas_preflight.get("decisions") or {}).get("external_vcf") or {} if early_gwas_decision.get("resolved") and early_gwas_decision.get("path"): req.gwasVcf = early_gwas_decision["path"] early_tassel_decision = (early_gwas_preflight.get("decisions") or {}).get("tassel") or {} if early_tassel_decision.get("executable") and early_tassel_decision.get("path"): req.tasselPipeline = early_tassel_decision["path"] scope_params = { "scopeFile": getattr(req, "scopeFile", ""), "scopeMode": getattr(req, "scopeMode", ""), "geneFamily": getattr(req, "geneFamily", ""), "targetSpecies": getattr(req, "targetSpecies", []) or [], "referenceSpecies": getattr(req, "referenceSpecies", []) or [], "identifyBlastReferenceSpecies": getattr(req, "identifyBlastReferenceSpecies", []) or [], "primaryReferenceSpecies": getattr(req, "primaryReferenceSpecies", []) or [], "refSpeciesList": getattr(req, "refSpeciesList", []) or [], "comparativeSpecies": getattr(req, "comparativeSpecies", []) or [], } technical_reference_species: list[str] = [] technical_reference_species, ppi_owned_ortholog, needs_reference_fallback = ( _prepare_execute_technical_reference_scope(req, scope_params) ) if needs_reference_fallback: try: fallback_plan_params = { "prefix": effective_prefix, "geneFamily": scope_params.get("geneFamily", ""), "targetSpecies": scope_params.get("targetSpecies", []), "referenceSpecies": [], "refSpeciesList": [], "comparativeSpecies": [], "scopeMode": "target_plus_reference", } fallback_intent = {"params": dict(fallback_plan_params)} fallback_details = [{"id": req.stage, "params": {}}] _apply_kaks_scope_defaults( job_id, effective_prefix, fallback_plan_params, fallback_intent, fallback_details, ) for key in ("scopeFile", "scopeMode", "geneFamily", "targetSpecies", "referenceSpecies", "refSpeciesList", "comparativeSpecies"): if fallback_plan_params.get(key) not in (None, "", []): scope_params[key] = fallback_plan_params[key] print(f"[API DEBUG] Recovered {req.stage} target-reference scope from job defaults: targets={scope_params.get('targetSpecies')}, refs={scope_params.get('referenceSpecies')}") except Exception as exc: print(f"[API DEBUG] Failed to recover {req.stage} target-reference scope defaults: {exc}") gene_family_identity_stages = { "identify", "phylogeny", "reference_classification", "subfamily_confirm", "motif", "domain", "structure", "combo", "promoter", "tfbs", "synteny", "interspecies_synteny", "karyotype", "pep", "ortholog", "ppi", "kaks", "expression", } if req.stage in gene_family_identity_stages: try: technical_ref_lookup = {str(item or "").strip().lower() for item in technical_reference_species if str(item or "").strip()} for species_key, role_label in ( ("targetSpecies", "targetSpecies"), ("referenceSpecies", "referenceSpecies"), ("comparativeSpecies", "comparativeSpecies"), ("refSpeciesList", "refSpeciesList"), ): values = scope_params.get(species_key) if values in (None, "", []): continue raw_values = _param_list(values) if req.stage in {"ortholog", "ppi", "kaks"} and species_key in {"referenceSpecies", "refSpeciesList"} and technical_ref_lookup: bundle_values: list[str] = [] canonical_input: list[str] = [] require_cds = req.stage == "kaks" for raw_value in raw_values: raw_code = str(raw_value or "").strip() if raw_code and raw_code.lower() in technical_ref_lookup: available, reason = _reference_bundle_species_available(job_id, raw_code, require_cds=require_cds) if not available: raise ValueError( f"PPI/KaKs 技术参考物种 {raw_code} 已选择但参考包不完整：{reason}；" "请重新下载或上传对应 PEP/CDS 后再执行。" ) if raw_code not in bundle_values: bundle_values.append(raw_code) else: canonical_input.append(raw_code) canonical_values = _canonicalize_species_codes_for_job_strict( job_id, canonical_input, role_label, ) if canonical_input else [] scope_params[species_key] = list(dict.fromkeys([*canonical_values, *bundle_values])) continue canonical_values = _canonicalize_species_codes_for_job_strict( job_id, raw_values, role_label, ) scope_params[species_key] = canonical_values if scope_params.get("referenceSpecies") and not scope_params.get("refSpeciesList"): scope_params["refSpeciesList"] = list(scope_params.get("referenceSpecies") or []) print( f"[API DEBUG] Canonical species identity for {req.stage}: " f"targets={scope_params.get('targetSpecies')}, refs={scope_params.get('referenceSpecies')}, " f"comps={scope_params.get('comparativeSpecies')}" ) except ValueError as exc: print(f"[API DEBUG] Species identity blocked for {req.stage}: {exc}") raise HTTPException(status_code=400, detail=str(exc)) _require_gene_family_target_species( req.stage, scope_params.get("targetSpecies") or [], ) scope_id_for_status = "" try: validated_owner_scope_id = str(scope_params.get("_validatedPpiOwnerScopeId") or "").strip() if ppi_owned_ortholog and validated_owner_scope_id: generated_scope_file = str(scope_params.get("scopeFile") or "").strip() generated_scope_id = validated_owner_scope_id else: generated_scope_file, generated_scope_id = _ensure_gene_family_scope_file( job_id=job_id, prefix=effective_prefix, stage=req.stage, params=scope_params, ) if generated_scope_file: scope_params["scopeFile"] = generated_scope_file if scope_params.get("prefix"): effective_prefix = str(scope_params["prefix"]) scope_id_for_status = generated_scope_id print(f"[API DEBUG] Scope prepared for {req.stage}: {generated_scope_file}") except ValueError as exc: print(f"[API DEBUG] Scope preparation blocked for {req.stage}: {exc}") raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: print(f"[API DEBUG] Scope preparation skipped/failed for {req.stage}: {exc}") stage_context = {} scope_data = {} try: if scope_params.get("scopeFile"): scope_data = read_scope(scope_params["scopeFile"]) request_route = str(getattr(req, "analysisRoute", "") or "").strip() request_member = str(getattr(req, "memberSetId", "") or "").strip() context = build_stage_context( stage=req.stage, prefix=effective_prefix, scope_data=scope_data, scope_file=scope_params.get("scopeFile", ""), analysis_route=request_route, member_set_id=request_member, ) stage_context = context.to_dict() ref_species_for_route = scope_params.get("referenceSpecies") or scope_params.get("refSpeciesList") or [] if req.stage == "interspecies_synteny" and ref_species_for_route: request_route = "multi_species" request_member = f"{stage_context.get('scope_id')}:multi_species:all_members" if stage_context.get("scope_id") else request_member confirmed_required = bool(getattr(req, "requiresConfirmedSubfamily", False)) confirmed_scope_id = str(stage_context.get("scope_id") or scope_params.get("scopeId") or scope_params.get("scope_id") or "").strip() is_subfamily_evidence_request = ( request_route == "subfamily_evidence" and req.stage in {"phylogeny", "motif", "domain", "structure", "combo"} ) is_subfamily_prediction_request = request_route == "subfamily_prediction" or req.stage in {"reference_classification", "subfamily_confirm"} if confirmed_required and confirmed_scope_id and not is_subfamily_evidence_request and not is_subfamily_prediction_request: request_member = f"{confirmed_scope_id}:confirmed_subfamily:all_members" req.memberSetId = request_member if not getattr(req, "subfamilyLabelSource", ""): req.subfamilyLabelSource = "confirmed_or_uploaded_subfamily" if request_route: stage_context["analysis_route"] = request_route if request_member: stage_context["member_set_id"] = request_member print(f"[API DEBUG] StageContext prepared: {stage_context}") except Exception as exc: print(f"[API DEBUG] StageContext preparation skipped/failed for {req.stage}: {exc}") trusted_genotype_scope = _trusted_genotype_scope_id( req.stage, { "gwasVcf": getattr(req, "gwasVcf", ""), "genotypeScopeId": getattr(req, "genotypeScopeId", ""), }, ) if req.stage == "gwas_fmt": # The selected VCF content is the identity authority. A client-provided # genotypeScopeId is display/cache data only and must not select results. scope_params["genotypeScopeId"] = trusted_genotype_scope stage_context["genotype_scope_id"] = trusted_genotype_scope stage_identity = None try: from gfa_core.mas.stage_catalog import is_utility_stage, is_repair_stage from gfa_core.mas.tools.stage_identity import ( explain_status_mismatch, get_stage_status_for_identity, resolve_stage_identity, ) identity_params = { **scope_params, "prefix": effective_prefix, "analysisRoute": stage_context.get("analysis_route") or getattr(req, "analysisRoute", ""), "memberSetId": stage_context.get("member_set_id") or getattr(req, "memberSetId", ""), "genotypeScopeId": trusted_genotype_scope, "phenotypeScopeId": getattr(req, "phenotypeScopeId", ""), "covariateScopeId": getattr(req, "covariateScopeId", ""), "modelScopeId": getattr(req, "modelScopeId", ""), "sourceModelScopeId": getattr(req, "sourceModelScopeId", ""), "analysisScopeId": getattr(req, "analysisScopeId", ""), } stage_identity = resolve_stage_identity( stage=req.stage, prefix=effective_prefix, params=identity_params, job_id=job_id, ) scope_id_for_status = stage_identity.scope_id if req.stage not in {"genome_prepare", "genome_prepare_v2"} and req.stage not in RNA_SEQ_STAGES: stage_context = build_stage_context( stage=req.stage, prefix=effective_prefix, scope_data=scope_data, scope_file=scope_params.get("scopeFile", ""), analysis_route=stage_identity.analysis_route, member_set_id=stage_identity.member_set_id, ).to_dict() stage_context["result_group"] = stage_identity.result_group stage_context.update({ "scope_id": stage_identity.scope_id, "scope_mode": stage_identity.scope_mode, "analysis_route": stage_identity.analysis_route, "member_set_id": stage_identity.member_set_id, "result_group": stage_identity.result_group, }) if req.stage == "gwas_fmt": stage_context["genotype_scope_id"] = trusted_genotype_scope if req.stage in {"genome_prepare", "genome_prepare_v2"} or req.stage in RNA_SEQ_STAGES: stage_context.pop("scope_id", None) stage_context.pop("scope_mode", None) stage_context.pop("analysis_route", None) stage_context.pop("member_set_id", None) stage_status = get_stage_status_for_identity( job_id=job_id, identity=stage_identity, strict_identity=True, ) print(f"[API DEBUG] Stage identity: {stage_identity.to_dict()}") print(f"[API DEBUG] Stage status from identity validator: {stage_status.to_dict()}") if not stage_status.completed and stage_status.errors: print("[API DEBUG] Stage identity mismatch detail:\n" + explain_status_mismatch(stage_identity, stage_status)) analysis_route_for_gate = str(stage_context.get("analysis_route") or getattr(req, "analysisRoute", "") or getattr(req, "analysis_route", "") or "").strip() is_subfamily_evidence_stage = ( analysis_route_for_gate == "subfamily_evidence" and req.stage in {"phylogeny", "motif", "domain", "structure", "combo"} ) if bool(getattr(req, "requiresConfirmedSubfamily", False)) and not is_subfamily_evidence_stage: pre_completed_context_gate = _validate_gene_family_stage_context( job_id=job_id, stage=req.stage, family=scope_params.get("geneFamily") or (effective_prefix.split(".")[-1] if "." in effective_prefix else ""), target_species=scope_params.get("targetSpecies", []) or [], comparative_species=scope_params.get("comparativeSpecies", []) or [], requires_confirmed_subfamily=True, ) if not pre_completed_context_gate.get("valid", True): print(f"[API DEBUG] Context gate failed before completed-skip: {pre_completed_context_gate}") return { "success": False, "async": False, "status": "preflight_failed", "message": "执行前上下文检查失败：缺少确认或上传的亚家族标签", "job_id": job_id, "stage": req.stage, "context_gate": pre_completed_context_gate, "semantic_preflight": { "valid": False, "issues": pre_completed_context_gate.get("issues", []), "context_gate": pre_completed_context_gate, }, "suggested_stages": ["reference_classification", "subfamily_confirm"], } explicit_interspecies_selection = ( req.stage == "interspecies_synteny" and (bool(getattr(req, "targetSpecies", []) or []) or bool(getattr(req, "refSpeciesList", []) or [])) ) explicit_reference_classification_selection = False if req.stage == "reference_classification" and (bool(getattr(req, "targetSpecies", []) or []) or bool(getattr(req, "comparativeSpecies", []) or [])): analysis_mode_for_policy = str(getattr(req, "analysisMode", "") or scope_params.get("analysisMode") or "").strip() explicit_reference_classification_selection = analysis_mode_for_policy == "per_species" if analysis_mode_for_policy == "per_species": try: final_dir_for_policy = WORKSPACE_ROOT / job_id / "final" family_for_policy = scope_params.get("geneFamily") or ( effective_prefix.split(".")[-1] if "." in effective_prefix else "" ) target_list_for_policy = [str(x) for x in (scope_params.get("targetSpecies") or []) if str(x).strip()] comparative_list_for_policy = [str(x) for x in (scope_params.get("comparativeSpecies") or []) if str(x).strip()] classifiable_list_for_policy = [] for item in [*target_list_for_policy, *comparative_list_for_policy]: if item and item not in classifiable_list_for_policy: classifiable_list_for_policy.append(item) policy_ready = bool(classifiable_list_for_policy) for species in classifiable_list_for_policy: for filename in ( f"{species}.{family_for_policy}.target_plus_reference.reference_classification.tree.png", f"{species}.{family_for_policy}.target_plus_reference.reference_classification.tree.circular_tree.png", f"{species}.{family_for_policy}.target_plus_reference.combo.png", f"{species}.{family_for_policy}.target_plus_reference.reference_classification.manifest.json", ): path = final_dir_for_policy / filename if not path.exists() or path.stat().st_size == 0: policy_ready = False break if not policy_ready: break if policy_ready and stage_status.manifest_path: try: manifest_data = json.loads(Path(stage_status.manifest_path).read_text(encoding="utf-8")) except Exception: manifest_data = {} policy_ready = ( str(manifest_data.get("analysis_mode") or "") == "per_species" or str(manifest_data.get("policy") or "").startswith("per_species") or int((manifest_data.get("metrics") or {}).get("per_species_count") or 0) >= len(classifiable_list_for_policy) ) explicit_reference_classification_selection = not policy_ready except Exception as exc: print(f"[API DEBUG] Reference classification per_species policy-ready check failed: {exc}") explicit_reference_classification_selection = True requested_force_rerun_stages = set(str(x) for x in (getattr(req, "forceRerunStages", []) or []) if str(x).strip()) force_rerun_requested = bool(getattr(req, "forceRerun", False)) or req.stage in requested_force_rerun_stages gwas_mlm_full_identity = all(( str(getattr(req, "genotypeScopeId", "") or "").strip(), str(getattr(req, "phenotypeScopeId", "") or "").strip(), str(getattr(req, "covariateScopeId", "") or "").strip(), str(getattr(req, "modelScopeId", "") or "").strip(), str(getattr(req, "analysisScopeId", "") or "").strip(), )) gwas_plot_full_identity = all(( str(getattr(req, "genotypeScopeId", "") or "").strip(), str(getattr(req, "phenotypeScopeId", "") or "").strip(), str(getattr(req, "covariateScopeId", "") or "").strip(), str(getattr(req, "sourceModelScopeId", "") or "").strip(), str(getattr(req, "analysisScopeId", "") or "").strip(), )) gwas_emmax_full_identity = all(( str(getattr(req, "genotypeScopeId", "") or "").strip(), str(getattr(req, "phenotypeScopeId", "") or "").strip(), str(getattr(req, "modelScopeId", "") or "").strip(), str(getattr(req, "analysisScopeId", "") or "").strip(), )) gwas_emplot_full_identity = all(( str(getattr(req, "genotypeScopeId", "") or "").strip(), str(getattr(req, "phenotypeScopeId", "") or "").strip(), str(getattr(req, "sourceModelScopeId", "") or "").strip(), str(getattr(req, "analysisScopeId", "") or "").strip(), )) gwas_glm_full_identity = all(( str(getattr(req, "genotypeScopeId", "") or "").strip(), str(getattr(req, "phenotypeScopeId", "") or "").strip(), str(getattr(req, "covariateScopeId", "") or "").strip(), str(getattr(req, "modelScopeId", "") or "").strip(), str(getattr(req, "analysisScopeId", "") or "").strip(), )) skip_completed_check = ( is_utility_stage(req.stage) or is_repair_stage(req.stage) or force_rerun_requested or (req.stage == "gwas_fmt" and not trusted_genotype_scope) or (req.stage == "gwas_mlm" and not gwas_mlm_full_identity) or (req.stage == "gwas_plot" and not gwas_plot_full_identity) or (req.stage == "gwas_emmax" and not gwas_emmax_full_identity) or (req.stage == "gwas_emplot" and not gwas_emplot_full_identity) or (req.stage == "gwas_glm" and not gwas_glm_full_identity) ) if force_rerun_requested and stage_status.completed: print(f"[API DEBUG] Stage '{req.stage}' has completed manifest but forceRerun=true; rerunning") if ( stage_status.completed and not explicit_interspecies_selection and not explicit_reference_classification_selection and not skip_completed_check ): print(f"[API DEBUG] Stage '{req.stage}' already completed, skipping execution") return { "success": True, "async": False, "message": f"{req.stage} 已有同一分析范围的有效结果，无需重复执行。", "job_id": job_id, "already_completed": True, "stage_status": stage_status.to_dict() } elif stage_status.completed and explicit_interspecies_selection: print("[API DEBUG] Interspecies selection provided; rerunning instead of skipping completed manifest") elif stage_status.completed and explicit_reference_classification_selection: print("[API DEBUG] Reference classification selection requires per-target policy refresh; rerunning") except Exception as e: print(f"[API DEBUG] Error checking completed stages: {e}") # 继续执行，不阻断 # ===== RNA-seq GO 背景预检：自动注释属于长任务，提交前需要用户确认 ===== try: auto_annotation_preflight = _rnaseq_auto_annotation_preflight(job_id, req) if auto_annotation_preflight: return { "success": False, "async": False, "status": "needs_auto_annotation_confirmation", "message": auto_annotation_preflight.get("message") or "需要确认自动注释", "job_id": job_id, "stage": req.stage, "auto_annotation_confirmation": auto_annotation_preflight, } except Exception as exc: print(f"[API DEBUG] RNA-seq auto-annotation preflight failed: {exc}") # ===== 执行能力检查：避免规划/接口提交尚未接入执行器的阶段 ===== try: from gfa_core.mas.stage_catalog import is_stage_executable if not is_stage_executable(req.stage): message = ( f"阶段 {req.stage} 暂未接入标准执行器，" "无法直接执行。请先补充 pipeline_backend.run_stage 分支和标准 manifest。" ) print(f"[API DEBUG] Unsupported stage rejected: {req.stage}") return { "success": False, "async": False, "status": "unsupported", "message": message, "job_id": job_id, "stage": req.stage, } except Exception as e: print(f"[API DEBUG] Error checking stage executability: {e}") # ===== 统一架构：所有stage都使用Celery + LangGraph ===== task_id = str(uuid.uuid4()) # 创建任务记录 tasks[task_id] = { "id": task_id, "stage": req.stage, "job_id": job_id, "prefix": effective_prefix, "session_id": req.session_id or job_id, "status": "pending", "message": f"任务 {req.stage} 已提交", "created_at": datetime.now().isoformat(), "scope_id": scope_id_for_status, "scopeFile": scope_params.get("scopeFile", ""), "stage_identity": stage_identity.to_dict() if stage_identity is not None else {}, "stage_context": stage_context, "force_rerun_policy": ( req.stage == "reference_classification" and (bool(getattr(req, "targetSpecies", []) or []) or bool(getattr(req, "comparativeSpecies", []) or [])) and 'explicit_reference_classification_selection' in locals() and explicit_reference_classification_selection ), "result": None } # 构建参数 params = { "prefix": effective_prefix, "cpu": req.cpu, "populationVcf": getattr(req, "populationVcf", ""), "populationSampleGroup": getattr(req, "populationSampleGroup", ""), "popPlink": getattr(req, "popPlink", "") or str(ASAW_BIN / "plink"), "popTassel": getattr(req, "popTassel", "") or TASSEL_PIPELINE, "popFasttree": getattr(req, "popFasttree", "") or str(ASAW_BIN / "FastTree"), "popAdmixture": getattr(req, "popAdmixture", "") or str(ASAW_BIN / "admixture"), "popBcftools": getattr(req, "popBcftools", "scripts/pysam_bcftools.py"), "popLddecay": getattr(req, "popLddecay", "") or str(ASAW_BIN / "PopLDdecay"), "popPlotOne": getattr(req, "popPlotOne", "scripts/plot_pop_ld_decay.py"), "popPlotMulti": getattr(req, "popPlotMulti", "scripts/plot_pop_ld_decay.py"), "popRscript": getattr(req, "popRscript", "") or str(ASAW_RSCRIPT), "popKMin": getattr(req, "popKMin", 2), "popKMax": getattr(req, "popKMax", 10), "popCvFolds": getattr(req, "popCvFolds", 5), "popRandomSeed": getattr(req, "popRandomSeed", 20260723), "popGwasK": getattr(req, "popGwasK", 0), "popChromosomePolicy": getattr(req, "popChromosomePolicy", "all_vcf_contigs"), "popChromosomes": getattr(req, "popChromosomes", ""), "gwasVcf": getattr(req, "gwasVcf", ""), "tasselPipeline": getattr(req, "tasselPipeline", "run_pipeline.pl"), "tasselXmx": getattr(req, "tasselXmx", "10g"), "gwasUnsupportedMarkerPolicy": getattr(req, "gwasUnsupportedMarkerPolicy", "mask"), "genotypeScopeId": trusted_genotype_scope if req.stage == "gwas_fmt" else getattr(req, "genotypeScopeId", ""), "phenotypeFile": getattr(req, "phenotypeFile", ""), "phenotypeSampleColumn": getattr(req, "phenotypeSampleColumn", ""), "phenotypeTraitColumn": getattr(req, "phenotypeTraitColumn", ""), "phenotypeTraitName": getattr(req, "phenotypeTraitName", ""), "gwasQFile": getattr(req, "gwasQFile", ""), "gwasFamFile": getattr(req, "gwasFamFile", ""), "gwasQK": getattr(req, "gwasQK", 2), "gwasSortedVcf": getattr(req, "gwasSortedVcf", ""), "gwasKinshipFile": getattr(req, "gwasKinshipFile", ""), "gwasTraitFile": getattr(req, "gwasTraitFile", ""), "gwasCovariateFile": getattr(req, "gwasCovariateFile", ""), "phenotypeScopeId": getattr(req, "phenotypeScopeId", ""), "covariateScopeId": getattr(req, "covariateScopeId", ""), "modelScopeId": getattr(req, "modelScopeId", ""), "sourceModelScopeId": getattr(req, "sourceModelScopeId", ""), "analysisScopeId": getattr(req, "analysisScopeId", ""), "gwasMlmPvalues": getattr(req, "gwasMlmPvalues", ""), "gwasCmlmPvalues": getattr(req, "gwasCmlmPvalues", ""), "gwasMlmTestedHypotheses": getattr(req, "gwasMlmTestedHypotheses", None), "gwasCmlmTestedHypotheses": getattr(req, "gwasCmlmTestedHypotheses", None), "gwasRscript": getattr(req, "gwasRscript", "") or str(ASAW_RSCRIPT), "gwasPlink": getattr(req, "gwasPlink", "") or str(ASAW_BIN / "plink"), "gwasEmmax": getattr(req, "gwasEmmax", "") or GWAS_EMMAX_BIN, "gwasRequestedPcs": getattr(req, "gwasRequestedPcs", 3), "gwasEmmaxPvalues": getattr(req, "gwasEmmaxPvalues", ""), "gwasEmmaxManifest": getattr(req, "gwasEmmaxManifest", ""), "gwasEmmaxTestedHypotheses": getattr(req, "gwasEmmaxTestedHypotheses", None), "requiredPfams": req.requiredPfams, "refSpecies": req.refSpecies, "idEvalue": req.idEvalue, "idCoverage": req.idCoverage, "blastEvalue": req.blastEvalue, "blastIdentity": req.blastIdentity, "mafftParams": req.mafftParams, "iqtreeParams": req.iqtreeParams, "treeLayout": req.treeLayout, "msaStartCol": getattr(req, "msaStartCol", 0), "msaEndCol": getattr(req, "msaEndCol", 0), "msaAutoConserved": getattr(req, "msaAutoConserved", False), "msaWindowLen": getattr(req, "msaWindowLen", 60), "msaMinNongap": getattr(req, "msaMinNongap", 0.6), "msaFile": getattr(req, "msaFile", ""), "msaSourceImage": getattr(req, "msaSourceImage", ""), "msaReferenceId": getattr(req, "msaReferenceId", ""), "msaReferenceStartResidue": getattr(req, "msaReferenceStartResidue", 0), "msaReferenceEndResidue": getattr(req, "msaReferenceEndResidue", 0), "treeFile": getattr(req, "treeFile", ""), "treeSourceImage": getattr(req, "treeSourceImage", ""), "treeSourceStage": getattr(req, "treeSourceStage", ""), "treeOutputLabel": getattr(req, "treeOutputLabel", ""), "treeLabelOffset": getattr(req, "treeLabelOffset", 0.4), "treeTipLabelSize": getattr(req, "treeTipLabelSize", 3.0), "treeBranchColor": getattr(req, "treeBranchColor", "black"), "treeBranchWidth": getattr(req, "treeBranchWidth", 0.5), "treeShowBootstrap": getattr(req, "treeShowBootstrap", True), "treeBootstrapSize": getattr(req, "treeBootstrapSize", 2.0), "treeTipPointSize": getattr(req, "treeTipPointSize", 2.0), "treeSubfamilyArcEnabled": getattr(req, "treeSubfamilyArcEnabled", True), "treeSubfamilyArcGap": getattr(req, "treeSubfamilyArcGap", 1.2), "treeSubfamilyArcLabelGap": getattr(req, "treeSubfamilyArcLabelGap", 0.6), "treeSubfamilyArcLabelSize": getattr(req, "treeSubfamilyArcLabelSize", 0.0), "treeTargetShape": getattr(req, "treeTargetShape", "triangle"), "treeReferenceShape": getattr(req, "treeReferenceShape", "circle"), "treeMarkerMode": getattr(req, "treeMarkerMode", "auto"), "syntenySourceImage": getattr(req, "syntenySourceImage", ""), "syntenySourceStage": getattr(req, "syntenySourceStage", "synteny"), "syntenyOutputLabel": getattr(req, "syntenyOutputLabel", "synteny_replot_circos"), "syntenyGeneLabelSize": getattr(req, "syntenyGeneLabelSize", 38.0), "syntenyGeneTrackR0": getattr(req, "syntenyGeneTrackR0", 1.03), "syntenyGeneTrackR1": getattr(req, "syntenyGeneTrackR1", 1.30), "syntenyChromLabelRadius": getattr(req, "syntenyChromLabelRadius", "1r - 60p"), "syntenyChromLabelSize": getattr(req, "syntenyChromLabelSize", 40.0), "syntenyIdeogramThickness": getattr(req, "syntenyIdeogramThickness", 90.0), "promoterElements": getattr(req, "promoterElements", ""), "promoterCategories": getattr(req, "promoterCategories", ""), "promoterMatchMode": getattr(req, "promoterMatchMode", "exact"), "promoterPlotModes": getattr(req, "promoterPlotModes", "position_distribution,heatmap"), "promoterColorBy": getattr(req, "promoterColorBy", "auto"), "promoterTopN": getattr(req, "promoterTopN", 30), "promoterOutputLabel": getattr(req, "promoterOutputLabel", ""), "promoterFile": getattr(req, "promoterFile", ""), "promoterTreeFile": getattr(req, "promoterTreeFile", ""), "promoterSubfamilyFile": getattr(req, "promoterSubfamilyFile", ""), "tfbsUpstreamLength": getattr(req, "tfbsUpstreamLength", 2000), "tfbsMotifDbMode": getattr(req, "tfbsMotifDbMode", "auto"), "tfbsFimoPvalue": getattr(req, "tfbsFimoPvalue", 0.0001), "tfbsFimoQvalue": getattr(req, "tfbsFimoQvalue", 0.05), "tfbsBackgroundMode": getattr(req, "tfbsBackgroundMode", "none"), "tfbsMaxNearestRank": getattr(req, "tfbsMaxNearestRank", "order"), "tfbsTopMotifs": getattr(req, "tfbsTopMotifs", 30), "tfbsScanBothStrands": getattr(req, "tfbsScanBothStrands", True), "comboPanels": list(getattr(req, "comboPanels", []) or []), "comboOutputLabel": getattr(req, "comboOutputLabel", ""), "referenceSubfamilyFile": getattr(req, "referenceSubfamilyFile", ""), "subfamilyConfirmFile": getattr(req, "subfamilyConfirmFile", ""), "referenceTargetPrefix": getattr(req, "referenceTargetPrefix", ""), "referencePrefix": getattr(req, "referencePrefix", "At"), "referenceOverwriteGeneSubfamily": getattr(req, "referenceOverwriteGeneSubfamily", False), "nmotifs": req.nmotifs, "minw": req.minw, "maxw": req.maxw, "domEvalue": req.domEvalue, "domCoverage": req.domCoverage, "idTransform": req.idTransform, "scopeFile": scope_params.get("scopeFile", ""), "scopeMode": scope_params.get("scopeMode", ""), "analysisRoute": stage_context.get("analysis_route") or getattr(req, "analysisRoute", ""), "memberSetId": stage_context.get("member_set_id") or getattr(req, "memberSetId", ""), "contextRoute": getattr(req, "contextRoute", ""), "subfamilyLabelSource": getattr(req, "subfamilyLabelSource", ""), "requiresConfirmedSubfamily": bool(getattr(req, "requiresConfirmedSubfamily", False)), "forceRerun": bool(getattr(req, "forceRerun", False)), "forceRerunStages": list(getattr(req, "forceRerunStages", []) or []), "analysisMode": getattr(req, "analysisMode", ""), "speciesQueue": list(getattr(req, "speciesQueue", []) or []), "confirmPrefixBySpecies": dict(getattr(req, "confirmPrefixBySpecies", {}) or {}), "stageContext": stage_context, "geneFamily": scope_params.get("geneFamily", ""), "geoId": getattr(req, 'geoId', ''), # RNA-seq参数 "rnaseqMode": getattr(req, 'rnaseqMode', 'upload'), "speciesName": getattr(req, 'speciesName', ''), "speciesCode": getattr(req, 'speciesCode', ''), "rnaseqPrefix": getattr(req, 'rnaseqPrefix', ''), "rnaseqOutputPrefix": getattr(req, 'rnaseqOutputPrefix', ''), # RNA-seq Enrich参数 "topPathways": getattr(req, 'topPathways', 20), "enrichBackgroundSource": getattr(req, 'enrichBackgroundSource', 'auto'), "enrichSpecies": getattr(req, 'enrichSpecies', ''), "enrichKeyType": getattr(req, 'enrichKeyType', 'auto'), "enrichType": getattr(req, 'enrichType', 'go'), "keggOrganism": getattr(req, 'keggOrganism', ''), "sampleParallelJobs": getattr(req, 'sampleParallelJobs', 0), "threadsPerSample": getattr(req, 'threadsPerSample', 0), "rnaseqQcTool": getattr(req, 'rnaseqQcTool', 'fastp'), "rnaseqFastpBin": getattr(req, 'rnaseqFastpBin', 'tools/fastp'), "rnaseqFastpDetectAdapterForPe": bool(getattr(req, 'rnaseqFastpDetectAdapterForPe', True)), "rnaseqFastpCutFront": bool(getattr(req, 'rnaseqFastpCutFront', True)), "rnaseqFastpCutTail": bool(getattr(req, 'rnaseqFastpCutTail', True)), "rnaseqFastpCutWindowSize": getattr(req, 'rnaseqFastpCutWindowSize', 4), "rnaseqFastpCutMeanQuality": getattr(req, 'rnaseqFastpCutMeanQuality', 20), "rnaseqFastpQualifiedQualityPhred": getattr(req, 'rnaseqFastpQualifiedQualityPhred', 20), "rnaseqFastpUnqualifiedPercentLimit": getattr(req, 'rnaseqFastpUnqualifiedPercentLimit', 30), "rnaseqFastpNBaseLimit": getattr(req, 'rnaseqFastpNBaseLimit', 5), "rnaseqFastpLengthRequired": getattr(req, 'rnaseqFastpLengthRequired', 30), "rnaseqSpliceSiteSource": getattr(req, 'rnaseqSpliceSiteSource', 'auto'), "rnaseqKnownSpliceSite": getattr(req, 'rnaseqKnownSpliceSite', ''), "rnaseqQuantMode": getattr(req, 'rnaseqQuantMode', 'dual'), "rnaseqAbundanceEngine": getattr(req, 'rnaseqAbundanceEngine', 'auto'), "rnaseqTranscriptomeFasta": getattr(req, 'rnaseqTranscriptomeFasta', ''), "rnaseqTx2gene": getattr(req, 'rnaseqTx2gene', ''), "rnaseqSalmonLibtype": getattr(req, 'rnaseqSalmonLibtype', 'A'), "rnaseqTximportCountsFromAbundance": getattr(req, 'rnaseqTximportCountsFromAbundance', 'no'), "rnaseqKallistoBias": bool(getattr(req, 'rnaseqKallistoBias', False)), "rnaseqDePadj": getattr(req, 'rnaseqDePadj', 0.05), "rnaseqDeLog2fc": getattr(req, 'rnaseqDeLog2fc', 1.0), "enrichPvalueCutoff": getattr(req, 'enrichPvalueCutoff', 0.05), "enrichQvalueCutoff": getattr(req, 'enrichQvalueCutoff', 0.05), "goObo": getattr(req, 'goObo', 'data/go/go-basic.obo'), "keggLevel": getattr(req, 'keggLevel', 'pathway'), "keggFilterScope": getattr(req, 'keggFilterScope', 'plant'), "keggHighlightMode": getattr(req, 'keggHighlightMode', 'auto'), "keggNoNetwork": bool(getattr(req, 'keggNoNetwork', False)), "keggSkipPathview": bool(getattr(req, 'keggSkipPathview', False)), "enrichPlotType": getattr(req, 'enrichPlotType', 'r_plot'), "rnaseqForegroundMode": getattr(req, 'rnaseqForegroundMode', 'auto'), "confirmedAutoAnnotation": bool(getattr(req, 'confirmedAutoAnnotation', False)), "annotationApplications": getattr(req, 'annotationApplications', ["Pfam", "ProSiteProfiles"]), "rnaseqSampleConfig": getattr(req, 'rnaseqSampleConfig', ''), "wgsSampleConfig": getattr(req, 'wgsSampleConfig', ''), "wgsReferenceFasta": getattr(req, 'wgsReferenceFasta', ''), "wgsQcManifest": getattr(req, 'wgsQcManifest', ''), "wgsIndexManifest": getattr(req, 'wgsIndexManifest', ''), "wgsSampleSetId": getattr(req, 'wgsSampleSetId', ''), "wgsReferenceId": getattr(req, 'wgsReferenceId', ''), "wgsAnalysisId": getattr(req, 'wgsAnalysisId', ''), "wgsFastp": getattr(req, 'wgsFastp', '') or str(ASAW_BIN / "fastp"), "wgsSamtools": get_wgs_samtools(req.stage, getattr(req, 'wgsSamtools', '')), "wgsBwaMem2": getattr(req, 'wgsBwaMem2', '') or str(ASAW_BIN / "bwa-mem2"), "wgsGatk": getattr(req, 'wgsGatk', '') or str(ASAW_BIN / "gatk"), "wgsThreads": getattr(req, 'wgsThreads', 1), "wgsMaxJobs": getattr(req, 'wgsMaxJobs', 1), "wgsSampleConfig": getattr(req, 'wgsSampleConfig', ''), "wgsFastp": getattr(req, 'wgsFastp', '') or str(ASAW_BIN / "fastp"), "wgsThreads": getattr(req, 'wgsThreads', 1), "wgsMaxJobs": getattr(req, 'wgsMaxJobs', 1), "wgsReferenceFasta": getattr(req, 'wgsReferenceFasta', ''), "wgsSamtools": get_wgs_samtools(req.stage, getattr(req, 'wgsSamtools', '')), "wgsBwaMem2": getattr(req, 'wgsBwaMem2', '') or str(ASAW_BIN / "bwa-mem2"), "wgsGatk": getattr(req, 'wgsGatk', '') or str(ASAW_BIN / "gatk"), "wgsQcManifest": getattr(req, 'wgsQcManifest', ''), "wgsIndexManifest": getattr(req, 'wgsIndexManifest', ''), "wgsAlignManifest": getattr(req, 'wgsAlignManifest', ''), "wgsMarkdupManifest": getattr(req, 'wgsMarkdupManifest', ''), "wgsBamQcManifest": getattr(req, 'wgsBamQcManifest', ''), "wgsHaplotypeManifest": getattr(req, 'wgsHaplotypeManifest', ''), "wgsCombineManifest": getattr(req, 'wgsCombineManifest', ''), "wgsJointManifest": getattr(req, 'wgsJointManifest', ''), "wgsSelectManifest": getattr(req, 'wgsSelectManifest', ''), "wgsFilterManifest": getattr(req, 'wgsFilterManifest', ''), "wgsImputeManifest": getattr(req, 'wgsImputeManifest', ''), "wgsSampleSetId": getattr(req, 'wgsSampleSetId', ''), "wgsReferenceId": getattr(req, 'wgsReferenceId', ''), "wgsAnalysisId": getattr(req, 'wgsAnalysisId', ''), "wgsIndexAnalysisId": getattr(req, 'wgsIndexAnalysisId', ''), "wgsMarkdupAnalysisId": getattr(req, 'wgsMarkdupAnalysisId', ''), "wgsBamQcAnalysisId": getattr(req, 'wgsBamQcAnalysisId', ''), "wgsHaplotypeAnalysisId": getattr(req, 'wgsHaplotypeAnalysisId', ''), "wgsCombineAnalysisId": getattr(req, 'wgsCombineAnalysisId', ''), "wgsJointAnalysisId": getattr(req, 'wgsJointAnalysisId', ''), "wgsSelectAnalysisId": getattr(req, 'wgsSelectAnalysisId', ''), "wgsFilterAnalysisId": getattr(req, 'wgsFilterAnalysisId', ''), "wgsImputeAnalysisId": getattr(req, 'wgsImputeAnalysisId', ''), "wgsNativePairHmmThreads": getattr(req, 'wgsNativePairHmmThreads', 1), "wgsMaxAlternateAlleles": getattr(req, 'wgsMaxAlternateAlleles', 6), "wgsCallingThreshold": getattr(req, 'wgsCallingThreshold', 10), "wgsIncludeNonVariantSites": bool(getattr(req, 'wgsIncludeNonVariantSites', False)), "wgsMethod": getattr(req, 'wgsMethod', 'gatk'), "wgsJava": get_wgs_java(getattr(req, 'wgsJava', '')), "wgsBeagleJar": getattr(req, 'wgsBeagleJar', '') or WGS_BEAGLE_JAR, "wgsMemory": getattr(req, 'wgsMemory', '8g'), "wgsWindow": getattr(req, 'wgsWindow', 40.0), "wgsOverlap": getattr(req, 'wgsOverlap', 2.0), "wgsNe": getattr(req, 'wgsNe', 100000), "wgsIterations": getattr(req, 'wgsIterations', 12), "wgsPhaseStates": getattr(req, 'wgsPhaseStates', 280), "wgsImpStates": getattr(req, 'wgsImpStates', 1600), "wgsCluster": getattr(req, 'wgsCluster', 0.005), "wgsMaf": getattr(req, 'wgsMaf', 0.05), "wgsMaxMissing": getattr(req, 'wgsMaxMissing', 0.5), "wgsBcftools": get_wgs_bcftools(getattr(req, 'wgsBcftools', '')), "wgsCoverageThreshold": getattr(req, 'wgsCoverageThreshold', 1), "wgsPlatform": getattr(req, 'wgsPlatform', 'illumina'), "rnaseqCountMatrix": getattr(req, 'rnaseqCountMatrix', ''), "rnaseqExpressionMatrix": getattr(req, 'rnaseqExpressionMatrix', ''), "rnaseqSampleTable": getattr(req, 'rnaseqSampleTable', ''), "fgseaComparison": getattr(req, 'fgseaComparison', ''), "fgseaMinSize": getattr(req, 'fgseaMinSize', 15), "fgseaMaxSize": getattr(req, 'fgseaMaxSize', 500), "fgseaNperm": getattr(req, 'fgseaNperm', 10000), "fgseaSeed": getattr(req, 'fgseaSeed', 1), "wgcnaMinMean": getattr(req, 'wgcnaMinMean', 1.0), "wgcnaMaxGenes": getattr(req, 'wgcnaMaxGenes', 8000), "wgcnaMinModuleSize": getattr(req, 'wgcnaMinModuleSize', 30), "wgcnaMergeCutHeight": getattr(req, 'wgcnaMergeCutHeight', 0.25), "wgcnaPowers": getattr(req, 'wgcnaPowers', "1,2,3,4,5,6,7,8,9,10,12,14,16,18,20"), "candidateMaxDegProteins": getattr(req, 'candidateMaxDegProteins', 500), "candidateRunInterproscan": getattr(req, 'candidateRunInterproscan', "auto"), "ncbiQuery": getattr(req, 'ncbiQuery', ''), "ncbiGeneIds": getattr(req, 'ncbiGeneIds', ''), "ncbiMode": getattr(req, 'ncbiMode', 'gene'), "ncbiIdType": getattr(req, 'ncbiIdType', 'auto'), "ncbiTaxid": getattr(req, 'ncbiTaxid', ''), "ncbiSpecies": getattr(req, 'ncbiSpecies', ''), "ncbiSeqTypes": getattr(req, 'ncbiSeqTypes', 'both'), "ncbiTranscriptPolicy": getattr(req, 'ncbiTranscriptPolicy', 'refseq_preferred'), "ncbiAllowFamilyDownload": getattr(req, 'ncbiAllowFamilyDownload', False), "ncbiEmail": getattr(req, 'ncbiEmail', ''), "sequenceExtractFasta": getattr(req, 'sequenceExtractFasta', ''), "sequenceExtractIdsFile": getattr(req, 'sequenceExtractIdsFile', ''), "sequenceExtractIds": getattr(req, 'sequenceExtractIds', ''), "sequenceExtractSeqType": getattr(req, 'sequenceExtractSeqType', 'auto'), "sequenceExtractRenameMode": getattr(req, 'sequenceExtractRenameMode', 'keep'), "sequenceExtractCleanHeaders": getattr(req, 'sequenceExtractCleanHeaders', False), # PPI参数 "ppiSpecies": getattr(req, 'ppiSpecies', 'auto'), "normalizationSpeciesLabel": getattr(req, 'normalizationSpeciesLabel', ''), "ppiReferenceTaxonId": getattr(req, 'ppiReferenceTaxonId', ''), "ppiReferenceStorageKey": getattr(req, 'ppiReferenceStorageKey', ''), "ppiReferenceSelectionMode": getattr(req, 'ppiReferenceSelectionMode', ''), "ppiReferenceUserConfirmed": bool(getattr(req, 'ppiReferenceUserConfirmed', False)), "dependencyOwner": getattr(req, 'dependencyOwner', ''), "ppiKaksReferenceSpecies": [] if ppi_owned_ortholog else list(technical_reference_species), "ppiReferenceSpecies": "" if ppi_owned_ortholog else (technical_reference_species[0] if technical_reference_species else getattr(req, "ppiReferenceSpecies", "")), "referenceBundleSpecies": "" if ppi_owned_ortholog else (technical_reference_species[0] if technical_reference_species else getattr(req, "referenceBundleSpecies", "")), "ppiScore": getattr(req, 'ppiScore', 200), # KaKs参数 "kaksNeutralRate": getattr(req, 'kaksNeutralRate', 7.08e-9), # Genome Prepare参数 "genomePrepareType": getattr(req, 'genomePrepareType', 'auto'), # Interspecies species selection "targetSpecies": scope_params.get("targetSpecies", []) or [], "referenceSpecies": scope_params.get("referenceSpecies", []) or [], "identifyBlastReferenceSpecies": scope_params.get("identifyBlastReferenceSpecies", []) or [], "primaryReferenceSpecies": scope_params.get("primaryReferenceSpecies", []) or [], "refSpeciesList": scope_params.get("refSpeciesList", []) or [], "comparativeSpecies": scope_params.get("comparativeSpecies", []) or [], "blastEngine": getattr(req, 'blastEngine', 'diamond') or 'diamond', "interspeciesPlotMode": getattr(req, 'interspeciesPlotMode', 'both') or 'both', } try: semantic_preflight = ( { "valid": False, "issues": [{ "severity": "error", "code": "invalid_wgs_manifest_prefix", "message": wgs_prefix_error, }], "decisions": {}, "materialized_params": {}, } if wgs_prefix_error else early_gwas_preflight or _api_semantic_preflight(job_id, req.stage, effective_prefix, params) ) params.update(semantic_preflight.get("materialized_params") or {}) params["semanticPreflight"] = semantic_preflight params["resolvedArtifacts"] = _semantic_resolved_artifacts(semantic_preflight) tasks[task_id]["semantic_preflight"] = semantic_preflight tasks[task_id]["resolved_artifacts"] = params["resolvedArtifacts"] if semantic_preflight.get("needs_confirmation"): tasks[task_id]["status"] = "waiting_confirmation" tasks[task_id]["message"] = semantic_preflight["issues"][0].get("message", "需要确认执行参数") tasks[task_id]["result"] = { "success": False, "status": "needs_confirmation", "message": tasks[task_id]["message"], "semantic_preflight": semantic_preflight, "resolved_artifacts": params["resolvedArtifacts"], "stage": req.stage, } return { "success": False, "async": False, "status": "needs_confirmation", "message": tasks[task_id]["message"], "job_id": job_id, "task_id": task_id, "semantic_preflight": semantic_preflight, "resolved_artifacts": params["resolvedArtifacts"], } if not semantic_preflight.get("valid", True): suggested_stages = _suggest_upstream_stages_for_preflight(req.stage, semantic_preflight) diagnosis = _run_preflight_diagnosis_for_task( job_id=job_id, stage=req.stage, prefix=effective_prefix, params=params, semantic_preflight=semantic_preflight, ) message = "执行前语义预检失败" if semantic_preflight.get("issues"): message = str(semantic_preflight["issues"][0].get("message") or message) if suggested_stages: message = f"{message}，建议先执行上游阶段: {', '.join(suggested_stages)}" tasks[task_id]["status"] = "failed" tasks[task_id]["message"] = message tasks[task_id]["diagnosis"] = diagnosis tasks[task_id]["result"] = { "success": False, "status": "preflight_failed", "message": message, "semantic_preflight": semantic_preflight, "resolved_artifacts": params["resolvedArtifacts"], "suggested_stages": suggested_stages, "diagnosis": diagnosis, "stage": req.stage, } return { "success": False, "async": False, "status": "preflight_failed", "message": message, "job_id": job_id, "task_id": task_id, "semantic_preflight": semantic_preflight, "resolved_artifacts": params["resolvedArtifacts"], "suggested_stages": suggested_stages, "diagnosis": diagnosis, } except Exception as exc: print(f"[API DEBUG] Semantic preflight failed non-fatally: {exc}") # Celery 并行执行时，真正的资源保护必须在后端做： # 前端传入的 cpu 是用户期望值，这里按阶段资源画像裁剪并路由到对应队列。 celery_queue = None resource_profile = {} try: from gfa_core.mas.resource_governor import record_task from gfa_core.mas.stage_catalog import clamp_stage_cpu, get_stage_queue, get_stage_resource_profile requested_cpu = params.get("cpu") resource_profile = get_stage_resource_profile(req.stage) try: from gfa_core.mas.resource_governor import get_resource_config saved_config = get_resource_config() saved_stage = (saved_config.get("stage_cpu_overrides") or {}).get(req.stage) or {} saved_default_cpu = int(saved_stage.get("default_cpu") or resource_profile.get("default_cpu") or requested_cpu or 1) request_value = int(requested_cpu or 0) default_request_cpu = getattr(getattr(ExecuteRequest, "model_fields", {}).get("cpu"), "default", 8) catalog_default_cpu = int(resource_profile.get("default_cpu") or default_request_cpu or 8) request_is_implicit = ( requested_cpu in (None, "", 0) or request_value == int(default_request_cpu or 8) or (bool(saved_stage) and request_value == catalog_default_cpu and saved_default_cpu != catalog_default_cpu) ) if req.stage not in RNA_SEQ_STAGES and request_is_implicit: requested_cpu = saved_default_cpu params["cpu"] = requested_cpu except Exception as resource_cpu_exc: print(f"[API DEBUG] Failed to apply saved stage CPU config: {resource_cpu_exc}") if req.stage in RNA_SEQ_STAGES: try: from gfa_core.mas.resource_governor import get_resource_config saved_config = get_resource_config() saved_stage = (saved_config.get("stage_cpu_overrides") or {}).get(req.stage) or {} saved_default_cpu = int(saved_stage.get("default_cpu") or resource_profile.get("default_cpu") or requested_cpu or 1) saved_project_cpu = int(saved_config.get("rnaseq_project_max_cpu") or saved_default_cpu) # Frontend requests historically carried the catalog default (8) even after # users edited the resource panel. Treat that stale default as implicit and # use the saved RNA-seq resource configuration instead. request_value = int(requested_cpu or 0) default_request_cpu = getattr(getattr(ExecuteRequest, "model_fields", {}).get("cpu"), "default", 8) request_is_implicit = requested_cpu in (None, "", 0) or request_value == int(default_request_cpu or 8) if request_is_implicit: requested_cpu = min(saved_default_cpu, saved_project_cpu) params["cpu"] = requested_cpu params["rnaseq_project_max_cpu"] = saved_project_cpu except Exception as resource_cpu_exc: print(f"[API DEBUG] Failed to apply saved RNA-seq CPU config: {resource_cpu_exc}") params["requested_cpu"] = requested_cpu params["cpu"] = clamp_stage_cpu(req.stage, requested_cpu) celery_queue = get_stage_queue(req.stage) tasks[task_id]["requested_cpu"] = requested_cpu tasks[task_id]["effective_cpu"] = params["cpu"] tasks[task_id]["celery_queue"] = celery_queue tasks[task_id]["resource_profile"] = resource_profile resource_record = record_task( task_id=task_id, stage=req.stage, job_id=job_id, prefix=effective_prefix, status="queued", cpu=params["cpu"], queue=celery_queue, resource_class=resource_profile.get("resource_class", ""), concurrency_group=resource_profile.get("concurrency_group", ""), message="任务已进入资源调度队列", ) tasks[task_id]["resource_state"] = resource_record print( f"[API DEBUG] Resource routing: stage={req.stage}, queue={celery_queue}, " f"requested_cpu={requested_cpu}, effective_cpu={params['cpu']}, " f"class={resource_profile.get('resource_class')}" ) except Exception as e: print(f"[API DEBUG] Resource routing fallback failed: {e}") # New ad-hoc utility tools should run in-process until all external Celery # workers are restarted with the same stage catalog. force_threadpool_stages = {"msa_region_plot", "tree_plot_revise", "synteny_plot_revise", "promoter_plot_revise", "combo"} # 使用新的Celery任务（统一架构） if CELERY_AVAILABLE and req.stage not in force_threadpool_stages: queue_health, restart_result = _ensure_celery_queue_health(celery_queue or _default_celery_queue(), reason="submit_stage") tasks[task_id]["celery_queue_health"] = queue_health if restart_result: tasks[task_id]["celery_restart"] = restart_result if not queue_health.get("available"): payload = _queue_unavailable_payload(req.stage, celery_queue or _default_celery_queue(), queue_health, submitted=False, restart_result=restart_result) tasks[task_id]["status"] = "queue_unavailable" tasks[task_id]["message"] = payload["message"] tasks[task_id]["result"] = payload tasks[task_id]["action_required"] = True tasks[task_id]["retryable"] = True tasks[task_id]["recommended_action"] = "restart_worker" try: from gfa_core.mas.resource_governor import release tasks[task_id]["resource_state"] = release(task_id, status="failed", message=payload["message"]) except Exception as e: print(f"[API DEBUG] Failed to release resource ledger for queue_unavailable: {e}") return { **payload, "async": False, "job_id": job_id, "task_id": task_id, "unified_arch": True, "resource_profile": resource_profile, "requested_cpu": params.get("requested_cpu"), "effective_cpu": params.get("cpu"), } try: from tasks import execute_single_stage_task celery_task = execute_single_stage_task.apply_async( kwargs={ "task_id": task_id, "stage": req.stage, "job_id": job_id, "prefix": effective_prefix, "params": params, "session_id": req.session_id or job_id, }, queue=celery_queue, ) print(f"[API DEBUG] Celery single stage task submitted: {celery_task.id}, queue={celery_queue}") tasks[task_id]["celery_task_id"] = celery_task.id try: from gfa_core.mas.resource_governor import record_task tasks[task_id]["resource_state"] = record_task( task_id=task_id, stage=req.stage, job_id=job_id, prefix=effective_prefix, status="queued", cpu=params.get("cpu") or 1, queue=celery_queue or "", resource_class=resource_profile.get("resource_class", ""), concurrency_group=resource_profile.get("concurrency_group", ""), celery_task_id=celery_task.id, message="任务已提交到 Celery，等待 worker 获取资源", ) except Exception as e: print(f"[API DEBUG] Failed to update resource ledger after Celery submit: {e}") except Exception as e: print(f"[API DEBUG] Failed to submit Celery task: {e}") # 降级到线程池执行 executor.submit(run_stage_sync, task_id, req.stage, job_id, params) else: # Celery不可用，使用线程池 executor.submit(run_stage_sync, task_id, req.stage, job_id, params) return { "success": True, "async": True, "task_id": task_id, "message": f"{req.stage} 任务已提交（{'threadpool' if req.stage in force_threadpool_stages else (celery_queue or 'threadpool')}，CPU {params.get('cpu')}），请在后台等待完成", "job_id": job_id, "unified_arch": True, # 标记使用统一架构 "celery_queue": celery_queue, "resource_profile": resource_profile, "requested_cpu": params.get("requested_cpu"), "effective_cpu": params.get("cpu"), "semantic_preflight": params.get("semanticPreflight"), "resolved_artifacts": params.get("resolvedArtifacts", {}), } @app.get("/api/task/{task_id}") async def get_task_status(task_id: str): """查询异步任务状态（支持Celery任务）""" if task_id not in tasks: recovered_reference_task = _load_reference_bundle_task(task_id) if recovered_reference_task: tasks[task_id] = recovered_reference_task else: try: from gfa_core.mas.resource_governor import get_task as get_resource_task resource_payload = get_resource_task(task_id) resource_task = resource_payload.get("task") if isinstance(resource_payload, dict) else None if isinstance(resource_task, dict): recovered_status = str(resource_task.get("status") or "").lower() status = "success" if recovered_status in {"completed", "success"} else ("failed" if recovered_status in {"failed", "failure", "cancelled", "revoked"} else recovered_status or "running") recovered_message = resource_task.get("message") or ("任务完成" if status == "success" else "任务状态已从资源记录恢复") tasks[task_id] = { "id": task_id, "stage": resource_task.get("stage", ""), "job_id": resource_task.get("job_id", ""), "prefix": resource_task.get("prefix", ""), "status": status, "message": recovered_message, "created_at": resource_task.get("created_at") or datetime.now().isoformat(), "start_time": resource_task.get("started_at"), "end_time": resource_task.get("ended_at"), "celery_task_id": resource_task.get("celery_task_id"), "celery_queue": resource_task.get("queue"), "requested_cpu": resource_task.get("requested_cpu"), "effective_cpu": resource_task.get("cpu"), "recovered_from_resource_ledger": True, "result": { "status": "success" if status == "success" else status, "success": status == "success", "message": recovered_message, "stage": resource_task.get("stage", ""), "manifest_recovery_pending": True, }, } else: raise HTTPException(status_code=404, detail="Task not found") except HTTPException: raise HTTPException(status_code=404, detail="Task not found") except Exception as exc: print(f"[API DEBUG] Failed to recover missing task {task_id} from resource ledger: {exc}") raise HTTPException(status_code=404, detail="Task not found") task = tasks[task_id] _apply_reference_bundle_stale_watchdog(task) # 如果是Celery任务，查询Celery状态 celery_task_id = task.get("celery_task_id") if celery_task_id and CELERY_AVAILABLE and not ( task.get("recovered_from_resource_ledger") and str(task.get("status") or "").lower() in {"failed", "success", "completed", "cancelled", "revoked"} ): try: from celery_app import app as celery_app celery_result = celery_app.AsyncResult(celery_task_id) # 更新本地任务状态 if celery_result.state == 'SUCCESS': # 检查Celery任务result内部的状态 result_data = celery_result.result if isinstance(result_data, dict): # 优先使用result_data中的status字段 result_status = result_data.get('status') if result_data.get('success') == False or result_status == 'failed': task["status"] = "failed" task["message"] = result_data.get('message', '任务执行失败') try: from gfa_core.mas.resource_governor import release task["resource_state"] = release(task_id, status="failed", message=task["message"]) except Exception as e: print(f"[API DEBUG] Failed to mark resource task failed: {e}") if isinstance(result_data.get("diagnosis"), dict): task["diagnosis"] = result_data.get("diagnosis") if isinstance(result_data.get("pending_confirmation"), dict): pending = result_data.get("pending_confirmation") sid = pending.get("session_id") or task.get("session_id") or task.get("job_id") jid = pending.get("job_id") or task.get("job_id") if sid and jid and _pending_should_be_visible(pending) and not _pending_resolved_by_manifest(pending): task["pending_confirmation"] = pending pending_confirmations[f"{sid}_{jid}"] = { **pending, "session_id": sid, "job_id": jid, "created_at": pending.get("created_at") or datetime.now().isoformat(), "deadline": pending.get("deadline") or (datetime.now() + timedelta(minutes=30)).isoformat(), } else: task.pop("pending_confirmation", None) elif result_status == 'completed': task["status"] = "success" task["message"] = result_data.get('message', '任务完成') try: from gfa_core.mas.resource_governor import release task["resource_state"] = release(task_id, status="success", message=task["message"]) except Exception as e: print(f"[API DEBUG] Failed to mark resource task success: {e}") else: # 其他状态（如waiting_confirmation） task["status"] = result_status or "success" task["message"] = result_data.get('message', '任务完成') task["result"] = result_data else: task["status"] = "success" task["result"] = result_data task["message"] = "任务完成" elif celery_result.state == 'FAILURE': task["status"] = "failed" task["message"] = str(celery_result.result) if celery_result.result else "任务失败" try: from gfa_core.mas.resource_governor import release task["resource_state"] = release(task_id, status="failed", message=task["message"]) except Exception as e: print(f"[API DEBUG] Failed to mark resource task failed: {e}") elif celery_result.state in {'RUNNING', 'PENDING', 'WAITING_RESOURCE'}: if celery_result.state == "WAITING_RESOURCE": task["status"] = "waiting_resource" elif celery_result.state == "PENDING": task["status"] = "queued" task["message"] = "任务已提交，等待 Celery worker 接收" _apply_pending_watchdog(task, celery_result.state) else: task["status"] = "running" # 获取meta信息 meta = celery_result.info if meta and isinstance(meta, dict): if meta.get("status") == "waiting_resource": task["status"] = "waiting_resource" task["message"] = meta.get("message", task.get("message", "任务运行中")) task["progress"] = meta.get("progress", 0) task["current_stage"] = meta.get("current_stage", task.get("stage")) if isinstance(meta.get("resource_usage"), dict): task["resource_usage"] = meta.get("resource_usage") # 检查是否有等待确认的状态 if meta.get("status") == "waiting_confirmation": pending = meta.get("pending_confirmation", {}) if isinstance(pending, dict) and _pending_should_be_visible(pending) and not _pending_resolved_by_manifest(pending): task["status"] = "waiting_confirmation" task["pending_confirmation"] = pending else: task.pop("pending_confirmation", None) except Exception as e: print(f"[API DEBUG] Error querying Celery task: {e}") _attach_single_stage_status(task) _attach_annotation_progress(task) # 读取日志文件内容（如果存在） log_content = "" try: job_id = task.get("job_id") stage = task.get("stage") if job_id and stage: latest_log = None result = task.get("result") if isinstance(task.get("result"), dict) else {} is_completed_gwas_failure = ( stage == "gwas_fmt" and str(task.get("status") or "").lower() in {"failed", "failure"} ) if is_completed_gwas_failure: result_log_path = str(result.get("log_path") or "").strip() safe_logs = _standard_stage_log_snapshot(WORKSPACE_ROOT / job_id, stage) if result_log_path: try: resolved_result_log = str(Path(result_log_path).resolve(strict=True)) except OSError: resolved_result_log = "" if resolved_result_log in safe_logs: latest_log = Path(resolved_result_log) else: latest_log = _latest_standard_stage_log(WORKSPACE_ROOT / job_id, stage) if latest_log: with latest_log.open('r', encoding='utf-8', errors='ignore') as f: log_content = f.read() except Exception as e: print(f"[API DEBUG] Error reading log file: {e}") diagnosis_payload = task.get("diagnosis") if not diagnosis_payload and isinstance(task.get("result"), dict): diagnosis_payload = task["result"].get("diagnosis") pending_payload = task.get("pending_confirmation") if isinstance(pending_payload, dict): if not _pending_should_be_visible(pending_payload) or _pending_resolved_by_manifest(pending_payload): task.pop("pending_confirmation", None) pending_payload = None resource_snapshot = None resource_task = None try: from gfa_core.mas.resource_governor import get_task, snapshot resource_task_payload = get_task(task_id) if isinstance(resource_task_payload, dict): resource_task = resource_task_payload.get("task") resource_snapshot = {"usage": resource_task_payload.get("usage")} stage_status_payload = task.get("stage_status") if isinstance(task.get("stage_status"), dict) else {} if ( isinstance(resource_task, dict) and stage_status_payload.get("completed") is True and str(resource_task.get("status") or "").lower() == "failed" ): from gfa_core.mas.resource_governor import release as release_resource_task repaired = release_resource_task(task_id, status="success", message=task.get("message") or "任务已由 manifest 校验恢复成功") if isinstance(repaired, dict): resource_task = repaired.get("task") or resource_task resource_snapshot = {"usage": repaired.get("usage")} else: resource_snapshot = {"usage": snapshot(include_terminal=False).get("usage")} except Exception as e: print(f"[API DEBUG] Error reading resource governor state: {e}") return { "id": task["id"], "stage": task["stage"], "status": task["status"], "message": task["message"], "created_at": task["created_at"], "start_time": task.get("start_time"), "end_time": task.get("end_time"), "result": task.get("result"), "stage_status": task.get("stage_status"), "log": log_content, # 添加日志内容 "progress": task.get("progress", 0), "current_stage": task.get("current_stage", task.get("stage")), "pending_confirmation": pending_payload, "diagnosis": diagnosis_payload, "annotation_progress": task.get("annotation_progress"), "unified_arch": task.get("unified_arch", False), "celery_queue": task.get("celery_queue"), "resource_profile": task.get("resource_profile"), "requested_cpu": task.get("requested_cpu"), "effective_cpu": task.get("effective_cpu"), "pending_seconds": task.get("pending_seconds"), "celery_queue_health": task.get("celery_queue_health"), "celery_restart": task.get("celery_restart"), "action_required": task.get("action_required", False), "retryable": task.get("retryable", False), "recommended_action": task.get("recommended_action"), "resource_task": resource_task, "resource_usage": task.get("resource_usage") or (resource_snapshot or {}).get("usage"), } class ConfirmedPlanStageFailure(RuntimeError): def __init__(self, payload: Dict[str, Any]): self.payload = dict(payload) super().__init__(str(self.payload.get("message") or "analysis stage failed")) def _confirmed_plan_stages( request: ConfirmedPlanExecutionRequest, ) -> List[ConfirmedPlanStage]: """Topologically order exactly the user-confirmed stages without adding work.""" stages = list(request.stages) stage_by_id = {stage.id: stage for stage in stages} if not stages or len(stage_by_id) != len(stages) or any(not stage.id for stage in stages): raise HTTPException(status_code=400, detail="confirmed plan stages are invalid") selected = set(stage_by_id) if any(dependency not in selected for stage in stages for dependency in stage.dependencies): raise HTTPException( status_code=400, detail="confirmed plan contains a dependency outside the confirmed stage set", ) ordered: List[ConfirmedPlanStage] = [] remaining = list(stages) completed: set[str] = set() while remaining: ready = [stage for stage in remaining if set(stage.dependencies) <= completed] if not ready: raise HTTPException(status_code=400, detail="confirmed plan dependencies are cyclic") for stage in ready: ordered.append(stage) completed.add(stage.id) remaining.remove(stage) return ordered def _confirmed_plan_status_path(job_id: str, run_id: str) -> Path: safe_job_id = str(job_id or "").strip() if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", safe_job_id): raise HTTPException(status_code=400, detail="invalid confirmed plan job_id") if not re.fullmatch(r"[0-9a-f]{20}", str(run_id or "")): raise HTTPException(status_code=400, detail="invalid confirmed plan run_id") return WORKSPACE_ROOT / safe_job_id / "final" / ".confirmed_plans" / f"{run_id}.json" def _write_confirmed_plan_status(path: Path, payload: Dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(".tmp") temporary.write_text( json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) temporary.replace(path) def _serialize_confirmed_param(name: str, value: Any) -> Any: field = ExecuteRequest.model_fields.get(name) if field is not None and field.annotation is str and isinstance(value, (list, tuple)): return ",".join(str(item) for item in value if str(item).strip()) return value def _confirmed_execute_request( request: ConfirmedPlanExecutionRequest, stage: ConfirmedPlanStage, ) -> ExecuteRequest: params = { key: _serialize_confirmed_param(key, value) for key, value in stage.params.items() if key in ExecuteRequest.model_fields and key not in {"stage", "job_id"} } params["stage"] = stage.id params["job_id"] = request.job_id params["prefix"] = str(params.get("prefix") or request.prefix) return ExecuteRequest(**params) async def _wait_for_submitted_stage( task_id: str, stage_id: str, timeout_seconds: float = 7200, ) -> Dict[str, Any]: deadline = time.monotonic() + timeout_seconds while time.monotonic() < deadline: status = await get_task_status(task_id) state = str(status.get("status") or "").lower() pending = status.get("pending_confirmation") if pending or state in {"paused", "waiting_confirmation", "pending_confirmation"}: return { "status": "paused", "task_id": task_id, "stage": stage_id, "pause": pending or {"type": "runtime_confirmation_required"}, } if state in {"success", "completed", "partial_success"}: return status if state in {"failed", "failure", "error", "cancelled", "canceled"}: raise ConfirmedPlanStageFailure(status) await asyncio.sleep(1) raise ConfirmedPlanStageFailure( { "status": "timeout", "message": f"{stage_id} exceeded the execution timeout", "task_id": task_id, } ) def _deduplicate_failure_reason(message: str) -> str: words = str(message or "analysis stage failed").split() for width in range(1, len(words) // 2 + 1): if len(words) % width == 0 and words == words[:width] * (len(words) // width): return " ".join(words[:width]) return " ".join(words) async def _run_confirmed_plan_execution( request: ConfirmedPlanExecutionRequest, run_id: str, status_path: Path, ) -> None: ordered = _confirmed_plan_stages(request) status: Dict[str, Any] = { "run_id": run_id, "plan_id": request.plan_id, "job_id": request.job_id, "status": "running", "stages": [stage.id for stage in ordered], "completed_stages": [], } _write_confirmed_plan_status(status_path, status) for stage in ordered: status["current_stage"] = stage.id _write_confirmed_plan_status(status_path, status) try: submitted = await execute_stage(_confirmed_execute_request(request, stage)) if not submitted.get("success", False): raise ConfirmedPlanStageFailure(submitted) task_id = str(submitted.get("task_id") or "") result = ( await _wait_for_submitted_stage(task_id, stage.id) if task_id else submitted ) if result.get("status") == "paused": status.update( { "status": "paused", "current_stage": stage.id, "pause": result.get("pause") or {}, } ) _write_confirmed_plan_status(status_path, status) return status["completed_stages"].append(stage.id) except ConfirmedPlanStageFailure as exc: reason = _deduplicate_failure_reason(str(exc.payload.get("message") or exc)) status.update( { "status": "failed", "current_stage": stage.id, "failure": { "stage": stage.id, "reason": reason, "user_message": reason, "log_path": str(exc.payload.get("log_path") or ""), }, } ) _write_confirmed_plan_status(status_path, status) return status.update({"status": "success", "current_stage": ""}) _write_confirmed_plan_status(status_path, status) @app.post("/api/agent/execute-confirmed") async def agent_execute_confirmed(request: ConfirmedPlanExecutionRequest): _confirmed_plan_stages(request) request_data = request.model_dump(mode="json") canonical = json.dumps( request_data, ensure_ascii=False, sort_keys=True, separators=(",", ":"), ) run_id = hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:20] status_path = _confirmed_plan_status_path(request.job_id, run_id) if status_path.is_file(): try: existing = json.loads(status_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): existing = {} if existing.get("status") in {"queued", "running", "paused", "success"}: return {**existing, "run_id": run_id, "duplicate": True} queued = { "run_id": run_id, "plan_id": request.plan_id, "job_id": request.job_id, "status": "queued", "stages": [stage.id for stage in _confirmed_plan_stages(request)], } _write_confirmed_plan_status(status_path, queued) asyncio.create_task(_run_confirmed_plan_execution(request, run_id, status_path)) return {**queued, "duplicate": False} def _get_celery_overview() -> Dict[str, Any]: overview: Dict[str, Any] = { "success": True, "checked_at": datetime.now().isoformat(), "workers": [], "queues": [], "active_tasks": [], "summary": { "worker_count": 0, "queue_count": 0, "active_task_count": 0, "all_expected_queues_online": False, }, "errors": [], } try: from gfa_core.mas.stage_catalog import get_queue_prefix prefix = get_queue_prefix() expected_queues = [f"{prefix}.{name}" for name in ["default", "heavy", "medium", "light", "rnaseq"]] except Exception: prefix = os.environ.get("BBAV5_QUEUE_PREFIX", "plantai_dev_7880").strip() or "plantai_dev_7880" expected_queues = [f"{prefix}.{name}" for name in ["default", "heavy", "medium", "light", "rnaseq"]] queue_map = { name: {"name": name, "workers": [], "worker_count": 0, "online": False} for name in expected_queues } if not CELERY_AVAILABLE: overview["success"] = False overview["errors"].append("Celery is not available in this API process") overview["queues"] = list(queue_map.values()) overview["ops"] = _current_celery_profile() return overview try: from celery_app import app as celery_app inspector = celery_app.control.inspect(timeout=CELERY_HEALTH_TIMEOUT_SEC) active_queues = inspector.active_queues() or {} active_tasks = inspector.active() or {} worker_queue_lookup: Dict[str, List[str]] = {} for worker_name, queues in active_queues.items(): worker_queue_names = [] for queue in queues or []: qname = str(queue.get("name") or queue.get("routing_key") or "") if not qname: continue worker_queue_names.append(qname) queue_map.setdefault(qname, {"name": qname, "workers": [], "worker_count": 0, "online": False}) queue_map[qname]["workers"].append(worker_name) worker_queue_lookup[worker_name] = sorted(set(worker_queue_names)) overview["workers"].append({ "name": worker_name, "queues": worker_queue_lookup[worker_name], "active_count": len(active_tasks.get(worker_name) or []), }) for worker_name, task_list in active_tasks.items(): queues = worker_queue_lookup.get(worker_name, []) for task in task_list or []: overview["active_tasks"].append({ "worker": worker_name, "queue": queues[0] if len(queues) == 1 else "", "queues": queues, "id": task.get("id"), "name": task.get("name"), "args": task.get("args"), "kwargs": task.get("kwargs"), "time_start": task.get("time_start"), }) for queue in queue_map.values(): queue["workers"] = sorted(set(queue.get("workers") or [])) queue["worker_count"] = len(queue["workers"]) queue["online"] = queue["worker_count"] > 0 overview["queues"] = [queue_map[name] for name in sorted(queue_map)] overview["workers"] = sorted(overview["workers"], key=lambda item: item.get("name") or "") overview["summary"] = { "worker_count": len(overview["workers"]), "queue_count": len(overview["queues"]), "active_task_count": len(overview["active_tasks"]), "all_expected_queues_online": all(queue_map[name]["online"] for name in expected_queues), } except Exception as exc: overview["success"] = False overview["errors"].append(f"Celery overview failed: {exc}") overview["queues"] = list(queue_map.values()) try: from gfa_core.mas.resource_governor import snapshot resource = snapshot(include_terminal=False) overview["resource_usage"] = resource.get("usage") overview["resource_tasks"] = resource.get("tasks", [])[:20] except Exception as exc: overview["errors"].append(f"Resource governor snapshot failed: {exc}") overview["thresholds"] = { "pending_warn_sec": CELERY_PENDING_WARN_SEC, "pending_stale_sec": CELERY_PENDING_STALE_SEC, "auto_restart": CELERY_AUTO_RESTART, "restart_cooldown_sec": CELERY_RESTART_COOLDOWN_SEC, } overview["ops"] = _current_celery_profile() return overview @app.get("/api/celery/overview") def get_celery_overview(): """Return worker, queue, active task, and resource status for the frontend monitor.""" return _get_celery_overview() @app.get("/api/celery/health") def get_celery_health(queue: str = "", auto_restart: bool = False): """Return live Celery worker/queue health for execution watchdogs.""" if auto_restart: health, restart_result = _ensure_celery_queue_health(queue, reason="health_endpoint") else: health = _get_celery_queue_health(queue) restart_result = {} return { "success": True, "health": health, "celery_restart": restart_result, "ops": _current_celery_profile(), "thresholds": { "pending_warn_sec": CELERY_PENDING_WARN_SEC, "pending_stale_sec": CELERY_PENDING_STALE_SEC, "auto_restart": CELERY_AUTO_RESTART, "restart_cooldown_sec": CELERY_RESTART_COOLDOWN_SEC, }, } class AdminOpsRequest(BaseModel): confirm: str force: bool = False token: Optional[str] = None def _validate_admin_ops_request(request: Request, payload: AdminOpsRequest, expected_confirm: str) -> Dict[str, Any]: if payload.confirm != expected_confirm: raise HTTPException(status_code=400, detail=f"confirm must be {expected_confirm}") provided_token = (payload.token or request.headers.get("X-PlantAI-Ops-Token") or "").strip() if ADMIN_OPS_TOKEN: if provided_token != ADMIN_OPS_TOKEN: raise HTTPException(status_code=403, detail="invalid admin ops token") return {"auth_mode": "token"} if ADMIN_OPS_ALLOW_CONFIRM_ONLY: return {"auth_mode": "confirm_only"} client_host = (request.client.host if request.client else "") or "" if client_host in {"127.0.0.1", "::1", "localhost"}: return {"auth_mode": "loopback"} raise HTTPException(status_code=403, detail="admin ops disabled; set BBAV5_ADMIN_OPS_TOKEN or BBAV5_ADMIN_OPS_ALLOW_CONFIRM_ONLY=1") @app.post("/api/admin/restart-celery") def restart_celery_workers(request: Request, payload: AdminOpsRequest): """Controlled Celery worker restart/recovery for the runtime monitor.""" auth = _validate_admin_ops_request(request, payload, "restart-celery") profile = _current_celery_profile() before = _get_celery_overview() active_tasks = before.get("active_tasks") if isinstance(before.get("active_tasks"), list) else [] if active_tasks and not payload.force: return { "success": False, "status": "active_tasks_present", "message": "Celery has active tasks; restart was not attempted. Retry after tasks finish or use force=true.", "ops": profile, "auth": auth, "active_tasks": active_tasks, "before": before, } processes_before = _find_celery_worker_pids(profile) stop_result = _terminate_celery_worker_pids(processes_before) start_result = _run_celery_start_script(profile) time.sleep(max(0.0, CELERY_RESTART_WAIT_SEC)) after = _get_celery_overview() queue_ok = bool((after.get("summary") or {}).get("all_expected_queues_online")) return { "success": bool(start_result.get("success") and queue_ok), "status": "restarted" if start_result.get("success") else "start_failed", "message": "Celery restart completed; inspect queue health in after.summary.", "ops": profile, "auth": auth, "processes_before": processes_before, "stop_result": stop_result, "start_result": start_result, "before": before, "after": after, } @app.post("/api/admin/restart-project") def restart_project(request: Request, payload: AdminOpsRequest): """Launch a configured external project restart script; never kill this API process inline.""" auth = _validate_admin_ops_request(request, payload, "restart-project") profile = _current_celery_profile() script = Path(str(profile.get("project_restart_script") or "")) if not script.exists(): return { "success": False, "status": "not_configured", "message": "Project restart script is not configured. Set PLANTAI_RESTART_SCRIPT or add restart_plantai_9000.sh.", "ops": profile, "auth": auth, } log_path = PROJECT_ROOT / "logs" / "project_restart_admin.log" log_path.parent.mkdir(parents=True, exist_ok=True) try: log_handle = open(log_path, "a", encoding="utf-8") subprocess.Popen( ["bash", str(script)], cwd=str(PROJECT_ROOT), stdout=log_handle, stderr=subprocess.STDOUT, start_new_session=True, ) return { "success": True, "status": "restart_submitted", "message": "Project restart script submitted; the web process may disconnect while restarting.", "ops": profile, "auth": auth, "log": str(log_path), } except Exception as exc: return { "success": False, "status": "submit_failed", "message": str(exc), "ops": profile, "auth": auth, "log": str(log_path), } class ResourceConfigRequest(BaseModel): total_cpu: Optional[int] = None max_running_tasks: Optional[int] = None gene_family_project_max_cpu: Optional[int] = None rnaseq_project_max_cpu: Optional[int] = None rnaseq_qc_sample_parallel_jobs: Optional[int] = None rnaseq_align_sample_parallel_jobs: Optional[int] = None stage_cpu_overrides: Optional[Dict[str, Dict[str, int]]] = None @app.get("/api/resource/status") async def get_resource_status(include_terminal: bool = False): """Return the global local resource ledger used before stage execution.""" try: from gfa_core.mas.resource_governor import get_resource_config, snapshot return {"success": True, "config": get_resource_config(), **snapshot(include_terminal=include_terminal)} except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) @app.post("/api/resource/config") async def set_resource_config(req: ResourceConfigRequest): """Update runtime resource limits for future submitted tasks.""" try: from gfa_core.mas.resource_governor import get_resource_config, snapshot, update_resource_config updates = req.dict(exclude_none=True) config = update_resource_config(updates) return {"success": True, "config": config, **snapshot(include_terminal=False)} except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) def _kill_job_stage_processes(job_id: str, stage: str, timeout_sec: float = 3.0) -> dict: """Best-effort kill of detached subprocesses for a given job+stage (PlantAI-aligned). Celery revoke(terminate=True) may not stop external tools (FastQC/cutadapt/hisat2/Rscript...). """ job_id = (job_id or "").strip() stage = (stage or "").strip() if not job_id or not stage: return {"matched": 0, "terminated": 0, "killed": 0, "pids": []} stage_dir = f"/workspace/{job_id}/{stage}" script_marker = f"apps/cli/run_{stage}.py" def iter_matches(): matches = [] try: for proc_dir in Path("/proc").iterdir(): if not proc_dir.name.isdigit(): continue pid = int(proc_dir.name) try: cmdline = (proc_dir / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", "ignore").strip() except Exception: continue if not cmdline: continue if job_id not in cmdline: continue if stage_dir in cmdline or script_marker in cmdline: matches.append((pid, cmdline)) except Exception as exc: print(f"[API DEBUG] _kill_job_stage_processes scan failed: {exc}") return matches matches = iter_matches() pids = sorted({pid for pid, _ in matches}) terminated = 0 for pid in pids: try: os.kill(pid, signal.SIGTERM) terminated += 1 except ProcessLookupError: pass except PermissionError: pass end = time.time() + max(0.1, float(timeout_sec)) while time.time() < end: alive = [] for pid in pids: try: os.kill(pid, 0) alive.append(pid) except ProcessLookupError: pass except PermissionError: alive.append(pid) if not alive: break time.sleep(0.2) killed = 0 remaining = [] for pid in pids: try: os.kill(pid, 0) remaining.append(pid) except ProcessLookupError: pass except PermissionError: remaining.append(pid) for pid in remaining: try: os.kill(pid, signal.SIGKILL) killed += 1 except ProcessLookupError: pass except PermissionError: pass return {"matched": len(pids), "terminated": terminated, "killed": killed, "pids": pids} @app.post("/api/task/{task_id}/cancel") async def cancel_task(task_id: str): """Cancel /api/execute task: Celery revoke + best-effort OS kill (PlantAI-aligned).""" if task_id not in tasks: return {"success": False, "message": "Task not found"} task = tasks[task_id] if task.get("status") in {"success", "completed", "failed", "cancelled"}: return {"success": False, "message": "Task already finished; cannot cancel"} celery_task_id = task.get("celery_task_id") if celery_task_id and CELERY_AVAILABLE: try: from celery_app import app as celery_app celery_app.control.revoke(celery_task_id, terminate=True, signal="SIGTERM") except Exception as e: print(f"[API DEBUG] Failed to revoke Celery task {celery_task_id}: {e}") job_id = task.get("job_id", "") or "" stage = task.get("stage", "") or "" kill_report = _kill_job_stage_processes(job_id=job_id, stage=stage) for other in tasks.values(): if ( other.get("job_id") == job_id and other.get("stage") == stage and other.get("status") in {"pending", "running"} ): other["status"] = "cancelled" other["message"] = "Task cancelled" other["end_time"] = datetime.now().isoformat() message = "Task cancelled" if kill_report.get("matched", 0): message += f"; terminated {kill_report.get('terminated', 0)} procs (killed {kill_report.get('killed', 0)})" return {"success": True, "message": message, "task_id": task_id, "kill": kill_report} @app.get("/api/methodology/bbav5-config") async def methodology_bbav5_config(batch_dir: str = "0505/methodology_batch_Ft_spl"): """Load methodology-to-bbav5 review config from gene-family-mas-v2 outputs.""" if not METHODOLOGY_CONFIG_AVAILABLE: raise HTTPException(status_code=500, detail="方法学配置模块不可用，请检查 GF_MAS_ROOT") try: return build_bbav5_methodology_config(batch_dir) except FileNotFoundError as e: raise HTTPException(status_code=404, detail=str(e)) except Exception as e: raise HTTPException(status_code=500, detail=f"读取方法学配置失败: {e}") @app.post("/api/methodology/extract-literature") async def methodology_extract_literature( file: UploadFile = File(...), paper_id: str = Form("uploaded_paper"), family: str = Form(""), output_name: str = Form(""), ): """Proxy literature upload to the RAG methodology MinerU+LLM service.""" try: content = await file.read() return _methodology_rag_request( "post", "methodology/extract-upload", files={"file": (file.filename or "paper.pdf", content, file.content_type or "application/pdf")}, data={"paper_id": paper_id, "family": family, "output_name": output_name}, timeout=30, ) except Exception as e: raise HTTPException(status_code=502, detail=f"无法触发 RAG MinerU+LLM 方法学提取: {e}") @app.get("/api/methodology/extract-jobs") async def methodology_extract_jobs(limit: int = 50, status: str = ""): try: params = {"limit": limit} if status: params["status"] = status return _methodology_rag_request("get", "methodology/extract-jobs", params=params, timeout=15) except Exception as e: raise HTTPException(status_code=502, detail=f"无法读取 RAG 方法学任务列表: {e}") @app.get("/api/methodology/extract-status/{job_id}") async def methodology_extract_status(job_id: str): try: return _methodology_rag_request("get", f"methodology/extract-jobs/{job_id}", timeout=15) except Exception as e: raise HTTPException(status_code=502, detail=f"无法读取 RAG 方法学任务状态: {e}") @app.get("/api/methodology/extract-config/{job_id}") async def methodology_extract_config(job_id: str): try: return _methodology_rag_request("get", f"methodology/extract-jobs/{job_id}/bbav5-config", timeout=30) except Exception as e: raise HTTPException(status_code=502, detail=f"无法读取 RAG 方法学配置: {e}") @app.get("/api/methodology/extract-plan/{job_id}") async def methodology_extract_plan(job_id: str): try: return _methodology_rag_request("get", f"methodology/extract-jobs/{job_id}/plan", timeout=30) except Exception as e: raise HTTPException(status_code=502, detail=f"无法读取 RAG 方法学方案: {e}") @app.get("/api/methodology/sandbox/example-batches") async def methodology_sandbox_example_batches(): """Proxy methodology sandbox examples through bbav5 so the frontend stays same-origin.""" try: return _methodology_rag_request("get", "methodology/sandbox/example-batches", timeout=15) except Exception as e: raise HTTPException(status_code=502, detail=f"无法读取 RAG 方法学 sandbox 示例: {e}") @app.post("/api/methodology/sandbox/build") async def methodology_sandbox_build(req: MethodologySandboxBuildRequest): """Build a temporary methodology QA sandbox without writing to bbav5 production stores.""" try: return _methodology_rag_request("post", "methodology/sandbox/build", json=req.dict(), timeout=60) except Exception as e: raise HTTPException(status_code=502, detail=f"无法构建 RAG 方法学 sandbox: {e}") @app.post("/api/methodology/sandbox/answer") async def methodology_sandbox_answer(req: MethodologySandboxAnswerRequest): """Ask the temporary methodology QA sandbox through the RAG service.""" try: return _methodology_rag_request("post", "methodology/sandbox/answer", json=req.dict(), timeout=30) except Exception as e: raise HTTPException(status_code=502, detail=f"无法查询 RAG 方法学 sandbox: {e}") async def _methodology_plan_from_request(req: MethodologyBindPlanRequest | MethodologyReadinessRequest) -> Dict[str, Any]: if req.plan: return req.plan if req.plan_path: path = Path(req.plan_path).expanduser() if not path.is_absolute(): path = PROJECT_ROOT / path if not path.exists(): raise HTTPException(status_code=404, detail=f"方法学方案文件不存在: {path}") return json.loads(path.read_text(encoding="utf-8")) if req.methodology_job_id: try: return _methodology_rag_request("get", f"methodology/extract-jobs/{req.methodology_job_id}/plan", timeout=30) except HTTPException: raise except Exception as exc: raise HTTPException(status_code=502, detail=f"无法读取 RAG 方法学方案: {exc}") from exc if req.job_id: bound = load_bound_methodology_plan(INPUT_ROOT, req.job_id) if bound: return bound raise HTTPException(status_code=400, detail="缺少 plan、plan_path、methodology_job_id 或已绑定 job plan") @app.post("/api/methodology/bind-plan") async def methodology_bind_plan(req: MethodologyBindPlanRequest): global current_job_id job_id = req.job_id or current_job_id if not job_id: job_id = datetime.now().strftime("%Y%m%d_%H%M%S_%f")[:18] current_job_id = job_id plan = await _methodology_plan_from_request(MethodologyBindPlanRequest(**{**req.dict(), "job_id": job_id})) result = bind_methodology_plan(plan, job_id=job_id, input_root=INPUT_ROOT, workspace_root=WORKSPACE_ROOT, source=req.source or req.plan_path or req.methodology_job_id) current_job_id = job_id return result @app.post("/api/methodology/plan-readiness") async def methodology_plan_readiness_api(req: MethodologyReadinessRequest): job_id = req.job_id or current_job_id if not job_id: raise HTTPException(status_code=400, detail="job_id is required for methodology readiness") plan = await _methodology_plan_from_request(MethodologyReadinessRequest(**{**req.dict(), "job_id": job_id})) return methodology_plan_readiness(plan, job_id=job_id, input_root=INPUT_ROOT, workspace_root=WORKSPACE_ROOT) @app.post("/api/methodology/execute-confirmed") async def methodology_execute_confirmed(req: MethodologyExecutionRequest): """Methodology execution is planned by the MAS planner from the frontend.""" return { "success": False, "status": "deprecated", "message": "方法学执行不再使用独立后端编排器；请通过 /api/agent/plan 生成多智能体规划后再逐阶段提交 /api/execute。", } @app.get("/api/rnaseq/defaults") async def get_rnaseq_defaults(job_id: str, prefix: str = "", stage: str = "rnaseq_fgsea"): """Infer RNA-seq species defaults from samples.yaml metadata, job resources, or prefix.""" try: resolved_prefix = resolve_effective_prefix(job_id, {"prefix": prefix}, stage or "rnaseq_fgsea") if job_id else prefix defaults = _infer_rnaseq_species_defaults(job_id, resolved_prefix or prefix, stage or "rnaseq_fgsea") if job_id else {} return {"success": True, **defaults} except Exception as exc: return {"success": False, "error": str(exc), "prefix": prefix, "speciesName": "", "keggOrganism": ""} @app.get("/api/stage-status") async def get_stage_status( job_id: str, stage: str, prefix: str = "", scope_id: str = "", scope_file: str = "", analysis_route: str = "", member_set_id: str = "", genotype_scope_id: str = "", phenotype_scope_id: str = "", covariate_scope_id: str = "", model_scope_id: str = "", source_model_scope_id: str = "", analysis_scope_id: str = "", wgs_sample_set_id: str = "", wgs_reference_id: str = "", wgs_analysis_id: str = "", ): """检查阶段是否已完成（统一 manifest/artifacts 校验）""" from gfa_core.mas.tools.stage_identity import get_stage_status_for_identity, resolve_stage_identity from gfa_core.mas.tools.stage_status import get_stage_status as get_manifest_stage_status prefix = resolve_effective_prefix(job_id, {"prefix": prefix}, stage) mapped_stage = stage identity = resolve_stage_identity( stage=mapped_stage, prefix=prefix, params={ "prefix": prefix, "scopeFile": scope_file, "scopeId": scope_id, "analysisRoute": analysis_route, "memberSetId": member_set_id, "genotypeScopeId": genotype_scope_id, "phenotypeScopeId": phenotype_scope_id, "covariateScopeId": covariate_scope_id, "modelScopeId": model_scope_id, "sourceModelScopeId": source_model_scope_id, "analysisScopeId": analysis_scope_id, "wgsSampleSetId": wgs_sample_set_id, "wgsReferenceId": wgs_reference_id, "wgsAnalysisId": wgs_analysis_id, }, job_id=job_id, workspace_root=WORKSPACE_ROOT, ) if mapped_stage == "subfamily_confirm": analysis_route = "confirmed_subfamily" identity.analysis_route = "confirmed_subfamily" if identity.scope_id: member_set_id = f"{identity.scope_id}:confirmed_subfamily:all_members" identity.member_set_id = member_set_id identity.result_group = "gene_family_subfamily_confirm" explicit_identity_requested = bool( scope_id or scope_file or analysis_route or member_set_id or genotype_scope_id or phenotype_scope_id or covariate_scope_id or analysis_scope_id or model_scope_id or source_model_scope_id or wgs_sample_set_id or wgs_reference_id or wgs_analysis_id ) if explicit_identity_requested: report = get_stage_status_for_identity( job_id=job_id, identity=identity, workspace_root=WORKSPACE_ROOT, strict_identity=True, ) else: # Plain status probes should not turn an inferred fallback route such as # custom into a strict expectation. They are asking whether the stage has # any valid completed manifest for the prefix. report = get_manifest_stage_status( job_id=job_id, prefix=identity.prefix, stage=identity.stage, workspace_root=WORKSPACE_ROOT, strict_identity=False, ) payload = report.to_dict() if not report.completed and report.status == "not_found" and analysis_route and prefix: final_dir = WORKSPACE_ROOT / job_id / "final" prefix_species, _prefix_family = _prefix_species_family(prefix) recovered = None if final_dir.exists(): for manifest_path in sorted(final_dir.glob(f"*.{mapped_stage}.manifest.json")): try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: continue if str(manifest.get("stage") or "") != mapped_stage: continue if str(manifest.get("prefix") or "") != prefix: continue manifest_route = str( manifest.get("analysis_route") or (manifest.get("parameters") or {}).get("analysis_route") or (manifest.get("stage_context") or {}).get("analysis_route") or "" ) if manifest_route != analysis_route: continue manifest_scope = str(manifest.get("scope_id") or (manifest.get("stage_context") or {}).get("scope_id") or "") if scope_id and manifest_scope != scope_id: continue manifest_member_set = str(manifest.get("member_set_id") or (manifest.get("stage_context") or {}).get("member_set_id") or "") if member_set_id and manifest_member_set != member_set_id: continue if prefix_species: manifest_targets = manifest.get("target_species") or (manifest.get("stage_context") or {}).get("target_species") or [] if isinstance(manifest_targets, list) and manifest_targets and prefix_species not in {str(item) for item in manifest_targets}: continue recovered = manifest recovered["path"] = str(manifest_path) break if recovered is not None: payload = { "stage": mapped_stage, "status": "completed", "completed": True, "manifest_path": recovered.get("path", ""), "manifest_status": recovered.get("status"), "artifacts_checked": len(recovered.get("artifacts") or {}), "missing_artifacts": [], "errors": [], "prefix": recovered.get("prefix") or prefix, "output_prefix": recovered.get("output_prefix") or "", "scope_id": recovered.get("scope_id") or (recovered.get("stage_context") or {}).get("scope_id") or "", "scope_mode": recovered.get("scope_mode") or (recovered.get("stage_context") or {}).get("scope_mode") or "", "analysis_route": analysis_route, "member_set_id": recovered.get("member_set_id") or (recovered.get("stage_context") or {}).get("member_set_id") or "", } report_manifest = recovered else: report_manifest = report.manifest else: report_manifest = report.manifest payload["requested_stage"] = stage payload["mapped_stage"] = mapped_stage payload["status"] = "completed" if payload.get("completed") else payload.get("status", report.status) payload["identity"] = identity.to_dict() if report_manifest is not None: payload["manifest"] = report_manifest def _mark_api_evidence_completed(payload: dict[str, Any]) -> None: warnings = list(payload.get("missing_artifacts") or []) + list(payload.get("errors") or []) if warnings: quality = payload.setdefault("manifest_quality", {}) existing = list(quality.get("api_completed_with_artifact_warnings") or []) quality["api_completed_with_artifact_warnings"] = existing + warnings payload["missing_artifacts"] = [] payload["errors"] = [] if stage == "annotation": final_dir = WORKSPACE_ROOT / job_id / "final" has_background = False if final_dir.exists(): has_background = any(final_dir.glob("*gene2go*.txt")) or any(final_dir.glob("*gene2ko*.txt")) has_annotation_manifest = any((final_dir / name).exists() for name in ( f"{prefix}.annotation.manifest.json", f"{prefix}.annotation_go.manifest.json", f"{prefix}.annotation_kegg.manifest.json", )) payload["completed"] = bool(has_annotation_manifest and has_background) payload["status"] = "completed" if payload["completed"] else "not_found" if payload["completed"]: _mark_api_evidence_completed(payload) if stage in {"rnaseq_go_enrich", "rnaseq_kegg_enrich"}: final_dir = WORKSPACE_ROOT / job_id / "final" if final_dir.exists(): if stage == "rnaseq_go_enrich": has_specific = any(final_dir.glob("*.GO_enrich.txt")) else: has_specific = any(final_dir.glob("*.KEGG_enrich.txt")) # GO/KEGG 子阶段用各自结果文件硬判定，避免共享 rnaseq_enrich manifest 导致串扰 payload["completed"] = bool(has_specific) payload["status"] = "completed" if payload["completed"] else "not_found" if payload["completed"]: _mark_api_evidence_completed(payload) return payload @app.get("/api/tasks") async def list_tasks(): """列出所有任务""" return { "tasks": [ { "id": t["id"], "stage": t["stage"], "status": t["status"], "message": t["message"], "created_at": t["created_at"] } for t in tasks.values() ] } @app.get("/api/results") async def get_results( job_id: str = Query(default=None, description="可选的job_id，默认使用当前job"), stage: str = Query(default="", description="可选，仅返回当前查看阶段相关结果"), result_prefix: str = "", rnaseq_scope_id: str = Query(default="", description="可选，仅返回指定 RNA-seq scope 的结果"), scope_id: str = Query(default="", description="可选，仅返回指定基因家族 scope 的结果"), genotype_scope_id: str = Query(default="", description="可选，仅返回指定 VCF SHA scope 的 GWAS 结果"), phenotype_scope_id: str = "", covariate_scope_id: str = "", model_scope_id: str = "", source_model_scope_id: str = "", analysis_scope_id: str = "", wgs_sample_set_id: str = "", wgs_reference_id: str = "", wgs_analysis_id: str = "", ): """获取任务结果，可指定job_id""" global current_job_id # 使用传入的job_id或当前job_id target_job_id = job_id or current_job_id if not target_job_id: return {"identify": {"images": [], "files": []}} workspace_dir = WORKSPACE_ROOT / target_job_id final_dir = workspace_dir / "final" stages = { "identify": {"images": [], "files": [], "elapsed_time": 0}, "identify_refine": {"images": [], "files": [], "elapsed_time": 0}, "phylogeny": {"images": [], "files": [], "elapsed_time": 0}, "phylogeny_msa": {"images": [], "files": [], "msa_groups": {"conserved": []}, "elapsed_time": 0}, "msa_region_plot": {"images": [], "files": [], "elapsed_time": 0}, "tree_plot_revise": {"images": [], "files": [], "elapsed_time": 0}, "synteny_plot_revise": {"images": [], "files": [], "elapsed_time": 0}, "reference_classification": {"images": [], "files": [], "msa_files": [], "elapsed_time": 0}, "subfamily_confirm": {"images": [], "files": [], "elapsed_time": 0}, "pep": {"images": [], "files": [], "elapsed_time": 0}, "motif": {"images": [], "files": [], "elapsed_time": 0}, "domain": {"images": [], "files": [], "elapsed_time": 0}, "structure": {"images": [], "files": [], "elapsed_time": 0}, "promoter": {"images": [], "files": [], "elapsed_time": 0}, "promoter_plot_revise": {"images": [], "files": [], "elapsed_time": 0}, "tfbs": {"images": [], "files": [], "elapsed_time": 0}, "synteny": {"images": [], "files": [], "elapsed_time": 0}, "karyotype": {"images": [], "files": [], "elapsed_time": 0}, "interspecies_synteny": {"images": [], "files": [], "elapsed_time": 0}, "combo": {"images": [], "files": [], "elapsed_time": 0}, "expression": {"images": [], "files": [], "elapsed_time": 0}, "wgs_qc": {"images": [], "files": [], "elapsed_time": 0}, "wgs_index": {"images": [], "files": [], "elapsed_time": 0}, "wgs_align": {"images": [], "files": [], "elapsed_time": 0}, "wgs_markdup": {"images": [], "files": [], "elapsed_time": 0}, "wgs_bam_qc": {"images": [], "files": [], "elapsed_time": 0}, "wgs_haplotype_calling": {"images": [], "files": [], "elapsed_time": 0}, "wgs_gvcf_combine": {"images": [], "files": [], "elapsed_time": 0}, "wgs_joint_genotyping": {"images": [], "files": [], "elapsed_time": 0}, "wgs_variant_select": {"images": [], "files": [], "elapsed_time": 0}, "wgs_variant_filter": {"images": [], "files": [], "elapsed_time": 0}, "wgs_impute": {"images": [], "files": [], "elapsed_time": 0}, "wgs_maf_missing_filter": {"images": [], "files": [], "elapsed_time": 0}, "rnaseq_qc": {"images": [], "files": [], "elapsed_time": 0}, "rnaseq_align": {"images": [], "files": [], "elapsed_time": 0}, "rnaseq_quant": {"images": [], "files": [], "elapsed_time": 0}, "rnaseq_sample_qc": {"images": [], "files": [], "elapsed_time": 0}, "rnaseq_de": {"images": [], "files": [], "elapsed_time": 0}, "annotation": {"images": [], "files": [], "elapsed_time": 0}, "rnaseq_go_enrich": {"images": [], "files": [], "elapsed_time": 0}, "rnaseq_kegg_enrich": {"images": [], "files": [], "elapsed_time": 0}, "rnaseq_fgsea": {"images": [], "files": [], "elapsed_time": 0}, "rnaseq_wgcna": {"images": [], "files": [], "elapsed_time": 0}, "rnaseq_candidate_family": {"images": [], "files": [], "elapsed_time": 0}, "genome_prepare": {"images": [], "files": [], "elapsed_time": 0}, "prepare_organize": {"images": [], "files": [], "elapsed_time": 0}, "prepare_normalize_gff": {"images": [], "files": [], "elapsed_time": 0}, "prepare_longest_transcript": {"images": [], "files": [], "elapsed_time": 0}, "prepare_extract_pep": {"images": [], "files": [], "elapsed_time": 0}, "prepare_extract_cds": {"images": [], "files": [], "elapsed_time": 0}, "prepare_filter_fasta": {"images": [], "files": [], "elapsed_time": 0}, "prepare_simplify_ncbi": {"images": [], "files": [], "elapsed_time": 0}, "sequence_extract_by_id": {"images": [], "files": [], "elapsed_time": 0}, "ppi": {"images": [], "files": [], "elapsed_time": 0}, "kaks": {"images": [], "files": [], "elapsed_time": 0}, "gwas_fmt": {"images": [], "files": [], "elapsed_time": 0}, "gwas_kinship": {"images": [], "files": [], "elapsed_time": 0}, "gwas_trait": {"images": [], "files": [], "elapsed_time": 0}, "gwas_cov": {"images": [], "files": [], "elapsed_time": 0}, "gwas_mlm": {"images": [], "files": [], "elapsed_time": 0}, "gwas_plot": {"images": [], "files": [], "elapsed_time": 0}, "gwas_emmax": {"images": [], "files": [], "elapsed_time": 0}, "gwas_emplot": {"images": [], "files": [], "elapsed_time": 0}, "gwas_glm": {"images": [], "files": [], "elapsed_time": 0}, "pop_pca": {"images": [], "files": [], "elapsed_time": 0}, "pop_phylogeny": {"images": [], "files": [], "elapsed_time": 0}, "pop_structure": {"images": [], "files": [], "elapsed_time": 0}, "pop_ld_decay": {"images": [], "files": [], "elapsed_time": 0}, "pop_snp_density": {"images": [], "files": [], "elapsed_time": 0}, } wgs_impute_result_roles = { "imputed_snp_vcf", "imputed_snp_vcf_index", "impute_report", "impute_log", } wgs_maf_missing_result_roles = { "final_snp_vcf", "final_snp_vcf_index", "maf_missing_report", "maf_missing_stats", } result_groups = {} hidden_intermediate_artifact_names: set[str] = set() hidden_intermediate_stage_prefixes: list[tuple[str, str]] = [] requested_rnaseq_scope_id = str( rnaseq_scope_id if isinstance(rnaseq_scope_id, str) else "" ).strip() requested_scope_id = str( scope_id if isinstance(scope_id, str) else "" ).strip() requested_wgs_sample_set_id = str(wgs_sample_set_id or "").strip() requested_wgs_reference_id = str(wgs_reference_id or "").strip() requested_wgs_analysis_id = str(wgs_analysis_id or "").strip() requested_genotype_scope_id = str(genotype_scope_id or "").strip() requested_population_prefix = str(result_prefix or "").strip() requested_wgs_sample_set_id = str(wgs_sample_set_id or "").strip() requested_wgs_reference_id = str(wgs_reference_id or "").strip() requested_wgs_analysis_id = str(wgs_analysis_id or "").strip() requested_gwas_scopes = { "genotype_scope_id": requested_genotype_scope_id, "phenotype_scope_id": str(phenotype_scope_id or "").strip(), "covariate_scope_id": str(covariate_scope_id or "").strip(), "model_scope_id": str(model_scope_id or "").strip(), "source_model_scope_id": str(source_model_scope_id or "").strip(), "analysis_scope_id": str(analysis_scope_id or "").strip(), } if ( stage in POPULATION_STAGES and requested_population_prefix and not requested_genotype_scope_id and not requested_gwas_scopes["analysis_scope_id"] ): try: population_identity = resolve_population_manifest_identity( workspace=workspace_dir, stage=stage, prefix=requested_population_prefix, ) except PopulationManifestError: population_identity = {} requested_genotype_scope_id = str(population_identity.get("genotypeScopeId") or "") requested_gwas_scopes["genotype_scope_id"] = requested_genotype_scope_id requested_gwas_scopes["analysis_scope_id"] = str( population_identity.get("analysisScopeId") or "" ) accepted_manifest_artifact_names: set[str] = set() accepted_manifest_artifact_paths: set[str] = set() accepted_manifest_files: set[str] = set() wgs_manifest_artifact_names: set[str] = set() rnaseq_manifest_scope_candidates: list[tuple[float, str, str]] = [] active_rnaseq_scope_id = requested_rnaseq_scope_id RNASEQ_RESULT_STAGES = { "rnaseq_qc", "rnaseq_align", "rnaseq_quant", "rnaseq_sample_qc", "rnaseq_de", "rnaseq_enrich", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea", "rnaseq_wgcna", "rnaseq_candidate_family", "expression", } RNASEQ_FILE_TOKENS = ( "rnaseq_", ".rnaseq_", "rnaseq-expression", "rnaseq_expression", "rnaseq_expression_plot", ".count_matrix", "fpkm_matrix", "tpm_matrix", "vst_matrix", "featurecounts", "gene_lengths", "normalization_info", "_ma_plot", "_volcano", "sample_correlation", "sample_distance", "sample_clustering", "soft_threshold", "module_trait", "gene_modules", "module_sizes", "candidate_family", ) def _artifact_url(path_value: str) -> str: path_text = str(path_value or "") if not path_text: return "" try: path_obj = Path(path_text) if path_obj.is_absolute(): path_text = str(path_obj.relative_to(workspace_dir)) except Exception: pass return f"/workspace/{target_job_id}/{path_text}" def _stage_tree_replot_group(stage_name: str): payload = stages.get(str(stage_name or "")) if not isinstance(payload, dict): return None return payload.setdefault("replot_groups", {}).setdefault("tree", []) def _stage_circos_replot_group(stage_name: str): payload = stages.get(str(stage_name or "")) if not isinstance(payload, dict): return None return payload.setdefault("replot_groups", {}).setdefault("circos", []) def _stage_combo_replot_group(stage_name: str): payload = stages.get(str(stage_name or "")) if not isinstance(payload, dict): return None return payload.setdefault("replot_groups", {}).setdefault("combo", []) def _manifest_combo_output_label(manifest: dict) -> str: params = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {} return str(params.get("output_label") or manifest.get("output_label") or "").strip() def _manifest_combo_panels(manifest: dict) -> list[str]: params = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {} raw_panels = params.get("panels") or manifest.get("panels") or [] if isinstance(raw_panels, str): raw_panels = [part.strip() for part in raw_panels.split(",")] if not isinstance(raw_panels, list): return [] allowed = {"motif", "structure", "domain"} panels: list[str] = [] for panel in raw_panels: panel_name = str(panel or "").strip().lower() if panel_name in allowed and panel_name not in panels: panels.append(panel_name) return panels def _manifest_stage_name(manifest_file: Path, manifest: dict) -> str: stage_name = str(manifest.get("stage") or "").strip() if stage_name == "rnaseq_enrich": mf_lower = manifest_file.name.lower() if "rnaseq_go_enrich" in mf_lower: return "rnaseq_go_enrich" if "rnaseq_kegg_enrich" in mf_lower: return "rnaseq_kegg_enrich" return stage_name def _manifest_params(manifest: dict) -> dict: params = manifest.get("parameters") return params if isinstance(params, dict) else {} def _manifest_rnaseq_scope(manifest: dict) -> str: params = _manifest_params(manifest) stage_context = manifest.get("stage_context") if isinstance(manifest.get("stage_context"), dict) else {} return str( manifest.get("rnaseq_scope_id") or params.get("rnaseq_scope_id") or stage_context.get("rnaseq_scope_id") or "" ).strip() def _manifest_scope(manifest: dict) -> str: params = _manifest_params(manifest) stage_context = manifest.get("stage_context") if isinstance(manifest.get("stage_context"), dict) else {} return str( manifest.get("scope_id") or params.get("scope_id") or stage_context.get("scope_id") or "" ).strip() def _register_manifest_scope_candidate(manifest_file: Path, manifest: dict) -> None: stage_name = _manifest_stage_name(manifest_file, manifest) if stage_name not in RNASEQ_RESULT_STAGES: return scope_value = _manifest_rnaseq_scope(manifest) if not scope_value: return try: mtime = manifest_file.stat().st_mtime except Exception: mtime = 0.0 rnaseq_manifest_scope_candidates.append((mtime, stage_name, scope_value)) def _rnaseq_scope_sample_count(scope_value: str) -> int: match = re.search(r"\.(\d+)samples\.", str(scope_value or "")) if not match: return 0 try: return int(match.group(1)) except Exception: return 0 def _rnaseq_scope_prefix(scope_value: str) -> str: return str(scope_value or "").split(".rnaseq.", 1)[0] def _infer_active_rnaseq_scope() -> str: if active_rnaseq_scope_id: return active_rnaseq_scope_id requested_stage = str(stage or "").strip() candidates = rnaseq_manifest_scope_candidates if requested_stage: stage_candidates = [item for item in candidates if item[1] == requested_stage] if stage_candidates: candidates = stage_candidates if not candidates: return "" priority = { "expression": 80, "rnaseq_de": 70, "rnaseq_sample_qc": 60, "rnaseq_quant": 50, "rnaseq_fgsea": 45, "rnaseq_wgcna": 40, "rnaseq_candidate_family": 35, } def candidate_key(item: tuple[float, str, str]) -> tuple[int, float, int]: mtime, stage_name, scope_value = item has_real_samples = 1 if _rnaseq_scope_sample_count(scope_value) > 0 else 0 return (has_real_samples, mtime, priority.get(stage_name, 0)) return sorted(candidates, key=candidate_key, reverse=True)[0][2] def _legacy_unscoped_rnaseq_manifest_can_follow_active_scope(manifest_file: Path, manifest: dict) -> bool: if not active_rnaseq_scope_id or _manifest_rnaseq_scope(manifest): return False stage_name = _manifest_stage_name(manifest_file, manifest) if stage_name not in RNASEQ_RESULT_STAGES: return False scope_prefix = _rnaseq_scope_prefix(active_rnaseq_scope_id) params = _manifest_params(manifest) manifest_prefix = str( manifest.get("prefix") or manifest.get("output_prefix") or params.get("prefix") or params.get("output_prefix") or "" ).strip() prefix_matches = not manifest_prefix or manifest_prefix == scope_prefix or manifest_file.name.startswith(f"{scope_prefix}.") if not prefix_matches: return False if stage_name in {"rnaseq_qc", "rnaseq_align"}: return True unique_scopes = {item[2] for item in rnaseq_manifest_scope_candidates if item[2] and _rnaseq_scope_sample_count(item[2]) > 0} return unique_scopes == {active_rnaseq_scope_id} def _zero_sample_rnaseq_manifest_can_follow_active_scope(manifest_file: Path, manifest: dict) -> bool: if not active_rnaseq_scope_id or _rnaseq_scope_sample_count(active_rnaseq_scope_id) <= 0: return False manifest_scope = _manifest_rnaseq_scope(manifest) if not manifest_scope or _rnaseq_scope_sample_count(manifest_scope) > 0: return False if _rnaseq_scope_prefix(manifest_scope) != _rnaseq_scope_prefix(active_rnaseq_scope_id): return False return _manifest_stage_name(manifest_file, manifest) in {"rnaseq_go_enrich", "rnaseq_kegg_enrich"} def _manifest_matches_requested_scope(manifest_file: Path, manifest: dict) -> bool: stage_name = _manifest_stage_name(manifest_file, manifest) if stage_name == "wgs_qc": # WGS QC artifacts belong to one exact sample set and analysis # configuration. Missing IDs must not fall back to a sibling run. return ( bool(requested_wgs_sample_set_id and requested_wgs_analysis_id) and str(manifest.get("wgs_sample_set_id") or "").strip() == requested_wgs_sample_set_id and str(manifest.get("wgs_analysis_id") or "").strip() == requested_wgs_analysis_id ) if stage_name == "wgs_index": return ( bool(requested_wgs_reference_id and requested_wgs_analysis_id) and str(manifest.get("wgs_reference_id") or "").strip() == requested_wgs_reference_id and str(manifest.get("wgs_analysis_id") or "").strip() == requested_wgs_analysis_id ) if stage_name == "wgs_align": return ( bool(requested_wgs_sample_set_id and requested_wgs_reference_id and requested_wgs_analysis_id) and str(manifest.get("wgs_sample_set_id") or "").strip() == requested_wgs_sample_set_id and str(manifest.get("wgs_reference_id") or "").strip() == requested_wgs_reference_id and str(manifest.get("wgs_analysis_id") or "").strip() == requested_wgs_analysis_id ) if stage_name.startswith("pop_"): requested_analysis_scope = requested_gwas_scopes["analysis_scope_id"] if not requested_genotype_scope_id or not requested_analysis_scope: return False if requested_population_prefix and str(manifest.get("prefix") or "").strip() != requested_population_prefix: return False return ( str(manifest.get("genotype_scope_id") or "").strip() == requested_genotype_scope_id and str(manifest.get("analysis_scope_id") or "").strip() == requested_analysis_scope ) return (bool(requested_wgs_sample_set_id and requested_wgs_analysis_id) and str(manifest.get("wgs_sample_set_id") or "").strip() == requested_wgs_sample_set_id and str(manifest.get("wgs_analysis_id") or "").strip() == requested_wgs_analysis_id) if stage_name == "wgs_index": return (bool(requested_wgs_reference_id and requested_wgs_analysis_id) and str(manifest.get("wgs_reference_id") or "").strip() == requested_wgs_reference_id and str(manifest.get("wgs_analysis_id") or "").strip() == requested_wgs_analysis_id) if stage_name in {"wgs_impute", "wgs_maf_missing_filter"}: if not ( requested_wgs_sample_set_id and requested_wgs_reference_id and requested_wgs_analysis_id and str(manifest.get("wgs_sample_set_id") or "").strip() == requested_wgs_sample_set_id and str(manifest.get("wgs_reference_id") or "").strip() == requested_wgs_reference_id and str(manifest.get("wgs_analysis_id") or "").strip() == requested_wgs_analysis_id ): return False try: from gfa_core.mas.tools.stage_status import ( get_stage_status as get_manifest_stage_status, ) exact = get_manifest_stage_status( target_job_id, str(manifest.get("prefix") or "").strip(), stage_name, workspace_root=WORKSPACE_ROOT, wgs_sample_set_id=requested_wgs_sample_set_id, wgs_reference_id=requested_wgs_reference_id, wgs_analysis_id=requested_wgs_analysis_id, ) return ( exact.completed and exact.manifest_path is not None and exact.manifest_path.resolve() == manifest_file.resolve() ) except Exception: return False if stage_name in {"wgs_align", "wgs_markdup", "wgs_bam_qc", "wgs_haplotype_calling", "wgs_gvcf_combine", "wgs_joint_genotyping", "wgs_variant_select", "wgs_variant_filter"}: return (bool(requested_wgs_sample_set_id and requested_wgs_reference_id and requested_wgs_analysis_id) and str(manifest.get("wgs_sample_set_id") or "").strip() == requested_wgs_sample_set_id and str(manifest.get("wgs_reference_id") or "").strip() == requested_wgs_reference_id and str(manifest.get("wgs_analysis_id") or "").strip() == requested_wgs_analysis_id) if stage_name == "gwas_fmt": if not requested_genotype_scope_id: return False manifest_genotype_scope = str( manifest.get("genotype_scope_id") or _manifest_params(manifest).get("genotype_scope_id") or (manifest.get("stage_context") or {}).get("genotype_scope_id") or "" ).strip() return manifest_genotype_scope == requested_genotype_scope_id if stage_name in _GWAS_RESULT_STAGES: applicable = _GWAS_STAGE_SCOPE_FIELDS.get(stage_name, ()) if stage_name in {"gwas_mlm", "gwas_plot", "gwas_emmax", "gwas_emplot", "gwas_glm"}: supplied = [requested_gwas_scopes.get(key, "") for key in applicable] if any(supplied) and not all(supplied): return False if not applicable: if not requested_genotype_scope_id: return False return str(manifest.get("genotype_scope_id") or "").strip() == requested_genotype_scope_id expected = {key: requested_gwas_scopes[key] for key in applicable if requested_gwas_scopes.get(key)} if not expected: return False return all(str(manifest.get(key) or "").strip() == value for key, value in expected.items()) if active_rnaseq_scope_id and stage_name in RNASEQ_RESULT_STAGES: manifest_scope = _manifest_rnaseq_scope(manifest) return ( manifest_scope == active_rnaseq_scope_id or _legacy_unscoped_rnaseq_manifest_can_follow_active_scope(manifest_file, manifest) or _zero_sample_rnaseq_manifest_can_follow_active_scope(manifest_file, manifest) ) if requested_scope_id and stage_name not in RNASEQ_RESULT_STAGES: manifest_scope = _manifest_scope(manifest) if manifest_scope != requested_scope_id: if stage_name == "combo" and not manifest_scope and len(_manifest_combo_panels(manifest)) == 2: return True return False return True def _manifest_artifact_records(manifest: dict) -> list[dict]: if str(manifest.get("stage") or "") == "gwas_fmt": return _gwas_fmt_result_artifacts( manifest, requested_genotype_scope_id, workspace_dir ) if str(manifest.get("stage") or "") in _GWAS_RESULT_STAGES: stage_name = str(manifest.get("stage") or "") applicable = _GWAS_STAGE_SCOPE_FIELDS.get(stage_name, ()) expected = ( { key: requested_gwas_scopes[key] for key in applicable if requested_gwas_scopes.get(key) } if applicable else ({"genotype_scope_id": requested_genotype_scope_id} if requested_genotype_scope_id else {}) ) return _validated_gwas_result_artifacts( manifest, workspace=workspace_dir, expected_scopes=expected, ) if str(manifest.get("stage") or "").startswith("pop_"): artifacts = manifest.get("artifacts") or {} records = list(artifacts.values()) if isinstance(artifacts, dict) else list(artifacts) validated: list[dict] = [] for artifact in records: if not isinstance(artifact, dict): return [] raw_path = str(artifact.get("path") or "").strip() declared_sha = str(artifact.get("sha256") or "").strip().lower() if not raw_path or not re.fullmatch(r"[0-9a-f]{64}", declared_sha): return [] candidate = Path(raw_path) candidate = candidate if candidate.is_absolute() else workspace_dir / candidate try: candidate = candidate.resolve() candidate.relative_to(workspace_dir.resolve()) except (OSError, ValueError): return [] if not candidate.is_file() or candidate.stat().st_size <= 0: return [] digest = hashlib.sha256() with candidate.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) if digest.hexdigest() != declared_sha: return [] validated.append(artifact) return validated records: list[dict] = [] artifacts = manifest.get("artifacts") or {} if isinstance(artifacts, dict): records.extend([item for item in artifacts.values() if isinstance(item, dict)]) elif isinstance(artifacts, list): records.extend([item for item in artifacts if isinstance(item, dict)]) figures = manifest.get("figures") or [] if isinstance(figures, list): records.extend([item for item in figures if isinstance(item, dict)]) return records def _remember_accepted_manifest_artifacts(manifest_file: Path, manifest: dict) -> None: accepted_manifest_files.add(manifest_file.name) for artifact in _manifest_artifact_records(manifest): manifest_stage = str(manifest.get("stage") or "") artifact_role = str( artifact.get("role") or artifact.get("artifact") or "" ) if ( manifest_stage == "wgs_impute" and artifact_role not in wgs_impute_result_roles ): continue if ( manifest_stage == "wgs_maf_missing_filter" and artifact_role not in wgs_maf_missing_result_roles ): continue art_path = artifact.get("path") or artifact.get("full_path") if not art_path: continue art_name = Path(str(art_path)).name accepted_manifest_artifact_names.add(art_name) accepted_manifest_artifact_paths.add(_artifact_url(art_path)) def _looks_like_rnaseq_file(fname: str) -> bool: lower = str(fname or "").lower() return any(token in lower for token in RNASEQ_FILE_TOKENS) def _allow_rnaseq_fallback_file(fname: str, target_stage_name: str = "") -> bool: if not active_rnaseq_scope_id: return True if target_stage_name in RNASEQ_RESULT_STAGES or _looks_like_rnaseq_file(fname): return fname in accepted_manifest_artifact_names or fname in accepted_manifest_files return True def _is_target_reference_evidence_manifest(manifest_file: Path, manifest: dict) -> bool: stage_name = str(manifest.get("stage") or "").lower() if stage_name not in {"phylogeny", "motif", "domain", "structure"}: return False params = manifest.get("parameters") or {} route = str(manifest.get("analysis_route") or params.get("analysis_route") or "").lower() if route in {"custom_phylogeny", "comparative_phylogeny", "single_species", "multi_species", "multi_target"}: return False haystack = " ".join( str(value or "") for value in ( manifest_file.name, manifest.get("scope_id"), manifest.get("output_prefix"), manifest.get("member_set_id"), manifest.get("analysis_route"), manifest.get("result_group"), params.get("scope_id"), params.get("output_prefix"), params.get("member_set_id"), params.get("analysis_route"), params.get("result_group"), ) ).lower() return "target_plus_reference" in haystack and ".ref-" in haystack def _is_subfamily_intermediate_manifest(manifest_file: Path, manifest: dict) -> bool: params = manifest.get("parameters") or {} stage_name = str(manifest.get("stage") or _manifest_stage_name(manifest_file, manifest) or "").lower() route = str(manifest.get("analysis_route") or params.get("analysis_route") or "").lower() group = str(manifest.get("result_group") or params.get("result_group") or "").lower() return ( (route == "subfamily_evidence" and stage_name in {"phylogeny", "motif", "domain", "combo"}) or route in {"subfamily_intermediate_legacy"} or group in {"subfamily_intermediate", "subfamily_confirmation_intermediate"} ) def _remember_hidden_manifest_artifacts(manifest_file: Path, manifest: dict) -> None: for artifact in _manifest_artifact_records(manifest): art_path = artifact.get("path") or artifact.get("full_path") if art_path: hidden_intermediate_artifact_names.add(Path(str(art_path)).name) stage_name = str(manifest.get("stage") or "").lower() output_prefix = str(manifest.get("output_prefix") or (manifest.get("parameters") or {}).get("output_prefix") or "").strip() if not output_prefix and stage_name and manifest_file.name.endswith(f".{stage_name}.manifest.json"): output_prefix = manifest_file.name[: -len(f".{stage_name}.manifest.json")] if stage_name and output_prefix: hidden_intermediate_stage_prefixes.append((stage_name, output_prefix)) def _is_hidden_intermediate_file(fname: str) -> bool: lower = str(fname or "").lower() if ".custom_phylogeny." in lower or ".comparative_phylogeny." in lower: return False if fname in hidden_intermediate_artifact_names: return True for stage_name, output_prefix in hidden_intermediate_stage_prefixes: if not fname.startswith(output_prefix): continue if stage_name == "phylogeny" and any(token in fname for token in (".tree", ".pep.msa", ".msa_plot", ".phylogeny")): return True if stage_name in {"motif", "domain", "structure", "combo"} and f".{stage_name}" in fname: return True return False def _is_public_reference_classification_name(fname: str) -> bool: lower = str(fname or "").lower() if "reference_classification.tree" in lower: return False if ".spl.target-" in lower and any(lower.endswith(ext) for ext in (".assignment.tsv", ".subfamily.tsv", ".confidence.tsv", ".geneid.subfamily", ".manifest.json")): return False if "target_plus_reference.combo" in lower and lower.endswith((".png", ".pdf")): return True if lower.endswith((".assignment.tsv", ".subfamily.tsv", ".confidence.tsv", ".geneid.subfamily", ".confirmed_subfamily.tsv", ".combo.pdf")): return True return False def _add_manifest_result_group(manifest_file: Path, manifest: dict): if "combo_panel_test" in manifest_file.name.lower(): return group_name = ( manifest.get("result_group") or (manifest.get("parameters") or {}).get("result_group") or manifest.get("stage") or "unknown" ) group = result_groups.setdefault(group_name, { "images": [], "files": [], "manifests": [], "elapsed_time": 0, "stage": manifest.get("stage", ""), "analysis_route": manifest.get("analysis_route") or (manifest.get("parameters") or {}).get("analysis_route", ""), "scope_id": manifest.get("scope_id") or (manifest.get("parameters") or {}).get("scope_id", ""), "output_prefix": manifest.get("output_prefix") or (manifest.get("parameters") or {}).get("output_prefix", ""), "member_set_id": manifest.get("member_set_id") or (manifest.get("parameters") or {}).get("member_set_id", ""), "rnaseq_scope_id": _manifest_rnaseq_scope(manifest), "scope_type": manifest.get("scope_type") or (manifest.get("parameters") or {}).get("scope_type", ""), "rnaseq_condition_signature": manifest.get("rnaseq_condition_signature") or (manifest.get("parameters") or {}).get("rnaseq_condition_signature", ""), "method": manifest.get("method") or (manifest.get("parameters") or {}).get("method", ""), }) elapsed = manifest.get("elapsed_sec") or 0 if elapsed: group["elapsed_time"] = round(max(float(group.get("elapsed_time") or 0), float(elapsed)), 1) group["manifests"].append({ "name": manifest_file.name, "path": f"/workspace/{target_job_id}/final/{manifest_file.name}", "stage": manifest.get("stage", ""), "status": manifest.get("status", ""), "method": manifest.get("method") or (manifest.get("parameters") or {}).get("method", ""), }) stage_target_name = str(manifest.get("stage") or "") if stage_target_name == "rnaseq_enrich": mf_lower = manifest_file.name.lower() if "rnaseq_go_enrich" in mf_lower: stage_target_name = "rnaseq_go_enrich" elif "rnaseq_kegg_enrich" in mf_lower: stage_target_name = "rnaseq_kegg_enrich" if stage_target_name == "expression" and ( _manifest_rnaseq_scope(manifest) or any("rnaseq_expression_plot" in str((artifact or {}).get("path") or (artifact or {}).get("full_path") or "").lower() for artifact in _manifest_artifact_records(manifest)) or "rnaseq_expression_plot" in manifest_file.name.lower() ): stage_target_name = "rnaseq_de" if stage_target_name in stages: if stage_target_name == "wgs_qc": stages[stage_target_name]["wgs_sample_set_id"] = str(manifest.get("wgs_sample_set_id") or "").strip() stages[stage_target_name]["wgs_analysis_id"] = str(manifest.get("wgs_analysis_id") or "").strip() if stage_target_name == "wgs_index": stages[stage_target_name]["wgs_reference_id"] = str(manifest.get("wgs_reference_id") or "").strip() stages[stage_target_name]["wgs_analysis_id"] = str(manifest.get("wgs_analysis_id") or "").strip() if stage_target_name == "wgs_align": stages[stage_target_name]["wgs_sample_set_id"] = str(manifest.get("wgs_sample_set_id") or "").strip() stages[stage_target_name]["wgs_reference_id"] = str(manifest.get("wgs_reference_id") or "").strip() stages[stage_target_name]["wgs_analysis_id"] = str(manifest.get("wgs_analysis_id") or "").strip() if stage_target_name == "wgs_markdup": stages[stage_target_name]["wgs_sample_set_id"] = str(manifest.get("wgs_sample_set_id") or "").strip() stages[stage_target_name]["wgs_reference_id"] = str(manifest.get("wgs_reference_id") or "").strip() stages[stage_target_name]["wgs_analysis_id"] = str(manifest.get("wgs_analysis_id") or "").strip() if stage_target_name == "wgs_bam_qc": stages[stage_target_name]["wgs_sample_set_id"] = str(manifest.get("wgs_sample_set_id") or "").strip() stages[stage_target_name]["wgs_reference_id"] = str(manifest.get("wgs_reference_id") or "").strip() stages[stage_target_name]["wgs_analysis_id"] = str(manifest.get("wgs_analysis_id") or "").strip() if stage_target_name == "wgs_haplotype_calling": stages[stage_target_name]["wgs_sample_set_id"] = str(manifest.get("wgs_sample_set_id") or "").strip() stages[stage_target_name]["wgs_reference_id"] = str(manifest.get("wgs_reference_id") or "").strip() stages[stage_target_name]["wgs_analysis_id"] = str(manifest.get("wgs_analysis_id") or "").strip() if stage_target_name == "wgs_gvcf_combine": stages[stage_target_name]["wgs_sample_set_id"] = str(manifest.get("wgs_sample_set_id") or "").strip() stages[stage_target_name]["wgs_reference_id"] = str(manifest.get("wgs_reference_id") or "").strip() stages[stage_target_name]["wgs_analysis_id"] = str(manifest.get("wgs_analysis_id") or "").strip() if stage_target_name == "wgs_joint_genotyping": stages[stage_target_name]["wgs_sample_set_id"] = str(manifest.get("wgs_sample_set_id") or "").strip() stages[stage_target_name]["wgs_reference_id"] = str(manifest.get("wgs_reference_id") or "").strip() stages[stage_target_name]["wgs_analysis_id"] = str(manifest.get("wgs_analysis_id") or "").strip() if stage_target_name == "wgs_variant_select": stages[stage_target_name]["wgs_sample_set_id"] = str(manifest.get("wgs_sample_set_id") or "").strip() stages[stage_target_name]["wgs_reference_id"] = str(manifest.get("wgs_reference_id") or "").strip() stages[stage_target_name]["wgs_analysis_id"] = str(manifest.get("wgs_analysis_id") or "").strip() stages[stage_target_name]["method"] = str(manifest.get("method") or "").strip() if stage_target_name == "wgs_variant_filter": stages[stage_target_name]["wgs_sample_set_id"] = str(manifest.get("wgs_sample_set_id") or "").strip() stages[stage_target_name]["wgs_reference_id"] = str(manifest.get("wgs_reference_id") or "").strip() stages[stage_target_name]["wgs_analysis_id"] = str(manifest.get("wgs_analysis_id") or "").strip() stages[stage_target_name]["method"] = str(manifest.get("method") or "").strip() if stage_target_name in {"wgs_impute", "wgs_maf_missing_filter"}: stages[stage_target_name]["wgs_sample_set_id"] = str(manifest.get("wgs_sample_set_id") or "").strip() stages[stage_target_name]["wgs_reference_id"] = str(manifest.get("wgs_reference_id") or "").strip() stages[stage_target_name]["wgs_analysis_id"] = str(manifest.get("wgs_analysis_id") or "").strip() stages[stage_target_name]["method"] = str(manifest.get("method") or "").strip() stages[stage_target_name]["metrics"] = ( manifest.get("metrics") if isinstance(manifest.get("metrics"), dict) else {} ) manifest_status = str(manifest.get("status") or "") if manifest_status in {"success", "partial_success"}: stages[stage_target_name]["status"] = "success" elif manifest_status: stages[stage_target_name]["status"] = "failed" stages[stage_target_name]["error"] = manifest.get("error") or (manifest.get("parameters") or {}).get("error") or manifest_status stage_target = stages.get(stage_target_name) if (stage_target_name == "gwas_fmt" or stage_target_name in _GWAS_RESULT_STAGES) and stage_target is not None: manifest_info = { "name": manifest_file.name, "size": manifest_file.stat().st_size, "path": f"/workspace/{target_job_id}/final/{manifest_file.name}", "role": "manifest", "artifact": manifest_file.name, "type": "application/json", "media_type": "application/json", "description": f"{stage_target_name} manifest", } stage_target["files"].append(manifest_info) for artifact in _manifest_artifact_records(manifest): art_path = artifact.get("path") or artifact.get("full_path") art_url = _artifact_url(art_path) if not art_url: continue art_name = Path(str(art_path)).name if (manifest.get("stage") == "reference_classification" or str(group_name).lower() == "subfamily_confirmation") and not _is_public_reference_classification_name(art_name): continue art_size = artifact.get("size", 0) if not art_size: try: local_art_path = Path(str(art_path)) if not local_art_path.is_absolute(): local_art_path = workspace_dir / local_art_path if local_art_path.exists(): art_size = local_art_path.stat().st_size except Exception: art_size = 0 info = { "name": art_name, "size": art_size, "path": art_url, "role": artifact.get("role") or artifact.get("artifact", ""), "artifact": artifact.get("artifact", ""), "description": artifact.get("description", ""), } suffix = Path(str(art_path)).suffix.lower() artifact_kind = str(artifact.get("artifact") or "") artifact_stage_name = stage_target_name art_name_lower = art_name.lower() if stage_target_name == "combo" and suffix in {".png", ".jpg", ".jpeg", ".svg", ".pdf"}: combo_panels = _manifest_combo_panels(manifest) combo_output_label = _manifest_combo_output_label(manifest) is_combo_replot = artifact_kind.startswith("combo_replot") or bool(combo_output_label) if combo_output_label and combo_output_label.lower() not in art_name_lower: is_combo_replot = False if is_combo_replot and len(combo_panels) == 2: combo_info = dict(info) combo_info["panels"] = combo_panels if not combo_info.get("description"): combo_info["description"] = f"Combo 修订图: {','.join(combo_panels)}" combo_group = _stage_combo_replot_group("combo") if combo_group is not None and art_url not in {item.get("path") for item in combo_group}: combo_group.append(combo_info) if stage_target_name == "phylogeny": if "tree_replot" in art_name_lower or artifact_kind.startswith("tree_replot"): artifact_stage_name = "tree_plot_revise" elif "msa_region_" in art_name_lower or artifact_kind.startswith("msa_region"): artifact_stage_name = "msa_region_plot" elif (".msa_plot" in art_name_lower or "msa_plot_msa" in art_name_lower or art_name_lower.endswith(".pep.msa.fa")): artifact_stage_name = "phylogeny_msa" stage_target = stages.get(artifact_stage_name) if suffix != ".json" and ("synteny_replot" in art_name_lower or artifact_kind.startswith("synteny_replot")): params = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {} display_stage = str( manifest.get("display_stage") or params.get("display_stage") or manifest.get("source_stage") or params.get("source_stage") or "synteny" ).strip() if display_stage not in stages or display_stage == "synteny_plot_revise": display_stage = "synteny" replot_group = _stage_circos_replot_group(display_stage) if replot_group is not None and art_url not in {item.get("path") for item in replot_group}: replot_group.append(info) revise_stage = stages.get("synteny_plot_revise") if revise_stage is not None: revise_group = revise_stage.setdefault("replot_groups", {}).setdefault("circos", []) if art_url not in {item.get("path") for item in revise_group}: revise_group.append(info) if suffix in {".png", ".jpg", ".jpeg", ".svg"}: if art_url not in revise_stage.get("images", []): revise_stage.setdefault("images", []).append(art_url) elif info.get("path") not in {item.get("path") for item in revise_stage.get("files", [])}: revise_stage.setdefault("files", []).append(info) if suffix in {".png", ".jpg", ".jpeg", ".svg"}: group["images"].append(art_url) else: group["files"].append(info) continue stage_target = stages.get(artifact_stage_name) if "tree_replot" in art_name_lower or artifact_kind.startswith("tree_replot"): params = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {} display_stage = str( manifest.get("display_stage") or params.get("display_stage") or manifest.get("source_stage") or params.get("source_stage") or "" ).strip() if display_stage not in stages or display_stage == "tree_plot_revise": source_image_name = Path(str(params.get("source_image") or manifest.get("source_image") or "")).name.lower() if ".motif.tree_motif." in source_image_name: display_stage = "motif" elif ".reference_classification.tree" in source_image_name: display_stage = "reference_classification" elif ".interspecies_tree" in source_image_name: display_stage = "interspecies_synteny" elif ".combo." in source_image_name: display_stage = "combo" else: display_stage = "" replot_group = _stage_tree_replot_group(display_stage) if display_stage else None if replot_group is not None and art_url not in {item.get("path") for item in replot_group}: replot_group.append(info) if suffix in {".png", ".jpg", ".jpeg", ".svg"}: group["images"].append(art_url) else: group["files"].append(info) continue seen_stage_paths = {item.get("path") for item in (stage_target or {}).get("files", [])} if stage_target else set() seen_stage_images = set((stage_target or {}).get("images", [])) if stage_target else set() if artifact_stage_name == "phylogeny" and artifact_kind.startswith("msa_conserved_region"): phy_msa = stages.get("phylogeny_msa") if phy_msa is not None: conserved_group = phy_msa.setdefault("msa_groups", {}).setdefault("conserved", []) if art_url not in {item.get("path") for item in conserved_group}: conserved_group.append(info) if suffix in {".png", ".jpg", ".jpeg", ".svg"}: continue is_image = artifact.get("type") == "image" or suffix in {".png", ".jpg", ".jpeg", ".svg", ".pdf"} if is_image: if suffix in {".png", ".jpg", ".jpeg", ".svg"}: group["images"].append(art_url) if stage_target is not None and art_url not in seen_stage_images: stage_target["images"].append(art_url) seen_stage_images.add(art_url) else: group["files"].append(info) if stage_target is not None and art_url not in seen_stage_paths: stage_target["files"].append(info) seen_stage_paths.add(art_url) else: group["files"].append(info) if stage_target is not None and art_url not in seen_stage_paths: stage_target["files"].append(info) seen_stage_paths.add(art_url) # 从final目录读取执行时间（manifest文件在final/下，不在reports/下） # 与 vue3copy0417 一致：manifest["stage"] in stages 则写 elapsed；富集 manifest 内 stage 常为 rnaseq_enrich，按文件名映射到子阶段 manifest_payloads: list[tuple[Path, dict]] = [] if final_dir.exists(): for manifest_file in final_dir.glob("*.manifest.json"): try: import json with open(manifest_file, 'r') as f: manifest = json.load(f) if not isinstance(manifest, dict): continue manifest_payloads.append((manifest_file, manifest)) if _manifest_stage_name(manifest_file, manifest) in {"wgs_qc", "wgs_index"}: for artifact in _manifest_artifact_records(manifest): artifact_path = artifact.get("path") or artifact.get("full_path") if artifact_path: wgs_manifest_artifact_names.add(Path(str(artifact_path)).name) _register_manifest_scope_candidate(manifest_file, manifest) except Exception: pass # Result browsing is a persisted job view, so apply manifests in execution # order. The newest attempt for a stage becomes its visible status instead # of depending on filesystem iteration order. manifest_payloads.sort( key=lambda item: (item[0].stat().st_mtime, item[0].name) ) # A refreshed GWAS format result request may not carry the genotype scope. # Rehydrate it only when the requested prefix (if any) identifies exactly # one successful scientific scope; otherwise remain fail-closed. if str(stage or "").strip() == "gwas_fmt" and not requested_genotype_scope_id: gwas_fmt_scopes: set[str] = set() for candidate_file, candidate_manifest in manifest_payloads: if _manifest_stage_name(candidate_file, candidate_manifest) != "gwas_fmt": continue if str(candidate_manifest.get("status") or "") not in {"success", "partial_success"}: continue candidate_prefix = str(candidate_manifest.get("prefix") or "").strip() if requested_population_prefix and candidate_prefix != requested_population_prefix: continue candidate_params = _manifest_params(candidate_manifest) candidate_context = ( candidate_manifest.get("stage_context") if isinstance(candidate_manifest.get("stage_context"), dict) else {} ) candidate_scope = str( candidate_manifest.get("genotype_scope_id") or candidate_params.get("genotype_scope_id") or candidate_context.get("genotype_scope_id") or "" ).strip() if candidate_scope: gwas_fmt_scopes.add(candidate_scope) if len(gwas_fmt_scopes) == 1: requested_genotype_scope_id = next(iter(gwas_fmt_scopes)) requested_gwas_scopes["genotype_scope_id"] = requested_genotype_scope_id active_rnaseq_scope_id = _infer_active_rnaseq_scope() # A rerun failure is authoritative for the same population-analysis identity; # never replay an older success merely because its artifacts still exist. population_latest_manifest: dict[tuple[str, str, str, str], Path] = {} population_latest_stamp: dict[tuple[str, str, str, str], tuple[str, float]] = {} for candidate_file, candidate_manifest in manifest_payloads: candidate_stage = _manifest_stage_name(candidate_file, candidate_manifest) if not candidate_stage.startswith("pop_"): continue candidate_key = ( candidate_stage, str(candidate_manifest.get("prefix") or ""), str(candidate_manifest.get("genotype_scope_id") or ""), str(candidate_manifest.get("analysis_scope_id") or ""), ) if not candidate_key[2] or not candidate_key[3]: continue try: candidate_mtime = candidate_file.stat().st_mtime except OSError: candidate_mtime = 0.0 candidate_stamp = (str(candidate_manifest.get("start_time") or ""), candidate_mtime) if candidate_stamp > population_latest_stamp.get(candidate_key, ("", -1.0)): population_latest_stamp[candidate_key] = candidate_stamp population_latest_manifest[candidate_key] = candidate_file for manifest_file, manifest in manifest_payloads: try: if not _manifest_matches_requested_scope(manifest_file, manifest): continue stage_name = _manifest_stage_name(manifest_file, manifest) if stage_name in {"wgs_align", "wgs_markdup", "wgs_bam_qc", "wgs_haplotype_calling", "wgs_gvcf_combine", "wgs_joint_genotyping", "wgs_variant_select", "wgs_variant_filter"}: from gfa_core.mas.tools.stage_identity import StageIdentity, get_stage_status_for_identity exact_status = get_stage_status_for_identity( target_job_id, StageIdentity( stage=stage_name, prefix=str(manifest.get("prefix") or ""), wgs_sample_set_id=str(manifest.get("wgs_sample_set_id") or ""), wgs_reference_id=str(manifest.get("wgs_reference_id") or ""), wgs_analysis_id=str(manifest.get("wgs_analysis_id") or ""), ), workspace_root=WORKSPACE_ROOT, ) if (exact_status.completed and exact_status.manifest_path and exact_status.manifest_path.resolve() != manifest_file.resolve()): continue if not exact_status.completed or not exact_status.manifest_path: if stage_name in stages: stages[stage_name]["status"] = exact_status.status stages[stage_name]["error"] = "; ".join(exact_status.errors) or "WGS alignment authority failed exact status validation" continue if stage_name.startswith("pop_"): population_key = ( stage_name, str(manifest.get("prefix") or ""), str(manifest.get("genotype_scope_id") or ""), str(manifest.get("analysis_scope_id") or ""), ) if population_latest_manifest.get(population_key) != manifest_file: continue from gfa_core.mas.tools.stage_identity import StageIdentity, get_stage_status_for_identity exact_status = get_stage_status_for_identity( target_job_id, StageIdentity( stage=stage_name, prefix=str(manifest.get("prefix") or ""), genotype_scope_id=str(manifest.get("genotype_scope_id") or ""), analysis_scope_id=str(manifest.get("analysis_scope_id") or ""), ), workspace_root=WORKSPACE_ROOT, ) if not exact_status.completed or not exact_status.manifest_path or exact_status.manifest_path.resolve() != manifest_file.resolve(): if stage_name in stages: stages[stage_name]["status"] = exact_status.status stages[stage_name]["error"] = "; ".join(exact_status.errors) or "Population manifest failed exact status validation" continue manifest_status = str(manifest.get("status") or "") if manifest_status not in {"success", "partial_success"}: if stage_name in stages: stages[stage_name]["status"] = "failed" stages[stage_name]["error"] = manifest.get("error") or (manifest.get("parameters") or {}).get("error") or manifest_status stages[stage_name]["manifest_status"] = manifest_status stages[stage_name]["manifest_path"] = f"/workspace/{target_job_id}/final/{manifest_file.name}" continue if (stage_name == "gwas_fmt" or stage_name in _GWAS_RESULT_STAGES or stage_name.startswith("pop_")) and not _manifest_artifact_records(manifest): stages[stage_name]["status"] = "failed" stages[stage_name]["error"] = "Scientific manifest identity or artifact integrity proof failed" continue _remember_accepted_manifest_artifacts(manifest_file, manifest) if manifest.get("stage") == "rnaseq_enrich" and stage_name == "rnaseq_enrich": continue elapsed = manifest.get("elapsed_sec", 0) if stage_name in stages and elapsed: stages[stage_name]["elapsed_time"] = round(elapsed, 1) if stage_name == "identify": hmm_models = manifest.get("hmm_models") if isinstance(manifest.get("hmm_models"), dict) else {} metrics = manifest.get("metrics") if isinstance(manifest.get("metrics"), dict) else {} stages["identify"]["hmm_audit"] = { "status": hmm_models.get("status", ""), "requested_pfams": hmm_models.get("requested_pfams") or [], "loaded_accessions": hmm_models.get("loaded_accessions") or [], "missing_requested_pfams": hmm_models.get("missing_requested_pfams") or [], "per_model_hits": hmm_models.get("per_model_hits") or {}, "hmm_members": metrics.get("hmm_members", 0), "blast_members": metrics.get("blast_members", 0), "candidate_members": metrics.get("candidate_members", 0), "final_members": metrics.get("final_members", 0), } _add_manifest_result_group(manifest_file, manifest) except Exception: pass # 处理final目录中的文件（与 vue3copy0417 一致：单一 stage_keywords + 单次 for 循环） if final_dir.exists(): for f in final_dir.iterdir(): if not f.is_file(): continue fname = f.name fname_lower = fname.lower() if fname in wgs_manifest_artifact_names and fname not in accepted_manifest_artifact_names: continue if ".gwas_" in fname_lower and fname not in accepted_manifest_files and fname not in accepted_manifest_artifact_names: # Compatibility filename scanning must never mix stale or failed # GWAS files into the manifest-authoritative result set. continue if ".pop_" in fname_lower and fname not in accepted_manifest_files and fname not in accepted_manifest_artifact_names: # Population results are manifest-authoritative; never mix sibling # prefixes, stale runs, or failed-attempt files via filename scans. continue if fname in wgs_manifest_artifact_names and fname not in accepted_manifest_artifact_names: # WGS artifacts are manifest-authoritative and require exact # sample-set and analysis identity matching above. continue if "combo_panel_test" in fname_lower: continue if active_rnaseq_scope_id and _looks_like_rnaseq_file(fname) and not _allow_rnaseq_fallback_file(fname): continue if requested_scope_id and fname not in accepted_manifest_files and fname not in accepted_manifest_artifact_names: continue if _is_hidden_intermediate_file(fname): continue is_subfamily_target_reference_msa = ( "target_plus_reference" in fname_lower and "msa_plot" in fname_lower and "target_only" not in fname_lower and ".custom_phylogeny." not in fname_lower and ".comparative_phylogeny." not in fname_lower and f.suffix.lower() in [".png", ".jpg", ".jpeg", ".svg", ".pdf", ".tex"] ) if is_subfamily_target_reference_msa: stages["reference_classification"].setdefault("msa_files", []).append({ "name": fname, "size": f.stat().st_size, "path": f"/workspace/{target_job_id}/final/{fname}" }) continue if "msa_conserved_region" in fname_lower and f.suffix.lower() in [".png", ".jpg", ".jpeg", ".svg", ".pdf", ".tex", ".tsv", ".fa", ".json"]: conserved_item = { "name": fname, "size": f.stat().st_size, "path": f"/workspace/{target_job_id}/final/{fname}" } group_items = stages["phylogeny_msa"].setdefault("msa_groups", {}).setdefault("conserved", []) if conserved_item["path"] not in {item.get("path") for item in group_items}: group_items.append(conserved_item) continue is_subfamily_intermediate_artifact = ( ( "subfamily_evidence" in fname_lower or "target_plus_reference" in fname_lower ) and ".ref-" in fname_lower and ".custom_phylogeny." not in fname_lower and ".comparative_phylogeny." not in fname_lower and any(token in fname_lower for token in (".tree", ".motif", ".domain", ".structure", ".combo", ".phylogeny", ".pep.msa")) ) if is_subfamily_intermediate_artifact: # subfamily_evidence artifacts are intermediate support for # reference classification; final user-facing figures are # exposed from the reference_classification result group. continue if ( "reference_classification.tree." in fname_lower and not any(token in fname_lower for token in ("combo", "evidence")) ): # 亚家族结果页只展示最终 combo 图和分类表；单独树图作为中间产物保留但不铺到前端。 continue if re.fullmatch(r"ko\d{5}\.(png|jpg|jpeg|svg|pdf|xml)", fname_lower): continue # 排除大文件（>100MB的原始数据文件） file_size = f.stat().st_size if file_size > 100 * 1024 * 1024: # 100MB # 跳过原始数据文件（FASTQ, BAM等） if any(x in fname.lower() for x in ['.fastq', '.fq', '.bam', '.sam', '.clean.']): continue if f.suffix.lower() not in ['.png', '.jpg', '.jpeg', '.svg', '.pdf', '.fa', '.fasta', '.json', '.md', '.id', '.nwk', '.msa', '.html', '.log', '.gz', '.zip', '.txt', '.tsv', '.csv']: continue file_info = { "name": fname, "size": file_size, "path": f"/workspace/{target_job_id}/final/{fname}" } # 注意：顺序很重要！更具体的匹配应该放在前面（写法对齐 apps/web/vue3copy0417/main.py） stage_keywords = { "prepare_simplify_ncbi": ["prepare_simplify_ncbi", ".simplified.", "_simplified_", "ncbi"], "identify_refine": ["identify_refine", "curated_members", "final_members.edited", "domain_review"], "prepare_longest_transcript": ["prepare_longest_transcript", ".longest.", "_longest_"], "prepare_normalize_gff": ["prepare_normalize_gff", ".normalized.", "_normalized_"], "prepare_extract_pep": ["prepare_extract_pep"], "prepare_extract_cds": ["prepare_extract_cds"], "sequence_extract_by_id": ["sequence_extract", "sequence_extract_by_id"], "prepare_filter_fasta": ["prepare_filter_fasta", ".filtered.", "_filtered_"], "prepare_organize": ["prepare_organize"], "genome_prepare": ["genome_prepare", "species_config"], "msa_region_plot": ["msa_region", "region.tsv", "conservation_profile"], "subfamily_confirm": ["subfamily_confirm", "confirmed_subfamily"], "rnaseq_fgsea": ["rnaseq_fgsea", "fgsea"], "rnaseq_go_enrich": ["rnaseq_go_enrich", ".go_enrich", "_go_"], "rnaseq_kegg_enrich": ["rnaseq_kegg_enrich", ".kegg_enrich", "_kegg_", "kegg_pathway", "kegg_pathview"], "interspecies_synteny": ["interspecies_synteny", "interspecies"], # 优先匹配种间共线性 "reference_classification": [ "reference_classification", "geneid.subfamily", "confirmed_subfamily", "target_plus_reference.combo", "target_plus_reference.combo_tree_motif_structure_domain", "subfamily_prediction.combo_tree_motif_structure_domain", ], "combo": ["combo"], "pep": ["pep", "pi_mw", "tmhmm", "subcellular", "等电点", "分子量", "跨膜"], "motif": ["motif"], "domain": ["domain"], "structure": ["structure"], "promoter_plot_revise": ["promoter_custom", "promoter_plot_revise"], "promoter": ["promoter", "plantcare"], "tfbs": ["tfbs", "binding-site", "binding_site", "fimo"], "synteny": [".synteny.", "collinearity", "mcscanx", "circos"], # 种内共线性，使用".synteny."避免匹配interspecies_synteny "karyotype": ["karyotype"], "phylogeny_msa": [ ".msa_plot", ".pep.msa.fa", "msa_plot_msa", ], "phylogeny": [ "phylogeny", ".pep.msa.", ".tree.nwk", ".tree.png", ".tree.pdf", ".tree.circular_tree", ], "identify": ["identify", "final_members"], "expression": [".expression.", "gene-exp", "gene-cor"], # 通用 FPKM 表达量分析 "rnaseq_qc": ["rnaseq_qc", "_fastqc", "_cutadapt", "_fastp"], # 移除 .clean.fastq 避免展示原始数据 "rnaseq_align": ["rnaseq_align", ".sorted.bam", ".bam.bai", "_hisat2.log", ".hisat2.splice_sites.ss", "splice_sites"], "rnaseq_quant": ["rnaseq_quant", ".count_matrix", "fpkm_matrix", "tpm_matrix", "gene_lengths", "normalization_info", "featurecounts", "_quant"], "rnaseq_de": ["rnaseq_de", "rnaseq_expression_plot", ".DE_results", ".DE_genes", "_MA_plot", "_volcano"], "rnaseq_sample_qc": ["rnaseq_sample_qc", ".pca", "sample_correlation", "sample_distance", "sample_clustering", "vst_matrix"], "rnaseq_wgcna": ["rnaseq_wgcna", "wgcna", "module_trait", "gene_modules", "module_sizes", "soft_threshold"], "rnaseq_candidate_family": ["rnaseq_candidate_family", "candidate_family", "候选家族"], "annotation": [".annotation.manifest", "gene2go", "gene2ko", "eggnog"], "ppi": ["ppi", "string", "network", "interaction"], # PPI蛋白互作分析 "kaks": ["kaks", "ka/ks", "ka_ks", "divergence"], # Ka/Ks分析 } target = None target_stage_name = "" for stage_name, keywords in stage_keywords.items(): if any(kw in fname_lower for kw in keywords): if not _allow_rnaseq_fallback_file(fname, stage_name): target = None target_stage_name = "" else: target = stages[stage_name] target_stage_name = stage_name break if ( target_stage_name == "expression" and active_rnaseq_scope_id and (fname in accepted_manifest_files or fname in accepted_manifest_artifact_names) and (fname_lower.endswith(".expression.manifest.json") or fname_lower.endswith(".expression_report.md")) ): target = stages["rnaseq_de"] target_stage_name = "rnaseq_de" if "tree_replot" in fname_lower: continue if f.suffix.lower() in ['.png', '.jpg', '.jpeg', '.svg']: if target: if target is stages["reference_classification"] and any(token in fname_lower for token in ["motif", "domain", "structure"]) and "combo" not in fname_lower: continue image_url = f"/workspace/{target_job_id}/final/{fname}" if image_url not in target["images"]: target["images"].append(image_url) else: for stage_name in ["genome_prepare", *PREPARE_SUBSTAGES, "identify_refine", "phylogeny", "pep", "motif", "domain", "structure", "identify", "expression", "promoter", "tfbs", "synteny", "karyotype", "interspecies_synteny", "combo", "rnaseq_qc", "rnaseq_align", "rnaseq_quant", "rnaseq_sample_qc", "rnaseq_de", "annotation", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea", "rnaseq_wgcna", "rnaseq_candidate_family"]: manifest_files = list(final_dir.glob(f"*.{stage_name}.manifest.json")) if manifest_files: stages[stage_name]["images"].append(f"/workspace/{target_job_id}/final/{fname}") break elif target: if target is stages["reference_classification"]: is_verbose_scoped_copy = ".spl.target-" in fname_lower allowed_subfamily_file = ( not is_verbose_scoped_copy and ( fname_lower.endswith(".subfamily.tsv") or fname_lower.endswith(".confidence.tsv") or fname_lower.endswith(".assignment.tsv") or fname_lower.endswith(".geneid.subfamily") or "confirmed_subfamily" in fname_lower or fname_lower.endswith(".combo.pdf") ) ) if not allowed_subfamily_file: continue if file_info["path"] not in {item.get("path") for item in target["files"]}: target["files"].append(file_info) # 处理Synteny阶段的特殊文件（在synteny/analysis/目录中） synteny_analysis_dir = workspace_dir / "synteny" / "analysis" if synteny_analysis_dir.exists(): for f in synteny_analysis_dir.iterdir(): if not f.is_file(): continue fname = f.name # 只处理图像文件 if f.suffix.lower() not in ['.png', '.jpg', '.jpeg', '.svg']: continue # 检查是否是Circos图像文件 if "circos" in fname.lower(): stages["synteny"]["images"].append(f"/workspace/{target_job_id}/synteny/analysis/{fname}") # 明确把静态 STRING PPI 图片挂到 PPI 分析结果，避免仅作为普通文件展示。 if final_dir.exists(): for ppi_img in final_dir.glob("*.ppi_network.string.png"): ppi_url = f"/workspace/{target_job_id}/final/{ppi_img.name}" if ppi_url not in stages["ppi"]["images"]: stages["ppi"]["images"].append(ppi_url) reset_root = final_dir / "ppi_reset" reset_groups = stages["ppi"].setdefault("reset_groups", []) if reset_root.exists(): for reset_dir in sorted(reset_root.glob("reset_*")): if not reset_dir.is_dir(): continue group = { "reset_id": reset_dir.name, "label": reset_dir.name, "images": [], "files": [], "metrics": {}, "manifest_path": "", } manifest_candidates = sorted(reset_dir.glob("*.ppi_reset.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True) if manifest_candidates: manifest_file = manifest_candidates[0] group["manifest_path"] = f"/workspace/{target_job_id}/final/ppi_reset/{reset_dir.name}/{manifest_file.name}" try: reset_manifest = json.loads(manifest_file.read_text(encoding="utf-8")) group["metrics"] = { "input_gene_count": reset_manifest.get("input_gene_count", 0), "deleted_count": reset_manifest.get("deleted_count", 0), "added_count": reset_manifest.get("added_count", 0), } except Exception: pass for item in sorted(reset_dir.iterdir()): if not item.is_file() or item.name.startswith("."): continue url = f"/workspace/{target_job_id}/final/ppi_reset/{reset_dir.name}/{item.name}" info = {"name": item.name, "size": item.stat().st_size, "path": url} if item.suffix.lower() in {".png", ".jpg", ".jpeg", ".svg"}: group["images"].append(url) else: group["files"].append(info) reset_groups.append(group) if final_dir.exists(): circos_group = stages["synteny"].setdefault("replot_groups", {}).setdefault("circos", []) revise_stage = stages.get("synteny_plot_revise") revise_circos_group = revise_stage.setdefault("replot_groups", {}).setdefault("circos", []) if revise_stage is not None else [] seen_circos_replots = {item.get("path") for item in circos_group if isinstance(item, dict)} seen_revise_replots = {item.get("path") for item in revise_circos_group if isinstance(item, dict)} for replot_img in sorted(final_dir.glob("*.synteny_replot*.png")) + sorted(final_dir.glob("*.synteny_replot*.svg")): url = f"/workspace/{target_job_id}/final/{replot_img.name}" info = { "name": replot_img.name, "size": replot_img.stat().st_size, "path": url, "artifact": "synteny_replot_circos", "description": "种内共线性 Circos 重绘图", } if url not in seen_circos_replots: circos_group.append(info) seen_circos_replots.add(url) if revise_stage is not None and url not in seen_revise_replots: revise_circos_group.append(info) seen_revise_replots.add(url) if replot_img.suffix.lower() in {".png", ".jpg", ".jpeg", ".svg"} and url not in revise_stage.get("images", []): revise_stage.setdefault("images", []).append(url) def _dedupe_synteny_images() -> None: images = list(dict.fromkeys(stages["synteny"].get("images", []))) if not images: return def score(url: str) -> tuple[int, int]: name = url.rsplit("/", 1)[-1].lower() in_final = "/final/" in url is_png = name.endswith(".png") is_scoped = any(token in name for token in (".target-", ".ref-", ".comp-")) is_circos = "circos" in name # Prefer the user-facing scoped final PNG; analysis-dir images are intermediates. return ( 100 if is_circos else 0, 40 if in_final else 0, 30 if is_scoped else 0, 20 if is_png else 0, ) circos_images = [u for u in images if "circos" in u.rsplit("/", 1)[-1].lower()] if circos_images: best = sorted(circos_images, key=score, reverse=True)[0] non_circos = [u for u in images if u not in circos_images] stages["synteny"]["images"] = [best] + non_circos else: stages["synteny"]["images"] = images _dedupe_synteny_images() # Subfamily prediction support artifacts are filtered above by scope # (target_plus_reference + ref-*). Do not clear the public traditional # target_only stages here; after confirming subfamilies, users still need # the normal phylogeny/motif/domain/structure chain to remain visible. requested_stage_filter = str(stage or "").strip() if requested_stage_filter: stage_aliases = { "phylogeny": {"phylogeny", "phylogeny_msa", "msa_region_plot"}, "reference_classification": {"reference_classification"}, "subfamily_confirm": {"reference_classification"}, "msa_region_plot": {"msa_region_plot"}, "promoter": {"promoter", "promoter_plot_revise"}, "promoter_plot_revise": {"promoter_plot_revise"}, "tfbs": {"tfbs"}, "synteny": {"synteny", "synteny_plot_revise"}, "synteny_plot_revise": {"synteny_plot_revise"}, "rnaseq_de": {"rnaseq_de"}, } visible_stage_names = stage_aliases.get(requested_stage_filter, {requested_stage_filter}) for stage_name, payload in list(stages.items()): if stage_name.startswith("_") or stage_name in visible_stage_names or not isinstance(payload, dict): continue payload["images"] = [] payload["files"] = [] payload["msa_files"] = [] payload["msa_groups"] = {"conserved": []} payload["replot_groups"] = {"tree": [], "circos": [], "combo": []} payload["elapsed_time"] = 0 for group_name in list(result_groups.keys()): group = result_groups.get(group_name) or {} group_stage = str(group.get("stage") or group_name or "").lower() if group_stage not in visible_stage_names: result_groups.pop(group_name, None) stages["_result_groups"] = result_groups stages["_scope"] = { "rnaseq_scope_id": active_rnaseq_scope_id, "requested_rnaseq_scope_id": requested_rnaseq_scope_id, "scope_id": requested_scope_id, } return stages def _resolve_ppi_manifest_for_reset(job_id: str, prefix: str) -> tuple[Path, dict]: final_dir = WORKSPACE_ROOT / job_id / "final" candidates = [ final_dir / f"{prefix}.ppi.manifest.json", *sorted(final_dir.glob(f"{prefix}.*.ppi.manifest.json"), key=lambda p: p.stat().st_mtime, reverse=True), ] for candidate in candidates: if not candidate.exists() or candidate.stat().st_size == 0: continue try: data = json.loads(candidate.read_text(encoding="utf-8")) except Exception: continue if str(data.get("stage") or "") in {"ppi", "ppi_reset"} and str(data.get("status") or "") == "success": return candidate, data raise HTTPException(status_code=404, detail=f"No successful PPI manifest found for {prefix}") def _resolve_ppi_gene_id_file_from_manifest(job_id: str, manifest: dict) -> Path: workspace_dir = WORKSPACE_ROOT / job_id candidates = [ ((manifest.get("parameters") or {}).get("gene_id_file") or ""), ((manifest.get("input") or {}).get("gene_ids") or ""), ] for value in candidates: if not value: continue path = Path(str(value)) if not path.is_absolute(): path = PROJECT_ROOT / path if not path.exists(): path = workspace_dir / str(value) if path.exists() and path.stat().st_size > 0: return path raise HTTPException(status_code=404, detail="PPI manifest does not reference a readable gene ID input file") @app.get("/api/ppi/reset/input") async def get_ppi_reset_input(job_id: str, prefix: str): manifest_path, manifest = _resolve_ppi_manifest_for_reset(job_id, prefix) gene_file = _resolve_ppi_gene_id_file_from_manifest(job_id, manifest) gene_ids = [] for line in gene_file.read_text(encoding="utf-8", errors="ignore").splitlines(): token = line.strip() if token and not token.startswith("#"): gene_ids.append(token) params = manifest.get("parameters") if isinstance(manifest.get("parameters"), dict) else {} input_data = manifest.get("input") if isinstance(manifest.get("input"), dict) else {} return { "success": True, "job_id": job_id, "prefix": prefix, "manifest_path": str(manifest_path), "source_gene_id_file": str(gene_file), "gene_ids": gene_ids, "rows": [{"gene_id": gene_id, "keep": True, "source": gene_file.name} for gene_id in gene_ids], "ppiSpecies": str(params.get("string_species") or params.get("ppi_species") or input_data.get("species") or "auto"), "ppiScore": int(params.get("score") or params.get("ppi_score") or 200), } @app.post("/api/ppi/species-resolution") async def resolve_ppi_species_resolution_api(req: PpiSpeciesResolutionRequest): workspace = WORKSPACE_ROOT / req.job_id if not workspace.exists(): raise HTTPException(status_code=404, detail=f"Job workspace not found: {req.job_id}") try: from apps.web.pipeline_backend import JobInfo, resolve_ppi_species_resolution job = JobInfo( job_id=req.job_id, input_dir=str(INPUT_ROOT / req.job_id), workspace_dir=str(workspace), logs_dir=str(workspace / "logs"), created_at="", ) resolution = resolve_ppi_species_resolution( job=job, prefix=req.prefix, ppi_species=req.ppiSpecies or "auto", scope_file=req.scopeFile or "", target_species=req.targetSpecies or [], reference_species=req.referenceSpecies or [], family=req.geneFamily or "", target_scientific_name=req.targetScientificName or "", ppi_target_tree_taxon_id=req.ppiTargetTreeTaxonId or "", ppi_reference_taxon_id=req.ppiReferenceTaxonId or "", ppi_reference_storage_key=req.ppiReferenceStorageKey or "", ppi_reference_selection_mode=req.ppiReferenceSelectionMode or "", ppi_reference_user_confirmed=bool(req.ppiReferenceUserConfirmed), dependency_owner=req.dependencyOwner or "", ignore_existing_ortholog=bool(req.ignoreExistingOrtholog), requested_stages=req.requestedStages or [], ) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) return { "success": True, "job_id": req.job_id, "prefix": req.prefix, "ppiScore": req.ppiScore, **resolution, } def _reference_bundle_download_payload(req: ReferenceBundleDownloadRequest) -> Dict[str, Any]: return { "job_id": req.job_id, "species_code": req.speciesCode, "species_name": req.speciesName, "taxon_id": req.taxonId, "requested_stages": req.requestedStages or [], "include_recommended": bool(req.includeRecommended), "accession": req.accession, "timeout_sec": max(30, int(req.timeoutSec or 300)), } def _reference_bundle_fallback_detail(message: str = "") -> Dict[str, Any]: return { "success": False, "async": False, "stage": "reference_bundle_download", "error": message, "can_fallback_ath": True, "fallback_species_code": "At", "fallback_string_species": "ath", "download_failed": True, "failure_kind": "reference_bundle_download_failed", } def _reference_bundle_task_status_path(task: Dict[str, Any]) -> Optional[Path]: job_id = str(task.get("job_id") or (task.get("payload") or {}).get("job_id") or "").strip() task_id = str(task.get("task_id") or task.get("id") or "").strip() if not job_id or not task_id: return None return WORKSPACE_ROOT / job_id / "final" / "reference_bundle_tasks" / f"{task_id}.json" def _persist_reference_bundle_task(task: Dict[str, Any]) -> None: if str(task.get("stage") or "") != "reference_bundle_download": return task["updated_at"] = datetime.now().isoformat() path = _reference_bundle_task_status_path(task) if not path: return try: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(task, ensure_ascii=False, indent=2, default=str), encoding="utf-8") except Exception as exc: print(f"[API DEBUG] Failed to persist reference bundle task {task.get('id')}: {exc}") def _load_reference_bundle_task(task_id: str) -> Optional[Dict[str, Any]]: pattern = f"*/final/reference_bundle_tasks/{task_id}.json" try: for path in WORKSPACE_ROOT.glob(pattern): try: data = json.loads(path.read_text(encoding="utf-8")) except Exception: continue if isinstance(data, dict) and str(data.get("id") or data.get("task_id") or "") == str(task_id): return data except Exception as exc: print(f"[API DEBUG] Failed to scan persisted reference bundle task {task_id}: {exc}") return None def _apply_reference_bundle_stale_watchdog(task: Dict[str, Any]) -> None: if str(task.get("stage") or "") != "reference_bundle_download": return status = str(task.get("status") or "").lower() if status not in {"pending", "running"}: return start_time = task.get("start_time") or task.get("created_at") age_sec = _elapsed_seconds_since(start_time) task["pending_seconds"] = round(age_sec, 1) task["status"] = "running" if status == "running" else "pending" progress = task.get("progress") if isinstance(task.get("progress"), dict) else {} progress = { "phase": progress.get("phase") or status or "running", "message": progress.get("message") or task.get("message") or "参考物种注释包仍在后台下载", "percentage": progress.get("percentage"), "indeterminate": progress.get("indeterminate", True), "pending_seconds": task["pending_seconds"], **{k: v for k, v in progress.items() if k not in {"phase", "message", "percentage", "indeterminate", "pending_seconds"}}, } task["progress"] = progress task.pop("action_required", None) task.pop("retryable", None) task.pop("recommended_action", None) if task.get("result", {}).get("download_failed"): task.pop("result", None) _persist_reference_bundle_task(task) def run_reference_bundle_download_task(task_id: str, payload: Dict[str, Any]) -> None: tasks[task_id].update({ "status": "running", "message": "正在后台下载参考物种注释包", "start_time": datetime.now().isoformat(), "timeout_sec": int(payload.get("timeout_sec") or 300), "payload": dict(payload), "progress": { "phase": "queued", "message": "参考物种注释包下载任务已启动", "percentage": 0, "indeterminate": True, }, }) _persist_reference_bundle_task(tasks[task_id]) try: from apps.web.pipeline_backend import JobInfo, download_reference_annotation_bundle job_id = payload["job_id"] workspace = WORKSPACE_ROOT / job_id job = JobInfo( job_id=job_id, input_dir=str(INPUT_ROOT / job_id), workspace_dir=str(workspace), logs_dir=str(workspace / "logs"), created_at="", ) def update_progress(event: Dict[str, Any]) -> None: progress = dict(event or {}) percentage = progress.get("percentage") if percentage is not None: try: percentage = max(0, min(100, float(percentage))) except (TypeError, ValueError): percentage = None progress["percentage"] = percentage progress.setdefault("indeterminate", percentage is None) tasks[task_id].update({ "status": "running", "message": progress.get("message") or "正在后台下载参考物种注释包", "progress": progress, }) _persist_reference_bundle_task(tasks[task_id]) result = download_reference_annotation_bundle( job=job, species_code=payload.get("species_code", ""), species_name=payload.get("species_name", ""), taxon_id=payload.get("taxon_id", ""), requested_stages=payload.get("requested_stages") or [], include_recommended=bool(payload.get("include_recommended")), accession=payload.get("accession", ""), timeout_sec=int(payload.get("timeout_sec") or 300), progress_callback=update_progress, ) success = result.get("status") == "success" tasks[task_id].update({ "status": "success" if success else "failed", "message": "参考物种注释包下载完成" if success else "参考物种注释包下载未满足必需文件", "end_time": datetime.now().isoformat(), "progress": { "phase": "completed" if success else "failed", "message": "参考物种注释包下载完成" if success else "参考物种注释包下载未满足必需文件", "percentage": 100 if success else None, "indeterminate": False if success else True, }, "result": { "success": success, "async": False, "stage": "reference_bundle_download", **result, }, }) if success: for key in ("action_required", "retryable", "recommended_action"): tasks[task_id].pop(key, None) _persist_reference_bundle_task(tasks[task_id]) except Exception as exc: message = str(exc) tasks[task_id].update({ "status": "failed", "message": message, "end_time": datetime.now().isoformat(), "action_required": True, "retryable": True, "recommended_action": "fallback_or_retry_reference_download", "result": _reference_bundle_fallback_detail(message), }) _persist_reference_bundle_task(tasks[task_id]) @app.post("/api/reference-bundle/download") async def download_reference_bundle_api(req: ReferenceBundleDownloadRequest): workspace = WORKSPACE_ROOT / req.job_id if not workspace.exists(): raise HTTPException(status_code=404, detail=f"Job workspace not found: {req.job_id}") payload = _reference_bundle_download_payload(req) if req.asyncMode: task_id = str(uuid.uuid4()) tasks[task_id] = { "id": task_id, "task_id": task_id, "stage": "reference_bundle_download", "job_id": req.job_id, "status": "pending", "message": "参考物种注释包下载任务已提交", "created_at": datetime.now().isoformat(), "species_code": req.speciesCode, "species_name": req.speciesName, "taxon_id": req.taxonId, "requested_stages": req.requestedStages or [], "timeout_sec": payload["timeout_sec"], "payload": dict(payload), } _persist_reference_bundle_task(tasks[task_id]) executor.submit(run_reference_bundle_download_task, task_id, payload) return { "success": True, "async": True, "task_id": task_id, "job_id": req.job_id, "stage": "reference_bundle_download", "species_code": req.speciesCode, "species_name": req.speciesName, "taxon_id": req.taxonId, "requested_stages": req.requestedStages or [], } try: from apps.web.pipeline_backend import JobInfo, download_reference_annotation_bundle job = JobInfo( job_id=req.job_id, input_dir=str(INPUT_ROOT / req.job_id), workspace_dir=str(workspace), logs_dir=str(workspace / "logs"), created_at="", ) result = download_reference_annotation_bundle( job=job, species_code=payload.get("species_code", ""), species_name=payload.get("species_name", ""), taxon_id=payload.get("taxon_id", ""), requested_stages=payload.get("requested_stages") or [], include_recommended=bool(payload.get("include_recommended")), accession=payload.get("accession", ""), timeout_sec=int(payload.get("timeout_sec") or 300), ) except Exception as exc: raise HTTPException(status_code=500, detail={ "message": str(exc), "can_fallback_ath": True, "fallback_species_code": "At", "fallback_string_species": "ath", "download_failed": True, "failure_kind": "reference_bundle_download_failed", }) return { "success": result.get("status") == "success", **result, } @app.post("/api/ppi/reset/submit") async def submit_ppi_reset(req: PpiResetSubmitRequest): workspace = WORKSPACE_ROOT / req.job_id if not workspace.exists(): raise HTTPException(status_code=404, detail=f"Job workspace not found: {req.job_id}") manifest_path, manifest = _resolve_ppi_manifest_for_reset(req.job_id, req.prefix) if req.sourceGeneIdFile: source_gene_id_file = Path(req.sourceGeneIdFile) else: source_gene_id_file = _resolve_ppi_gene_id_file_from_manifest(req.job_id, manifest) if not source_gene_id_file.is_absolute(): source_gene_id_file = PROJECT_ROOT / source_gene_id_file if not source_gene_id_file.exists(): raise HTTPException(status_code=404, detail=f"Source gene ID file not found: {source_gene_id_file}") clean_ids = [] seen = set() for item in req.gene_ids: token = str(item or "").strip() if token and token not in seen: seen.add(token) clean_ids.append(token) if not clean_ids: raise HTTPException(status_code=400, detail="PPI reset requires at least one gene ID") try: from apps.web.pipeline_backend import JobInfo, run_ppi_reset_analysis job = JobInfo( job_id=req.job_id, input_dir=str(INPUT_ROOT / req.job_id), workspace_dir=str(workspace), logs_dir=str(workspace / "logs"), created_at="", ) result = run_ppi_reset_analysis( job=job, prefix=req.prefix, source_gene_id_file=source_gene_id_file, submitted_gene_ids=clean_ids, instruction=req.instruction, ppi_species=req.ppiSpecies or "auto", ppi_score=req.ppiScore or 200, ) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) return { "success": bool(result.get("success")), "job_id": req.job_id, "prefix": req.prefix, "source_manifest": str(manifest_path), "reset_id": result.get("reset_id"), "reset_dir": str(result.get("reset_dir")), "manifest_path": str(result.get("manifest_file")), "log_file": str(result.get("log_file")), "exit_code": result.get("exit_code"), } def _promoter_big7_category_py(element: str, description: str = "") -> str: text = f"{element or ''} {description or ''}" rules = [ ("Core promoter", r"TATA|CAAT|Inr|Y-patch"), ("Light response", r"G-box|GT1|I-box|Sp1|Box 4|Box-4|Box.4|GATA|light"), ("Hormone signaling", r"ABRE|P-box|GARE|CGTCA|TGACG|TCA|as-1|TGA|AuxRE|ERE|ABA|gibberellin|jasmon|salicylic|auxin|ethylene"), ("Stress response", r"DRE|LTR|ARE|STRE|W.box|W-box|stress|drought|cold|anaerobic"), ("TF binding", r"MYB|MYC|WRKY"), ("Development", r"RY|AAGAA|CAT|GCN4|development|meristem|endosperm"), ] for label, pattern in rules: if re.search(pattern, text, flags=re.IGNORECASE): return label return "Other" def _resolve_promoter_filtered_table_for_api(job_id: str, prefix: str = "") -> Optional[Path]: workspace_dir = WORKSPACE_ROOT / job_id final_dir = workspace_dir / "final" if final_dir.exists(): manifest_patterns = [f"{prefix}*.promoter.manifest.json"] if prefix else ["*.promoter.manifest.json"] for pattern in manifest_patterns: for manifest_file in sorted(final_dir.glob(pattern), key=lambda p: p.stat().st_mtime, reverse=True): try: manifest = json.loads(manifest_file.read_text(encoding="utf-8")) except Exception: continue artifacts = manifest.get("artifacts") if isinstance(manifest.get("artifacts"), dict) else {} item = artifacts.get("filtered_plantcare_output") or artifacts.get("plantcare_output") if isinstance(item, dict): raw = item.get("full_path") or item.get("path") or item.get("relpath_or_hint") if raw: path = Path(str(raw)).expanduser() if not path.is_absolute(): path = workspace_dir / path if path.exists(): return path fallback = workspace_dir / "promoter" / "analysis" / "plantCARE_output_filtered.tab" return fallback if fallback.exists() else None @app.get("/api/promoter-elements") async def get_promoter_elements(job_id: str = "", prefix: str = ""): target_job_id = job_id or current_job_id if not target_job_id: raise HTTPException(status_code=400, detail="job_id is required") table = _resolve_promoter_filtered_table_for_api(target_job_id, prefix) if not table: return {"success": False, "message": "未找到已有 promoter filtered 表，请先完成启动子分析。", "elements": [], "categories": []} element_counts: Dict[str, int] = {} category_counts: Dict[str, int] = {} total_records = 0 try: with table.open("r", encoding="utf-8", errors="ignore") as fh: for line in fh: parts = line.rstrip("\n").split("\t") if len(parts) < 2: continue element = parts[1].strip() if not element: continue description = parts[7].strip() if len(parts) >= 8 else "" category = _promoter_big7_category_py(element, description) element_counts[element] = element_counts.get(element, 0) + 1 category_counts[category] = category_counts.get(category, 0) + 1 total_records += 1 except Exception as exc: raise HTTPException(status_code=500, detail=f"读取启动子元件失败: {exc}") categories_order = ["Core promoter", "Light response", "Hormone signaling", "Stress response", "TF binding", "Development", "Other"] return { "success": True, "job_id": target_job_id, "prefix": prefix, "source": str(table), "total_records": total_records, "elements": [{"element": k, "count": v} for k, v in sorted(element_counts.items(), key=lambda item: (-item[1], item[0]))], "categories": [{"category": k, "count": category_counts.get(k, 0)} for k in categories_order if category_counts.get(k, 0) > 0], } @app.get("/api/info") async def get_info(): """获取当前任务信息""" global current_job_id return { "job_id": current_job_id, "working_dir": str(WORKSPACE_ROOT / current_job_id) if current_job_id else "", "input_dir": str(INPUT_ROOT / current_job_id) if current_job_id else "", "output_dir": str(WORKSPACE_ROOT / current_job_id / "final") if current_job_id else "", "workspace_root": str(WORKSPACE_ROOT), "input_root": str(INPUT_ROOT) } @app.get("/workspace/{job_id}/{path:path}") async def serve_workspace_file(job_id: str, path: str): """提供工作目录文件访问""" file_path = WORKSPACE_ROOT / job_id / path if file_path.exists() and file_path.is_file(): return FileResponse(file_path) raise HTTPException(status_code=404, detail="File not found") @app.get("/api/mcscanx_html/{job_id}") async def get_mcscanx_html_files(job_id: str): """获取MCScanX HTML结果文件夹中的文件列表""" html_dir = WORKSPACE_ROOT / job_id / "synteny" / "analysis" / "Ft.html" if not html_dir.exists(): return {"exists": False, "files": [], "message": "HTML文件夹不存在"} files = [] for f in sorted(html_dir.iterdir()): if f.is_file() and f.suffix.lower() == '.html': files.append({ "name": f.name, "size": f.stat().st_size, "path": f"/workspace/{job_id}/synteny/analysis/Ft.html/{f.name}" }) return { "exists": True, "folder_path": str(html_dir), "files": files, "count": len(files) } class ChatRequest(BaseModel): message: str def _normalize_extension_match_text(value: str) -> str: return re.sub(r"[^a-z0-9]+", "", str(value or "").lower()) def _extract_extension_input_paths(message: str) -> list[str]: paths: list[str] = [] normalized = re.sub( r"([A-Za-z0-9_.-]+)\s+(\.(?:pep\.fasta|pep\.fa|cds\.fasta|cds\.fa|fasta|faa|fa|pep|hmm|geneids|geneid|geneld|ids|id|txt|tsv|csv)\b)", r"\1\2", message or "", flags=re.IGNORECASE, ) for raw in re.findall(r"(?:/[^\s,，;；]+|[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]+|[A-Za-z0-9_.-]+\.(?:pep\.fasta|pep\.fa|cds\.fasta|cds\.fa|fasta|faa|fa|pep|hmm|geneids|geneid|geneld|ids|id|txt|tsv|csv))", normalized, flags=re.IGNORECASE): token = raw.strip().strip("\'\"，,;；。") if token and token not in paths: paths.append(token) return paths def _extract_explicit_extension_target(text: str) -> str: value = str(text or "") patterns = [ r"【\s*(?:拓展分析阶段|扩展分析阶段|拓展分析|扩展分析|extension(?:[_\s-]?stage)?)\s*[：:]\s*([^】]+?)\s*】", r"(?:拓展分析阶段|扩展分析阶段|拓展分析|扩展分析|extension(?:[_\s-]?stage)?)\s*[：:]\s*([A-Za-z0-9_.-]{3,})", ] for pattern in patterns: match = re.search(pattern, value, flags=re.I) if not match: continue target = str(match.group(1) or "").strip().strip("[]()（）【】'\"，,;；。") target = re.split(r"[\s，,;；。]+", target, maxsplit=1)[0].strip() if target: return target return "" def _extension_identity_matches_exact_extension_id(ext: dict[str, Any], target: str) -> bool: normalized_target = _normalize_extension_match_text(target) normalized_extension_id = _normalize_extension_match_text(str(ext.get("extension_id") or "")) return bool(normalized_target and normalized_target == normalized_extension_id) def _extension_alias_tokens(ext: dict[str, Any]) -> set[str]: tokens: set[str] = set() fields = [ext.get("extension_id", ""), ext.get("name", ""), ext.get("display_name", ""), ext.get("description", "")] fields.extend(ext.get("nl_triggers") or []) fields.extend((ext.get("parameters") or {}).keys()) generic_tokens = {"seq", "ext", "cds", "pep", "rna", "dna", "run", "api", "v1", "v2", "v3"} for raw in fields: for word in re.split(r"[^A-Za-z0-9]+", str(raw or "").lower()): if len(word) >= 3 and word not in generic_tokens: tokens.add(word) if word.endswith("p") and len(word) > 4: tokens.add(word[:-1]) requirements = ext.get("requirements") or {} for tool in requirements.get("system") or []: word = str(tool or "").lower() if len(word) >= 3: tokens.add(word) if word.endswith("p") and len(word) > 4: tokens.add(word[:-1]) return tokens def _extract_extension_parameters(text: str, ext: dict[str, Any]) -> dict[str, Any]: params: dict[str, Any] = {} known = {str(k).lower(): k for k in ((ext.get("parameters") or {}).keys())} for match in re.finditer(r"([A-Za-z_][A-Za-z0-9_-]*)\s*(?:=|:|：)\s*([^\s,，;；]+)", text or ""): key = match.group(1).replace("-", "_").lower() if key in known: params[known[key]] = match.group(2).strip("'\"") return params def _extension_stage_plan_response(extension_intent: dict[str, Any], session_id: str, job_id: str = "") -> dict[str, Any]: params = extension_intent.get("params") or {} display_name = params.get("extensionName") or params.get("extension_id") or "拓展分析阶段" detail = { "id": "extension_stage", "name": display_name, "description": f"执行拓展分析阶段：{display_name}", "dependencies": [], "can_parallel": False, "estimated_time": "5分钟", "params": params, } plan = { "analysis_type": "extension_stage", "description": f"执行拓展分析阶段：{display_name}", "estimated_time": "5分钟", "estimated_total_time": "5分钟", "execution_strategy": "single", "max_parallel": 1, "total_stages": 1, "stages": ["extension_stage"], "stages_detail": [detail], "all_stages": ["extension_stage"], "all_stages_detail": [detail], "parallel_groups": [], "params": params, "prefix": "", } return { "success": True, "mode": "extension_stage", "plan": plan, "intent": extension_intent, "completed_stages": [], "session_id": session_id, "job_id": job_id, } def _match_extension_stage_intent(message: str, job_id: str = "") -> dict[str, Any]: text = str(message or "") lower = text.lower() rnaseq_markers = ["rnaseq", "rna-seq", "rna seq", "转录组", "fastq"] if any(marker in lower for marker in rnaseq_markers) and not any(token in lower for token in ["拓展分析", "扩展分析", "extension"]): return {} explicit_extension_trigger = any(token in lower for token in ["拓展分析", "扩展分析", "extension"]) trigger = explicit_extension_trigger or any(token in lower for token in ["hmmsearch", "hmmer", "spl hmm", "sphmm", "blast", "blastp"]) normalized_query = _normalize_extension_match_text(text) explicit_extension_target = _extract_explicit_extension_target(text) if explicit_extension_trigger else "" explicit_target_locked = False try: all_extensions = extension_stage_agent.list_extensions(include_drafts=True) for legacy in all_extensions: if legacy.get("status") != "archived": continue legacy_names = [legacy.get("extension_id", ""), legacy.get("name", ""), legacy.get("display_name", "")] if any(_normalize_extension_match_text(name) and _normalize_extension_match_text(name) in normalized_query for name in legacy_names): return {} explicit_candidates = [ext for ext in all_extensions if ext.get("status") != "archived"] if explicit_extension_target: exact_id_candidates = [ ext for ext in explicit_candidates if _extension_identity_matches_exact_extension_id(ext, explicit_extension_target) ] if exact_id_candidates: candidates = exact_id_candidates trigger = True explicit_target_locked = True else: candidates = extension_stage_agent.list_extensions(include_drafts=False) else: candidates = extension_stage_agent.list_extensions(include_drafts=False) except Exception as exc: print(f"[ExtensionStage NL] list skipped: {exc}") return {} if not candidates: return {} best: dict[str, Any] | None = candidates[0] if explicit_target_locked and len(candidates) == 1 else None best_score = 1000 if best is not None else 0 for ext in ([] if best is not None else candidates): score = 0 names = [ext.get("extension_id", ""), ext.get("name", ""), ext.get("display_name", "")] for name in names: normalized_name = _normalize_extension_match_text(name) if normalized_name and normalized_name in normalized_query: score += 8 if name == ext.get("display_name") else 6 requirements = ext.get("requirements") or {} for tool in requirements.get("system") or []: tool_text = str(tool).lower() if tool_text in lower: score += 4 if tool_text.endswith("p") and len(tool_text) > 4 and tool_text[:-1] in lower: score += 3 for trigger_word in ext.get("nl_triggers") or []: if str(trigger_word).strip() and str(trigger_word).lower() in lower: score += 8 alias_tokens = _extension_alias_tokens(ext) for word in alias_tokens: if len(word) >= 3 and word in lower: score += 2 analysis_type = str(ext.get("analysis_type") or "") ext_identity = " ".join(str(ext.get(k, "")).lower() for k in ("extension_id", "name", "display_name", "description")) has_hmm_input = bool(re.search(r"\.hmm\b", lower)) if (has_hmm_input or any(token in lower for token in ["hmmer", "hmmsearch", "结构域重建", "结构域", "domain rebuild"])) and analysis_type == "hmmer_search": score += 16 trigger = True if any(token in lower for token in ["蛋白序列", "protein sequence"]) and analysis_type == "sequence_extraction" and any(token in ext_identity for token in ["protein", "pep"]): score += 10 trigger = True if any(token in lower for token in ["蛋白", "protein", "pep"]) and not has_hmm_input and analysis_type == "sequence_extraction" and any(token in ext_identity for token in ["protein", "pep"]): score += 6 trigger = True if any(token in lower for token in ["cds", "编码序列"]) and analysis_type == "sequence_extraction" and "cds" in ext_identity: score += 10 trigger = True if any(token in lower for token in ["提取", "extract", "extraction"]) and analysis_type == "sequence_extraction": score += 4 trigger = True if "blast" in lower and "blast" in alias_tokens: score += 6 if trigger and len(candidates) == 1: score += 3 if score > best_score: best_score = score best = ext if not trigger: return {} if not best or best_score <= 0: return {} slots = resolve_extension_slots(text, best) input_paths = slots.get("input_paths") or _extract_extension_input_paths(text) runtime_parameters = _extract_extension_parameters(text, best) runtime_parameters.update(slots.get("parameters") or {}) if slots.get("file_roles"): runtime_parameters["file_roles"] = slots.get("file_roles") target_species = slots.get("targetSpecies") or (_extract_species_codes_from_text_for_job(job_id, text) if job_id else _extract_species_codes_from_text(text)) reference_species = slots.get("referenceSpecies") or (_reference_species_from_config(job_id, exclude=target_species) if job_id and target_species else []) display_name = best.get("display_name") or best.get("name") or best.get("extension_id") return { "stage": "extension_stage", "stages": ["extension_stage"], "primary_stage": "extension_stage", "params": { "extension_id": best.get("extension_id"), "extensionName": display_name, "archetype_id": best.get("archetype_id", ""), "template_gate_status": best.get("template_gate_status", ""), "input_paths": input_paths, "parameters": runtime_parameters, "targetSpecies": target_species, "referenceSpecies": reference_species, "file_roles": slots.get("file_roles") or {}, "intent_slots": slots, }, "prefix": "", "gene_family": "", "species": "", "task_kind": "extension_stage", "taskType": "extension_stage", "message": f"我理解你想执行拓展分析阶段：{display_name}。输入将使用当前 job 的 input/workspace 文件。", "confidence": min(0.99, 0.75 + best_score / 40), "job_id": job_id, "workspace": str(PROJECT_ROOT / "workspace" / job_id) if job_id else "", } def parse_natural_language(message: str) -> dict: """解析自然语言，提取分析意图和参数 支持识别多个阶段，例如："帮我执行基因家族鉴定和绘制进化树" 会识别出 ['identify', 'phylogeny'] 两个阶段 """ message_lower = message.lower() result = {"success": False, "intent": None, "message": ""} try: from gfa_core.mas.utils.intent_parser import ( parse_msa_region_plot_request, parse_ncbi_fetch_request, parse_promoter_plot_revise_request, parse_sequence_extract_request, ) ncbi_params = parse_ncbi_fetch_request(message) sequence_extract_params = {} if ncbi_params else parse_sequence_extract_request(message) msa_region_params = parse_msa_region_plot_request(message) if not ncbi_params and not sequence_extract_params else {} promoter_plot_params = parse_promoter_plot_revise_request(message) if not ncbi_params and not sequence_extract_params and not msa_region_params else {} except Exception: sequence_extract_params = {} ncbi_params = {} msa_region_params = {} promoter_plot_params = {} if ncbi_params: prefix = ncbi_params.get("prefix", "") intent = { "stage": "ncbi_fetch_sequences", "stages": ["ncbi_fetch_sequences"], "params": {"prefix": prefix, "cpu": 4, **ncbi_params}, "prefix": prefix, "gene_family": "NCBI.fetch", "species": ncbi_params.get("ncbiSpecies") or ncbi_params.get("ncbiTaxid") or "", "task_kind": "utility", "taskType": "utility", "utility_tool": "ncbi_fetch_sequences", "message": ( f"我理解你想从 NCBI 下载序列；输入为 " f"{ncbi_params.get('ncbiGeneIds') or ncbi_params.get('ncbiQuery') or '未指定'}。" ), } result["success"] = True result["intent"] = intent return result if sequence_extract_params: prefix = sequence_extract_params.get("prefix", "") intent = { "stage": "sequence_extract_by_id", "stages": ["sequence_extract_by_id"], "params": {"prefix": prefix, "cpu": 4, **sequence_extract_params}, "prefix": prefix, "gene_family": "Sequence.extract", "species": "", "task_kind": "utility", "taskType": "utility", "utility_tool": "fasta_header_clean" if sequence_extract_params.get("sequenceExtractCleanHeaders") else "sequence_extract_by_id", "message": ( f"我理解你想清理FASTA序列ID/header；FASTA来源为 " f"{sequence_extract_params.get('sequenceExtractFasta') or '自动从当前job查找FASTA'}。" if sequence_extract_params.get("sequenceExtractCleanHeaders") else f"我理解你想按ID从本地FASTA提取序列；ID来源为 " f"{sequence_extract_params.get('sequenceExtractIdsFile') or sequence_extract_params.get('sequenceExtractIds') or '未指定'}。" ), } result["success"] = True result["intent"] = intent return result if msa_region_params: prefix = msa_region_params.get("prefix", "") if msa_region_params.get("msaAutoConserved"): message_text = f"我理解你想自动寻找 MSA 保守窗口并绘图；窗口长度为 {msa_region_params.get('msaWindowLen', 60)}。" elif msa_region_params.get("msaStartCol") and msa_region_params.get("msaEndCol"): message_text = f"我理解你想对 MSA alignment 第 {msa_region_params.get('msaStartCol')}-{msa_region_params.get('msaEndCol')} 列进行局部绘图。" else: message_text = "我理解你想对 MSA 指定区间进行局部绘图；请在确认弹窗中填写 alignment 起止列。" intent = { "stage": "msa_region_plot", "stages": ["msa_region_plot"], "params": {"prefix": prefix, "cpu": 4, **msa_region_params}, "prefix": prefix, "gene_family": "MSA.region", "species": "", "task_kind": "utility", "taskType": "utility", "utility_tool": "msa_region_plot", "message": message_text, } result["success"] = True result["intent"] = intent return result if promoter_plot_params: prefix = promoter_plot_params.get("prefix", "") intent = { "stage": "promoter_plot_revise", "stages": ["promoter_plot_revise"], "params": {"prefix": prefix, "cpu": 4, **promoter_plot_params}, "prefix": prefix, "gene_family": "Promoter.custom", "species": "", "task_kind": "utility", "taskType": "utility", "utility_tool": "promoter_plot_revise", "message": "我理解你想基于已有 PlantCARE/promoter 结果进行启动子自定义筛选绘图，不重新跑启动子预测。", } result["success"] = True result["intent"] = intent return result try: from gfa_core.mas.utils.intent_parser import parse_intent_from_query parsed_intent = parse_intent_from_query(message) except Exception: parsed_intent = {} if parsed_intent.get("stages") and parsed_intent.get("task_kind") in {"utility", "repair", "pipeline"} and not parsed_intent.get("need_clarify"): stage = parsed_intent["stages"][0] inferred_prefix = parsed_intent.get("prefix", "") if not inferred_prefix and parsed_intent.get("task_kind") == "pipeline": species = parsed_intent.get("species", "") family = parsed_intent.get("gene_family", "") if species and family and family not in {"Sequence.extract", "NCBI.fetch"}: inferred_prefix = f"{species}.{family}" params = { "prefix": inferred_prefix, "cpu": 4 if parsed_intent.get("task_kind") in {"utility", "repair"} else 8, } params.update(parsed_intent.get("params") or {}) intent = { "stage": stage, "stages": parsed_intent["stages"], "params": params, "prefix": params.get("prefix", ""), "gene_family": parsed_intent.get("gene_family", ""), "species": parsed_intent.get("species", ""), "target_species": parsed_intent.get("target_species", []), "reference_species": parsed_intent.get("reference_species", []), "comparative_species": parsed_intent.get("comparative_species", []), "scopeMode": parsed_intent.get("scopeMode", ""), "task_kind": parsed_intent.get("task_kind"), "taskType": parsed_intent.get("task_kind"), "utility_tool": parsed_intent.get("utility_tool", ""), "message": parsed_intent.get("message") or f"我理解你想执行 {stage}。", } _apply_scope_selection_to_intent(intent, message) result["success"] = True result["intent"] = intent return result stage_keywords = { "msa_region_plot": ["msa区间", "msa绘图", "msa二次绘图", "多序列比对区间", "比对区间绘图", "保守区间绘图", "保守位置绘图", "alignment region", "msa region"], "identify": ["鉴定", "识别", "找到", "发现", "identify", "find", "discover"], "phylogeny": ["进化树", "系统发育", "phylogeny", "phylogenetic", "tree", "建树"], "reference_classification": ["亚家族分类", "参考分类", "参考亚家族", "subfamily", "reference classification", "reference_classification"], "pep": ["pep", "等电点", "分子量", "跨膜", "亚细胞", "理化性质", "pi_mw", "tmhmm", "subcellular"], "motif": ["motif", "保守", "元件", "motif"], "domain": ["domain", "结构域", "domain"], "structure": ["structure", "基因结构", "结构", "外显子", "intron"], "promoter_plot_revise": ["promoter_plot_revise", "启动子diy", "启动子 diy", "启动子自定义", "启动子自定义绘图", "启动子重绘", "启动子二次绘图", "plantcare重绘", "promoter custom", "promoter replot", "diy绘图"], "tfbs": ["tfbs", "fimo", "tf结合", "tf 结合", "转录因子结合", "转录因子结合位点", "启动子结合位点", "结合位点", "binding site", "binding-site", "binding_site", "tf motif", "motif binding"], "promoter": ["promoter", "顺式", "启动子", "元件预测", "plantcare"], "synteny": ["synteny", "共线", "circos", "mcscanx", "microsynteny"], "karyotype": ["karyotype", "染色体", "核型", "karyotype"], "expression": ["expression", "表达", "geo", "表达量"], "ppi": ["ppi", "蛋白互作", "string", "interaction"], "kaks": ["kaks", "ka/ks", "选择压力", "kaks"], "rnaseq_qc": ["rnaseq质控", "rna质控", "fastqc", "质控"], "rnaseq_align": ["rnaseq比对", "rna比对", "hisat2", "比对"], "rnaseq_quant": ["rnaseq定量", "rna定量", "stringtie", "定量"], "rnaseq_sample_qc": ["样本质量", "样本检测", "sample qc", "sample quality", "pca", "样本相关性"], "rnaseq_wgcna": ["wgcna", "共表达", "加权共表达", "模块性状", "module trait"], "rnaseq_de": ["rnaseq差异", "rna差异", "差异表达", "de分析"], "rnaseq_fgsea": ["fgsea", "gsea", "基因集富集", "排序富集"], "rnaseq_candidate_family": ["候选家族", "候选基因家族", "家族排名", "tf注释", "tf family", "tffamily", "pfam注释", "deg注释", "差异基因注释"], "annotation": ["背景注释", "功能注释", "基因组注释", "自动注释", "eggnog", "annotation"], "rnaseq_go_enrich": [ "rnaseq go富集", "rnaseq_go富集", "rna go富集", "go富集", "go分析", "rnaseq富集", "rna富集", "富集分析", "enrichment", ], "rnaseq_kegg_enrich": [ "rnaseq kegg", "rnaseq_kegg", "rna kegg", "kegg富集", "kegg分析", "rnaseq富集", "rna富集", "富集分析", "enrichment", ], "all": ["完整", "全部", "all", "full", "整个", "全套", "所有"] } import re prefix_match = re.search(r'[A-Za-z]+\.?[A-Za-z]*[A-Z][A-Za-z]*', message) prefix = prefix_match.group(0) if prefix_match else "" if prefix.upper() in {"GO", "KEGG", "RNA", "RNASEQ", "GSEA", "FGSEA", "WGCNA", "QC", "DE"}: prefix = "" # 收集所有匹配的阶段（不再用break，支持多阶段） detected_stages = [] for stage, keywords in stage_keywords.items(): if any(kw in message_lower for kw in keywords): detected_stages.append(stage) if "tfbs" in detected_stages and "promoter" in detected_stages: detected_stages = [stage for stage in detected_stages if stage != "promoter"] rnaseq_detected = any(stage.startswith("rnaseq") for stage in detected_stages) if rnaseq_detected: detected_stages = [stage for stage in detected_stages if stage != "expression"] if "rnaseq_go_enrich" in detected_stages or "rnaseq_kegg_enrich" in detected_stages: detected_stages = [stage for stage in detected_stages if stage != "rnaseq_enrich"] if "go" in message_lower and "kegg" not in message_lower: detected_stages = [stage for stage in detected_stages if stage != "rnaseq_kegg_enrich"] elif "kegg" in message_lower and "go" not in message_lower: detected_stages = [stage for stage in detected_stages if stage != "rnaseq_go_enrich"] # 去重并保持顺序 seen = set() unique_stages = [] for stage in detected_stages: if stage not in seen: seen.add(stage) unique_stages.append(stage) # 处理 "all" 关键词：展开为完整基因家族链路，但 Expression 仍需明确点名。 if "all" in unique_stages: full_gene_family_stages = [ "identify", "phylogeny", "motif", "domain", "structure", "combo", "promoter", "tfbs", "synteny", "interspecies_synteny", "karyotype", "pep", "ortholog", "kaks", "ppi", ] explicit_extra_stages = [stage for stage in unique_stages if stage not in {"all", *full_gene_family_stages}] unique_stages = [*full_gene_family_stages, *explicit_extra_stages] if not unique_stages: result["message"] = "抱歉，我没有理解你想做什么分析。请说明要执行的分析阶段（鉴定、进化树、Motif、结构域、基因结构等）。" return result # 单阶段：保持兼容 if len(unique_stages) == 1: intent = { "stage": unique_stages[0], "params": {"prefix": prefix, "cpu": 8}, "message": f"我理解你想执行 {unique_stages[0]} 分析，基因家族为 {prefix}。" } else: # 多阶段：返回阶段列表 intent = { "stages": unique_stages, "stage": unique_stages[0], # 保持兼容，主阶段为第一个 "params": {"prefix": prefix, "cpu": 8}, "message": f"我理解你想执行 {', '.join(unique_stages)} 分析，基因家族为 {prefix}。" } result["success"] = True result["intent"] = intent return result @app.post("/api/chat") async def chat(request: ChatRequest): """自然语言处理接口 (兼容旧版)""" try: result = parse_natural_language(request.message) return result except Exception as e: return {"success": False, "message": f"处理失败: {str(e)}"} class AgentChatRequest(BaseModel): message: str model: str = "kimi" # kimi, glm, minimax session_id: str = "" job_id: str = "" # 当前 job_id，用于查询已完成 stages history: list = [] methodology_sandbox_registry_cache: Dict[str, str] = {} def _looks_like_methodology_sandbox_qa(message: str) -> bool: text = str(message or "").strip() lower = text.lower() if not text: return False # Methodology sandbox is a paper/methodology-specific temporary KB. # Generic questions such as "如何开展基因家族分析" or # "有哪些物种开展过NAC基因家族分析" must go to the normal QA KB. explicit_source_terms = [ "方法学", "方法部分", "文献方法", "复现流程", "文献复现", "复现方案", "这篇文章", "这篇论文", "这篇文献", "该文章", "该论文", "该文献", "临时库", "方法学库", "sandbox", "methodology_batch", "methodology sandbox", "methodology", "methods section", "paper method", "paper methodology", ] paper_id_like = bool(re.search(r"\b(?:Si|Ft|At|Os|Cs|St|Ta|Zm|Gm|Sl|Lc|Ls)[._-](?:HDZIP|HD[-_ ]?ZIP|SPL|NAC|WRKY|MYB|bZIP|bHLH|AP2|ERF|TCP|GRAS)\b", text, flags=re.I)) generic_qa_terms = [ "有哪些物种", "哪些物种", "什么物种", "多少物种", "开展过", "如何开展基因家族分析", "基因家族分析怎么做", "基因家族分析有哪些步骤", ] if any(term in lower or term in text for term in generic_qa_terms) and not any(term in lower or term in text for term in explicit_source_terms): return False execution_terms = ["执行", "运行", "开始跑", "帮我跑", "确认并执行", "run ", "execute"] asks_execution = any(term in lower or term in text for term in execution_terms) has_explicit_source = any(term in lower or term in text for term in explicit_source_terms) or paper_id_like if not has_explicit_source: return False if asks_execution and not any(term in lower or term in text for term in ["方法学", "文献", "论文", "文章", "复现", "methodology", "paper"]): return False return True def _methodology_rag_base() -> str: return os.environ.get("METHODOLOGY_RAG_API", "http://127.0.0.1:8000/api/v1").rstrip("/") def _methodology_rag_context() -> str: rag_root = Path(os.environ.get("PLANTAI_RAG_ROOT", str(PLANTAI_RAG_ROOT))).expanduser() root_status = "exists" if rag_root.exists() else "missing" return ( f"METHODOLOGY_RAG_API={_methodology_rag_base()}, " f"PLANTAI_RAG_ROOT={rag_root} ({root_status}). " "请确认 gene-family-mas-v2 主 API 已用 `python -m uvicorn api.main:app --host 0.0.0.0 --port 8000` 启动。" ) def _methodology_rag_payload(response) -> Dict[str, Any]: try: payload = response.json() except ValueError as exc: preview = (getattr(response, "text", "") or "").strip()[:500] raise RuntimeError( f"RAG service returned non-JSON response " f"(HTTP {response.status_code}): {preview or ''}; {_methodology_rag_context()}" ) from exc if response.status_code >= 400: detail = payload.get("detail") if isinstance(payload, dict) else payload raise RuntimeError(f"RAG service returned HTTP {response.status_code}: {detail}; {_methodology_rag_context()}") if not isinstance(payload, dict): raise RuntimeError(f"RAG service returned a non-object JSON payload; {_methodology_rag_context()}") return payload def _methodology_rag_request(method: str, path: str, timeout: float = 30, **kwargs) -> Dict[str, Any]: import requests endpoint = f"{_methodology_rag_base()}/{path.strip('/')}" try: response = requests.request(method.upper(), endpoint, timeout=timeout, **kwargs) except requests.RequestException as exc: raise RuntimeError(f"RAG service request failed: {exc}; endpoint={endpoint}; {_methodology_rag_context()}") from exc return _methodology_rag_payload(response) def _extract_methodology_dir_from_text(message: str) -> str: for match in re.findall(r"(?:/[^\s,，;；]+|[A-Za-z0-9_./-]*methodology_batch[A-Za-z0-9_./-]*)", message or ""): token = match.strip().strip("'\"，,;；。") if "methodology_batch" in token: return token return "" def _sanitize_methodology_run_id(value: str) -> str: token = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(value or "")).strip("._-") return token or "methodology_qa" def _methodology_candidate_score(message: str, item: dict[str, Any]) -> int: haystack = " ".join(str(item.get(key, "")) for key in ("name", "path", "paper_id", "family", "source_filename", "output_dir", "batch_dir")).lower() query = str(message or "").lower() score = 0 if any(term in query for term in ["fagopyrum", "tataricum", "buckwheat", "苦荞", "ft"]): if any(term in haystack for term in ["ft_spl", "fagopyrum", "tataricum", "buckwheat", "苦荞"]): score += 12 if re.search(r"\bspl\b", query): if "spl" in haystack: score += 8 if any(term in query for term in ["fagopyrum", "tataricum", "buckwheat", "苦荞", "ft"]) and re.search(r"\bspl\b", query): if "ft_spl" in haystack or "methodology_batch_ft_spl" in haystack: score += 20 if any(term in query for term in ["pecan", "carya", "illinoinensis", "山核桃"]): if any(term in haystack for term in ["pecan", "carya", "illinoinensis"]): score += 12 if any(term in query for term in ["sunflower", "helianthus", "向日葵"]): if any(term in haystack for term in ["sunflower", "sunflowers", "helianthus"]): score += 12 if any(term in query for term in ["sesamum", "sesame", "芝麻", "sihdzip", "hdzip", "hd-zip"]): if any(term in haystack for term in ["sesamum", "sesame", "sihdzip", "hdzip", "hd-zip"]): score += 12 return score def _best_methodology_source_from_candidates(message: str, candidates: list[dict[str, Any]], source_label: str) -> dict[str, str]: ranked: list[tuple[int, dict[str, Any]]] = [] for item in candidates: path = str(item.get("methodology_sandbox_registry") or item.get("output_dir") or item.get("batch_dir") or item.get("path") or "").strip() if not path: continue ranked.append((_methodology_candidate_score(message, item), item)) if not ranked: return {} ranked.sort(key=lambda row: row[0], reverse=True) best_score, best = ranked[0] if best_score <= 0: return {} registry = str(best.get("methodology_sandbox_registry") or "").strip() if registry: return {"registry_path": registry, "source": f"{source_label}_matched", "match_score": str(best_score)} batch_dir = str(best.get("output_dir") or best.get("batch_dir") or best.get("path") or "").strip() return {"methodology_dir": batch_dir, "source": f"{source_label}_matched", "match_score": str(best_score)} if batch_dir else {} def _latest_methodology_sandbox_source(message: str = "") -> dict[str, str]: import requests rag_base = _methodology_rag_base() completed_jobs: list[dict[str, Any]] = [] example_batches: list[dict[str, Any]] = [] try: response = requests.get(f"{rag_base}/methodology/extract-jobs", params={"limit": 20}, timeout=15) if response.status_code == 200: for job in (response.json().get("jobs") or []): if job.get("status") != "completed": continue completed_jobs.append(job) except Exception as exc: print(f"[Methodology Sandbox QA] extract job lookup skipped: {exc}") try: response = requests.get(f"{rag_base}/methodology/sandbox/example-batches", timeout=15) if response.status_code == 200: example_batches = response.json().get("batches") or [] except Exception as exc: print(f"[Methodology Sandbox QA] example batch lookup skipped: {exc}") all_candidates = [ *[{**batch, "_source_label": "example_batch"} for batch in example_batches], *[{**job, "_source_label": "latest_extract_job"} for job in completed_jobs], ] ranked: list[tuple[int, int, dict[str, Any]]] = [] for idx, item in enumerate(all_candidates): score = _methodology_candidate_score(message, item) if score > 0: ranked.append((score, -idx, item)) if ranked: ranked.sort(key=lambda row: (row[0], row[1]), reverse=True) _, _, best = ranked[0] return _best_methodology_source_from_candidates(message, [best], best.get("_source_label", "matched")) for job in completed_jobs: registry = str(job.get("methodology_sandbox_registry") or "").strip() if registry: return {"registry_path": registry, "source": "latest_extract_job"} batch_dir = str(job.get("output_dir") or job.get("batch_dir") or "").strip() if batch_dir: return {"methodology_dir": batch_dir, "source": "latest_extract_job"} for batch in example_batches: path = str(batch.get("path") or "").strip() if path: return {"methodology_dir": path, "source": "example_batch"} return {} def _methodology_sandbox_registry_for_question(message: str) -> tuple[str, dict[str, Any]]: import requests env_registry = os.environ.get("METHODOLOGY_SANDBOX_REGISTRY", "").strip() if env_registry: return env_registry, {"source": "env_registry", "registry_path": env_registry} explicit_dir = _extract_methodology_dir_from_text(message) source = {"methodology_dir": explicit_dir, "source": "message_path"} if explicit_dir else _latest_methodology_sandbox_source(message) registry = str(source.get("registry_path") or "").strip() if registry: return registry, source methodology_dir = str(source.get("methodology_dir") or "").strip() if not methodology_dir: return "", source if methodology_dir in methodology_sandbox_registry_cache: return methodology_sandbox_registry_cache[methodology_dir], {**source, "cache": True} sandbox_root = os.environ.get("METHODOLOGY_SANDBOX_ROOT", "/tmp/methodology_qa_sandbox_bbav5") run_id = "bbav5_nl_" + _sanitize_methodology_run_id(Path(methodology_dir).name) response = requests.post( f"{_methodology_rag_base()}/methodology/sandbox/build", json={ "methodology_dir": methodology_dir, "sandbox_root": sandbox_root, "run_id": run_id, "include_figures": True, "content_list_path": "", }, timeout=60, ) payload = response.json() if response.status_code >= 400 or payload.get("success") is False: detail = payload.get("detail") or payload.get("message") or response.text raise RuntimeError(f"RAG methodology sandbox build failed: {detail}") registry = str((payload.get("report") or {}).get("registry_path") or "").strip() if not registry: raise RuntimeError("RAG methodology sandbox build did not return registry_path") methodology_sandbox_registry_cache[methodology_dir] = registry return registry, {**source, "registry_path": registry, "sandbox_built": True} def _md_cell(value: Any, limit: int = 120) -> str: text = re.sub(r"\s+", " ", str(value or "")).strip() text = text.replace("|", "/") if len(text) > limit: return text[: max(0, limit - 3)].rstrip() + "..." return text def _methodology_stage_label(stage_id: str) -> str: labels = { "genome_prepare": "Data preparation", "identify": "Gene family identification", "phylogeny": "Phylogenetic analysis", "motif": "Conserved motif analysis", "domain": "Domain validation", "structure": "Gene structure analysis", "combo": "Integrated evidence figure", "reference_classification": "Reference-based classification", "subfamily_confirm": "Subfamily confirmation", "synteny": "Intraspecies synteny", "interspecies_synteny": "Interspecies synteny", "karyotype": "Chromosomal distribution", "expression": "Expression analysis", "pep": "Protein property analysis", "external_or_manual": "External/manual validation", } return labels.get(stage_id, stage_id.replace("_", " ").title()) def _methodology_stage_purpose(stage_id: str) -> str: purposes = { "genome_prepare": "Prepare genome/protein/annotation inputs before analysis.", "identify": "Find SPL family members and confirm SBP-domain candidates.", "phylogeny": "Infer evolutionary relationships with reference species.", "motif": "Detect conserved motifs across target family proteins.", "domain": "Validate conserved domain composition.", "structure": "Compare exon-intron organization and coding/genomic structure.", "combo": "Integrate tree, motif, domain and structure evidence.", "reference_classification": "Classify members by reference-supported subfamilies.", "subfamily_confirm": "Confirm or curate subfamily assignments before downstream analysis.", "synteny": "Detect duplication and collinearity within the target genome.", "interspecies_synteny": "Compare collinearity with selected plant genomes.", "karyotype": "Map target family genes onto chromosomes.", "expression": "Assess expression patterns across tissues or fruit development.", "pep": "Calculate protein properties and localization-related features.", "external_or_manual": "Record evidence that requires external/manual validation.", } return purposes.get(stage_id, "Summarize the corresponding methodology step.") def _compact_methodology_parameters(raw: Any, limit: int = 5) -> str: text = str(raw or "") if not text.strip(): return "" tokens = [token.strip() for token in text.split(";") if token.strip()] compact: list[str] = [] pending_name = "" skip_prefixes = ("evidence_sentence=", "parameter_type=") for token in tokens: if token.startswith(skip_prefixes): continue if token.startswith("parameter_name="): pending_name = token.split("=", 1)[1].strip() continue if token.startswith("parameter_value="): value = token.split("=", 1)[1].strip() if pending_name and value: compact.append(f"{pending_name}={value}") pending_name = "" continue if "=" in token: key, value = token.split("=", 1) key = key.strip() value = value.strip() if key and value and key not in {"parameter_type", "evidence_sentence"}: compact.append(f"{key}={value}") deduped = [] seen = set() for item in compact: clean_item = _md_cell(item, 80) if clean_item and clean_item not in seen: seen.add(clean_item) deduped.append(clean_item) if len(deduped) >= limit: break return ", ".join(deduped) def _methodology_evidence_label(row: dict[str, Any]) -> str: evidence_type = str(row.get("evidence_type") or "") if evidence_type == "methods_explicit": return "Methods text" if evidence_type == "figure_table_inferred": figure = row.get("figure_id") or row.get("table_id") return f"Figure/Table evidence ({figure})" if figure else "Figure/Table evidence" if evidence_type == "bbav5_required_prerequisite": return "bbav5 execution prerequisite" return evidence_type.replace("_", " ") or "Methodology evidence" def _methodology_read_csv_row(csv_path: Path, paper_id: str) -> dict[str, str]: if not csv_path.exists(): return {} try: with csv_path.open("r", encoding="utf-8-sig", newline="") as handle: rows = list(csv.DictReader(handle)) except Exception: return {} if not rows: return {} for row in rows: if str(row.get("paper_id") or "").strip() == paper_id: return {str(k): str(v or "") for k, v in row.items()} return {str(k): str(v or "") for k, v in rows[0].items()} def _methodology_clean_reference_title(title: str, family: str, species: str) -> str: text = re.sub(r"\s+", " ", title or "").strip() if not text: return "" if "Genome-wide identification" in text and "Tartary Buckwheat" in text and family: species_name = "Fagopyrum tataricum" if "Fagopyrum tataricum" in species else "" species_part = f" ({species_name})" if species_name else "" return f"Genome-wide identification of the {family} gene family in Tartary Buckwheat{species_part} and expression analysis during fruit development stages" if family and "the gene family in Tartary Buckwheat" in text: text = text.replace("the gene family in Tartary Buckwheat", f"the {family} gene family in Tartary Buckwheat") if "Fagopyrum tataricumanalysis" in text: text = text.replace("Fagopyrum tataricumanalysis", "Fagopyrum tataricum) and expression analysis") if "( ) and expression" in text and "Fagopyrum tataricum" in species: text = text.replace("( ) and expression", "(Fagopyrum tataricum) and expression") return text def _methodology_reference_from_content_list(content_list_path: str, family: str, species: str) -> dict[str, str]: if not content_list_path: return {} path = Path(content_list_path) if not path.exists(): return {} try: data = json.loads(path.read_text(encoding="utf-8")) except Exception: return {"content_list_path": str(path)} texts: list[str] = [] title = "" for item in data if isinstance(data, list) else []: if not isinstance(item, dict): continue text = re.sub(r"\s+", " ", str(item.get("text") or "")).strip() if not text: continue texts.append(text) if not title and item.get("text_level") == 1 and len(text) > 30: lowered = text.lower() if "research article" not in lowered and "open access" not in lowered: title = text joined = "\n".join(texts[:80]) doi_match = re.search(r"(10\.\d{4,9}/[^\s，,；;。\"']+)", joined, flags=re.I) pmid_match = re.search(r"\bPMID\s*[:：]?\s*([A-Za-z0-9_.-]+)", joined, flags=re.I) journal_match = re.search(r"([A-Za-z]+ et al\. BMC Plant Biology \(2019\) 19:299)", joined) return { "title": _methodology_clean_reference_title(title, family, species), "doi": doi_match.group(1).rstrip(".") if doi_match else "", "pmid": pmid_match.group(1) if pmid_match else "", "journal": journal_match.group(1) if journal_match else "", "content_list_path": str(path), } def _methodology_reference_info(plan: dict[str, Any], stage_rows: list[dict[str, Any]]) -> dict[str, str]: paper_id = str(plan.get("paper_id") or (stage_rows[0].get("paper_id") if stage_rows else "") or "").strip() family = str(plan.get("gene_family") or (stage_rows[0].get("gene_family") if stage_rows else "") or "").strip() species = str(plan.get("species") or (stage_rows[0].get("species") if stage_rows else "") or "").strip() source_batch_dir = str(plan.get("source_batch_dir") or (stage_rows[0].get("source_batch_dir") if stage_rows else "") or "").strip() plan_path = str(plan.get("methodology_plan_path") or (stage_rows[0].get("methodology_plan_path") if stage_rows else "") or "").strip() summary = _methodology_read_csv_row(Path(source_batch_dir) / "summary.csv", paper_id) if source_batch_dir else {} status = _methodology_read_csv_row(Path(source_batch_dir) / "processing_status.csv", paper_id) if source_batch_dir else {} build_report: dict[str, Any] = {} if plan_path: report_path = Path(plan_path).with_name("methodology_kb_build_report.json") if report_path.exists(): try: build_report = json.loads(report_path.read_text(encoding="utf-8")) except Exception: build_report = {} content_list_path = str(build_report.get("figure_table_content_list_path") or "").strip() if not content_list_path: raw_path = str(summary.get("parsed_document_path") or status.get("parsed_document_path") or "").strip() if raw_path: candidate = Path(raw_path) content_list_path = str(candidate if candidate.is_absolute() else GF_MAS_ROOT / raw_path) extracted = _methodology_reference_from_content_list(content_list_path, family, species) title = extracted.get("title") or summary.get("paper_title") or status.get("paper_title") or paper_id return { "paper_id": paper_id, "title": title, "doi": extracted.get("doi", ""), "pmid": extracted.get("pmid", ""), "journal": extracted.get("journal", ""), "source_role": summary.get("source_paper_role") or status.get("source_paper_role") or "methodology_source", "source_batch_dir": source_batch_dir, "content_list_path": extracted.get("content_list_path") or content_list_path, } def _format_methodology_reference_section(plan: dict[str, Any], stage_rows: list[dict[str, Any]]) -> list[str]: ref = _methodology_reference_info(plan, stage_rows) if not ref.get("paper_id") and not ref.get("title"): return [] doi = ref.get("doi") or "" doi_text = f"[{doi}](https://doi.org/{doi})" if doi else "-" index_parts = [ ref.get("paper_id", ""), ref.get("source_role", ""), "methodology_plan", ] index_text = " · ".join(_md_cell(part, 60) for part in index_parts if part) lines = [ "", "### References", "", "| # | Reference | DOI | PMID | Index |", "|---:|---|---|---|---|", f"| 1 | {_md_cell(ref.get('title') or ref.get('paper_id') or 'Methodology source', 180)} | {doi_text} | {_md_cell(ref.get('pmid') or '-', 40)} | {_md_cell(index_text, 160)} |", ] if ref.get("journal"): lines.append(f"- Journal/source hint: {_md_cell(ref.get('journal'), 140)}") if ref.get("source_batch_dir"): lines.append(f"- Source batch: `{_md_cell(ref.get('source_batch_dir'), 180)}`") if ref.get("content_list_path"): lines.append(f"- Parsed document: `{_md_cell(ref.get('content_list_path'), 180)}`") return lines def _methodology_reference_sources_for_frontend(payload: dict[str, Any]) -> list[dict[str, Any]]: if payload.get("success") is False or str(payload.get("match_status") or "") in { "no_family_match", "family_match_species_miss", "no_species_match", "no_match", "no_candidates", }: return [] sources = payload.get("sources") or [] if not isinstance(sources, list) or not sources: return [] plan = next((row for row in sources if row.get("record_type") == "methodology_plan_card"), {}) stage_rows = [row for row in sources if row.get("record_type") == "methodology_stage_card" and row.get("bbav5_stage_id")] ref = _methodology_reference_info(plan, stage_rows) if not ref.get("paper_id") and not ref.get("title"): return [] content_parts = [ f"Paper: {ref.get('paper_id') or '-'}", f"Source role: {ref.get('source_role') or '-'}", f"Source batch: {ref.get('source_batch_dir') or '-'}", ] if ref.get("journal"): content_parts.append(f"Journal: {ref.get('journal')}") return [{ "doc_id": ref.get("paper_id") or "methodology_reference", "title": ref.get("title") or ref.get("paper_id") or "Methodology source", "doi": ref.get("doi", ""), "pmid": ref.get("pmid", ""), "score": 1.0, "content": " ".join(content_parts), "metadata": { "type": "methodology_reference", "paper_id": ref.get("paper_id", ""), "paper_title": ref.get("title", ""), "doi": ref.get("doi", ""), "pmid": ref.get("pmid", ""), "source_role": ref.get("source_role", ""), }, }] def _format_methodology_sandbox_answer(payload: dict[str, Any], question: str) -> str: sources = payload.get("sources") or [] if not isinstance(sources, list) or not sources: return str(payload.get("answer") or payload.get("message") or "") plan = next((row for row in sources if row.get("record_type") == "methodology_plan_card"), {}) stage_rows = [row for row in sources if row.get("record_type") == "methodology_stage_card" and row.get("bbav5_stage_id")] stage_rows.sort(key=lambda row: int(row.get("stage_order") or 999)) paper = _md_cell(plan.get("paper_id") or (stage_rows[0].get("paper_id") if stage_rows else ""), 80) family = _md_cell(plan.get("gene_family") or (stage_rows[0].get("gene_family") if stage_rows else ""), 40) matched_plan = payload.get("matched_plan") if isinstance(payload.get("matched_plan"), dict) else {} target_species = matched_plan.get("target_species") or plan.get("target_species") or (stage_rows[0].get("target_species") if stage_rows else "") reference_species = matched_plan.get("reference_species") or plan.get("reference_species") or (stage_rows[0].get("reference_species") if stage_rows else "") if isinstance(target_species, list): target_species = ";".join(str(item) for item in target_species if str(item or "").strip()) if isinstance(reference_species, list): reference_species = ";".join(str(item) for item in reference_species if str(item or "").strip()) species = _md_cell(plan.get("species") or (stage_rows[0].get("species") if stage_rows else ""), 180) match_status = str(payload.get("match_status") or "") lines = [ f"### {family or 'Gene family'} methodology workflow", "", f"**Paper:** {paper or 'unknown'} ", ] if target_species: lines.append(f"**Target species:** {_md_cell(target_species, 140)}") if reference_species: lines.append(f"**Reference species:** {_md_cell(reference_species, 180)}") if not target_species and not reference_species: lines.append(f"**Species:** {species or 'unknown'}") if match_status == "family_match_species_miss": constraints = payload.get("query_constraints") or {} requested_species = "; ".join(constraints.get("requested_species") or []) or "unknown" lines.extend([ "", f"> 注意：当前临时方法学库只命中了同家族 `{family or '-'}` 的方法学计划，但没有命中目标物种 `{requested_species}`。下面内容只能作为同家族通用参考，不能视为目标物种论文证据。", ]) lines.extend([ "", "The workflow can be read as: identify the gene family members, validate the conserved domain, build evolutionary and structural evidence, then add chromosome/synteny/expression/protein-property analyses where supported by the paper.", "", "| # | Step | Purpose | Tools | Key parameters | Evidence |", "|---:|---|---|---|---|---|", ]) seen_stages: set[str] = set() prerequisite_notes: list[str] = [] for row in stage_rows: stage_id = str(row.get("bbav5_stage_id") or "") if not stage_id or stage_id in seen_stages: continue seen_stages.add(stage_id) label = _methodology_stage_label(stage_id) purpose = _methodology_stage_purpose(stage_id) tools = _md_cell(row.get("tools") or row.get("selected_tool") or "Required by workflow", 100) params = _compact_methodology_parameters(row.get("parameters")) or "Not specified" evidence_label = _methodology_evidence_label(row) if row.get("evidence_type") == "bbav5_required_prerequisite": prerequisite_notes.append(f"`{stage_id}`") lines.append( f"| {len(seen_stages)} | **{label}** (`{stage_id}`) | {_md_cell(purpose, 120)} | {tools} | {_md_cell(params, 120)} | {_md_cell(evidence_label, 90)} |" ) explicit = sum(1 for row in stage_rows if row.get("evidence_type") == "methods_explicit") figure_based = sum(1 for row in stage_rows if row.get("evidence_type") == "figure_table_inferred") lines.extend(["", "**How to interpret this**"]) lines.append(f"- {explicit} step(s) are directly supported by Methods text; {figure_based} step(s) are inferred from figure/table captions.") if prerequisite_notes: lines.append(f"- {', '.join(prerequisite_notes)} are bbav5 execution prerequisites, not necessarily explicit article Methods steps.") lines.append("- The temporary methodology sandbox was used for this answer; production KG/vector stores were not modified.") lines.extend(_format_methodology_reference_section(plan, stage_rows)) return "\n".join(lines) def _answer_methodology_sandbox_qa(message: str) -> dict[str, Any]: import requests registry, source = _methodology_sandbox_registry_for_question(message) if not registry: return { "success": False, "message": "没有找到可用的方法学 sandbox 来源。请先完成方法学文献提取，或配置 METHODOLOGY_SANDBOX_REGISTRY。", "source": source, } response = requests.post( f"{_methodology_rag_base()}/methodology/sandbox/answer", json={"question": message, "registry_path": registry, "sandbox_dir": ""}, timeout=30, ) payload = response.json() if response.status_code >= 400: detail = payload.get("detail") if isinstance(payload, dict) else response.text raise RuntimeError(f"RAG methodology sandbox answer failed: {detail}") payload["raw_answer"] = payload.get("answer", "") payload["answer"] = _format_methodology_sandbox_answer(payload, message) payload["reference_sources"] = _methodology_reference_sources_for_frontend(payload) payload.setdefault("source", source) payload["registry_path"] = registry return payload async def _run_agent_chat( request: AgentChatRequest, progress_callback=None, ): """MainAgent conversation boundary. MainAgent may answer, query RAG/read-only project tools, or ask the original IntentAgentV2 to produce an analysis intent. It never plans or executes analysis stages here. """ session_id = request.session_id or f"session_{int(time.time())}" completed_stages: list[str] = [] if request.job_id: final_dir = PROJECT_ROOT / "workspace" / request.job_id / "final" if final_dir.exists(): for manifest_path in final_dir.glob("*.manifest.json"): try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if manifest.get("status") in {"success", "partial_success"}: stage = str(manifest.get("stage") or "").strip() if stage: completed_stages.append(stage) conversation_turn = None active_store = globals().get("agent_conversation_store") if active_store is not None: try: conversation_turn = active_store.begin_turn( session_id=session_id, job_id=request.job_id, query=request.message, client_history=request.history, ) except ValueError as exc: if str(exc) == "conversation job binding conflict": raise HTTPException(status_code=409, detail=str(exc)) from exc raise session_id = conversation_turn.conversation_id request.job_id = conversation_turn.job_id request.history = list(conversation_turn.history) router_context_terms: Dict[str, Any] = {} router_goal_ledger = ( conversation_turn.goal_ledger if conversation_turn is not None else None ) if router_goal_ledger is not None and request.job_id: from gfa_core.mas.confirmation_snapshot import build_confirmation_snapshot terms = _hydrate_goal_terms_from_current_job( dict(router_goal_ledger.model_dump(mode="json")["user_terms"]), request.job_id, ) expected_identity = { key: str(terms.get(key) or "") for key in ("prefix", "scope_id", "member_set_id") } try: confirmation = build_confirmation_snapshot( PROJECT_ROOT, request.job_id, expected_identity, ) except (OSError, ValueError): confirmation = { "job_id": request.job_id, "config": None, "samples": None, "subfamily": None, "config_confirmed": False, "samples_confirmed": False, "subfamily_confirmed": False, "errors": ("confirmation snapshot unavailable",), } confirmation_context = { **confirmation, "errors": list(confirmation.get("errors") or ()), } terms.update( { "config_confirmed": bool(confirmation["config_confirmed"]), "samples_confirmed": bool(confirmation["samples_confirmed"]), "subfamily_confirmed": bool(confirmation["subfamily_confirmed"]), "confirmation_snapshot": confirmation_context, } ) router_goal_ledger = router_goal_ledger.model_copy( update={ "revision": router_goal_ledger.revision + 1, "user_terms": terms, } ) router_context_terms = dict(terms) from gfa_core.mas.utils.intent_parser import parse_sequence_extract_request utility_params = parse_sequence_extract_request(request.message) if utility_params: utility_message = "我理解你想按ID从本地FASTA提取或处理序列。" utility_intent = { "stage": "sequence_extract_by_id", "stages": ["sequence_extract_by_id"], "primary_stage": "sequence_extract_by_id", "params": utility_params, "task_kind": "utility", "taskType": "utility", "utility_tool": ( "fasta_header_clean" if utility_params.get("sequenceExtractCleanHeaders") else "sequence_extract_by_id" ), "message": utility_message, "job_id": request.job_id, "workspace": ( str(PROJECT_ROOT / "workspace" / request.job_id) if request.job_id else "" ), } if conversation_turn is not None: active_store.finish_turn( conversation_turn, assistant_message=utility_message, interaction_kind="independent", action="propose_utility", intent=utility_intent, ) return { "success": True, "status": "ok", "action": "propose_utility", "intent": utility_intent, "sources": [], "tool_trace": [], "mode": "analysis", "completed_stages": completed_stages, "model_used": "rule", "session_id": session_id, } global llm_manager active_llm_manager = llm_manager if active_llm_manager is None: try: from gfa_core.mas.config import config as mas_config from gfa_core.mas.llm import LLMManager as RuntimeLLMManager runtime_config = mas_config.load_llm_config() runtime_config["openai_api_key"] = mas_config.settings.openai_api_key runtime_config["openai_base_url"] = mas_config.settings.openai_base_url active_llm_manager = RuntimeLLMManager(config=runtime_config) llm_manager = active_llm_manager except Exception as exc: print(f"[MainAgent] LLM manager unavailable: {exc}") return { "success": False, "status": "agent_unavailable", "action": None, "intent": None, "message": "The MainAgent model is currently unavailable.", "sources": [], "tool_trace": [], "mode": "error", "completed_stages": completed_stages, "model_used": request.model, "session_id": session_id, } try: from gfa_core.mas.agents.base import AgentMessage, AgentState from gfa_core.mas.agents.conversation_tools import ( build_plantai_conversation_tools, ) from gfa_core.mas.agents.router import RouterAgent router_tools = build_plantai_conversation_tools( llm_manager=active_llm_manager, project_root=PROJECT_ROOT, bound_job_id=request.job_id, goal_ledger=router_goal_ledger, allow_legacy_manifest_migration=False, progress_callback=progress_callback, ) router = RouterAgent(active_llm_manager, tools=router_tools) state = AgentState(session_id=session_id, job_id=request.job_id) payload = { "query": request.message, "history": list(request.history or []), "session_id": session_id, "job_id": request.job_id, } if router_goal_ledger is not None: payload["goal_ledger"] = router_goal_ledger state = await router.execute( state, AgentMessage( sender="user", receiver="router", message_type="classify", payload=payload, ), ) outcome = dict(state.results.get("router_outcome") or {}) status = str(outcome.get("status") or "invalid_response") action = outcome.get("action") if status == "ok" else None allowed_actions = { "reply", "clarify", "show_result", "propose_analysis", "propose_utility", "prepare_inputs", } raw_trace = outcome.get("tool_trace") or [] tool_trace = [] for item in raw_trace[:5]: if not isinstance(item, dict): continue tool_trace.append( { "tool_call_id": str(item.get("tool_call_id") or "")[:128], "name": str(item.get("name") or "")[:128], "arguments": str(item.get("arguments") or "")[:500], "result": str(item.get("result") or "")[:1000], } ) sources = [ item for item in (outcome.get("sources") or []) if isinstance(item, dict) ][:20] if status != "ok" or action not in allowed_actions: return { "success": False, "status": status, "action": None, "intent": None, "message": outcome.get("message") or "MainAgent did not return a valid response.", "sources": sources, "tool_trace": tool_trace, "mode": "error", "completed_stages": completed_stages, "model_used": request.model, "session_id": session_id, } verified_intent = outcome.get("intent") if action == "propose_analysis" and isinstance(verified_intent, dict): role_resolution = ( _reconcile_confirmed_gene_family_species( request.job_id, request.message, verified_intent, ) if _is_gene_family_analysis_intent(verified_intent) else {"applicable": False, "conflicts": []} ) if role_resolution.get("conflicts"): action = "clarify" outcome["message"] = _species_role_conflict_message(role_resolution["conflicts"]) outcome["proposal"] = None verified_intent = None else: verified_intent = _normalize_main_agent_analysis_intent( verified_intent, job_id=request.job_id, context_terms=router_context_terms, ) _apply_reconciled_species_to_intent(verified_intent, role_resolution) verified_stages = ( verified_intent.get("stages") if isinstance(verified_intent, dict) else None ) if action == "propose_analysis" and ( not isinstance(verified_intent, dict) or not isinstance(verified_stages, list) or not verified_stages or any( not isinstance(stage, str) or not stage.strip() for stage in verified_stages ) ): return { "success": False, "status": "invalid_response", "action": None, "intent": None, "message": "MainAgent proposed analysis without an IntentAgent result.", "sources": sources, "tool_trace": tool_trace, "mode": "error", "completed_stages": completed_stages, "model_used": request.model, "session_id": session_id, } if action == "propose_utility" and ( not isinstance(verified_intent, dict) or verified_intent.get("taskType") != "utility" or not str(verified_intent.get("utility_tool") or "").strip() or not isinstance(verified_stages, list) or not verified_stages or not isinstance(verified_intent.get("params"), dict) ): return { "success": False, "status": "invalid_response", "action": None, "intent": None, "message": "MainAgent proposed an invalid utility intent.", "sources": sources, "tool_trace": tool_trace, "mode": "error", "completed_stages": completed_stages, "model_used": request.model, "session_id": session_id, } task_type = { "reply": "faq", "clarify": "clarification", "show_result": "result_view", "prepare_inputs": "input_preparation", "propose_analysis": "analysis", "propose_utility": "utility", }[action] intent = ( dict(verified_intent) if action in {"propose_analysis", "propose_utility"} else {} ) outcome_message = str(outcome.get("message") or "") if action == "propose_analysis" and isinstance(outcome.get("proposal"), dict): outcome_message = ( "已将确认后的请求交给现有分析系统识别。接下来由 PlanningAgent " "生成可核对的计划；当前尚未执行任何分析。" ) intent.update( { "taskType": task_type, "message": outcome_message, "ui_action": outcome.get("ui_action"), "job_id": request.job_id, "workspace": ( str(PROJECT_ROOT / "workspace" / request.job_id) if request.job_id else "" ), } ) if action not in {"propose_analysis", "propose_utility"}: intent.update( { "stage": None, "stages": [], "primary_stage": None, "params": {}, "task_kind": task_type, } ) elif verified_stages and not intent.get("stage"): intent["stage"] = verified_stages[0] if conversation_turn is not None: active_store.finish_turn( conversation_turn, assistant_message=intent["message"], interaction_kind=str( outcome.get("interaction_kind") or "independent" ), action=action, intent=intent, proposal=( outcome.get("proposal") if isinstance(outcome.get("proposal"), dict) else None ), ) return { "success": True, "status": status, "action": action, "intent": intent, "sources": sources, "tool_trace": tool_trace, "mode": { "reply": "faq", "clarify": "consultation", "prepare_inputs": "input_preparation", "show_result": "interpretation", "propose_analysis": "analysis", "propose_utility": "analysis", }[action], "completed_stages": completed_stages, "model_used": request.model, "session_id": session_id, "input_paths": list(outcome.get("input_paths") or []), "handoff_request": ( str((outcome.get("proposal") or {}).get("request") or "").strip() if isinstance(outcome.get("proposal"), dict) else request.message ) or request.message, } except Exception as exc: print(f"[MainAgent] Error: {exc}") return { "success": False, "status": "internal_error", "action": None, "intent": None, "message": f"MainAgent failed: {exc}", "sources": [], "tool_trace": [], "mode": "error", "completed_stages": completed_stages, "model_used": request.model, "session_id": session_id, } @app.post("/api/agent/chat") async def agent_chat(request: AgentChatRequest): return await _run_agent_chat(request) AGENT_CHAT_HEARTBEAT_SECONDS = 5.0 _AGENT_CHAT_TOOL_PROGRESS = { "inspect_project_context": ("正在读取当前 job 的项目状态……", "当前 job 状态读取完成，主 Agent 正在继续处理……"), "inspect_analysis_results": ("正在读取当前 job 的分析结果……", "分析结果读取完成，主 Agent 正在继续处理……"), "inspect_job_file": ("正在读取当前 job 中的文件……", "文件读取完成，主 Agent 正在继续处理……"), "query_plantai_rag": ("正在检索知识库……", "知识库检索完成，主 Agent 正在继续处理……"), "recognize_analysis_intent": ("正在识别分析需求……", "分析需求识别完成，主 Agent 正在继续处理……"), "recognize_utility_intent": ("正在识别工具需求……", "工具需求识别完成，主 Agent 正在继续处理……"), } def _agent_chat_progress_payload(event: dict, elapsed_seconds: int) -> dict: stage = str(event.get("stage") or "") tool = str(event.get("tool") or "") messages = _AGENT_CHAT_TOOL_PROGRESS.get(tool) if stage == "tool_started" and messages: message = messages[0] elif stage == "tool_completed" and messages: message = messages[1] else: message = "主 Agent 正在处理请求……" return { "stage": stage or "agent_running", "tool": tool, "message": message, "elapsed_seconds": elapsed_seconds, } def _agent_chat_sse(event_name: str, payload: dict) -> str: return f"event: {event_name}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n" @app.post("/api/agent/chat/stream") async def agent_chat_stream(request: AgentChatRequest, http_request: Request): async def event_stream(): started_at = time.monotonic() progress_queue: asyncio.Queue[dict] = asyncio.Queue() def publish_progress(event: dict) -> None: progress_queue.put_nowait(dict(event)) runner = asyncio.create_task( _run_agent_chat(request, progress_callback=publish_progress) ) yield _agent_chat_sse( "progress", { "stage": "agent_started", "message": "正在理解你的问题并确定所需信息……", "elapsed_seconds": 0, }, ) try: while not runner.done() or not progress_queue.empty(): if await http_request.is_disconnected(): runner.cancel() return try: event = await asyncio.wait_for( progress_queue.get(), timeout=AGENT_CHAT_HEARTBEAT_SECONDS, ) except asyncio.TimeoutError: elapsed = int(time.monotonic() - started_at) yield _agent_chat_sse( "heartbeat", { "stage": "agent_running", "message": f"主 Agent 仍在处理，已用时 {elapsed} 秒……", "elapsed_seconds": elapsed, }, ) continue elapsed = int(time.monotonic() - started_at) yield _agent_chat_sse( "progress", _agent_chat_progress_payload(event, elapsed), ) result = await runner yield _agent_chat_sse("complete", result) except asyncio.CancelledError: runner.cancel() raise except Exception as exc: runner.cancel() yield _agent_chat_sse( "error", { "error_code": "agent_stream_failed", "message": f"主 Agent 流式请求失败：{exc}", }, ) finally: if not runner.done(): runner.cancel() return StreamingResponse( event_stream(), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, ) # ==================== RNA-seq 接口 ==================== class RnaseqRequest(BaseModel): """Legacy RNA-seq stage request. The main RNA-seq workflow is executed through /api/execute and pipeline_backend.py. This endpoint is kept for old callers only. """ job_id: str = "" stage: str = "" sample: str = "" samples: list = Field(default_factory=list) design: dict = Field(default_factory=dict) params: dict = Field(default_factory=dict) @app.post("/api/rnaseq/run") async def run_rnaseq(req: RnaseqRequest, background_tasks: BackgroundTasks): """运行旧版RNA-seq单阶段分析""" task_id = str(uuid.uuid4()) project_root = PROJECT_ROOT # 脚本路径 script_map = { "align": "apps/cli/run_rnaseq_align.py", "quant": "apps/cli/run_rnaseq_quant.py", "de": "apps/cli/run_rnaseq_de.py", "enrich": "apps/cli/run_rnaseq_enrich.py" } stage = (req.stage or req.params.get("stage") or "").strip() sample = (req.sample or req.params.get("sample") or "").strip() if not stage or not sample: return { "success": False, "message": "旧版 /api/rnaseq/run 需要 stage 和 sample；当前 RNA-seq 流程请使用 /api/execute。", } if stage not in script_map: return {"success": False, "message": f"未知阶段: {stage}"} script_path = project_root / script_map[stage] sample_dir = project_root / "rnaseq" / sample rnaseq_tasks[task_id] = { "id": task_id, "stage": stage, "sample": sample, "status": "running", "message": f"正在运行 {stage}...", "start_time": datetime.now().isoformat() } def run_rnaseq_task(): try: if stage == "align": cmd = [ "python3", str(script_path), "-i", str(sample_dir), "-x", str(project_root / "rnaseq" / "Ft.genome"), "-o", str(sample_dir), "-j", sample, "-t", "8" ] elif stage == "quant": cmd = [ "python3", str(script_path), "-i", str(sample_dir), "-o", str(sample_dir), "-j", sample, "-t", "8" ] else: cmd = ["python3", str(script_path), "-i", str(project_root / "rnaseq")] result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode == 0: rnaseq_tasks[task_id]["status"] = "success" rnaseq_tasks[task_id]["message"] = "完成" else: rnaseq_tasks[task_id]["status"] = "failed" rnaseq_tasks[task_id]["message"] = (result.stderr or result.stdout or f"exit code {result.returncode}")[-2000:] rnaseq_tasks[task_id]["end_time"] = datetime.now().isoformat() except Exception as e: rnaseq_tasks[task_id]["status"] = "failed" rnaseq_tasks[task_id]["message"] = str(e) rnaseq_tasks[task_id]["end_time"] = datetime.now().isoformat() background_tasks.add_task(run_rnaseq_task) return {"success": True, "task_id": task_id, "message": f"任务已提交: {stage}"} @app.get("/api/rnaseq/status/{task_id}") async def get_rnaseq_status(task_id: str): """获取RNA-seq任务状态""" if task_id not in rnaseq_tasks: return {"success": False, "message": "任务不存在"} return {"success": True, "task": rnaseq_tasks[task_id]} @app.get("/api/rnaseq/samples") async def list_rnaseq_samples(): """列出RNA-seq样本""" project_root = PROJECT_ROOT rnaseq_dir = project_root / "rnaseq" samples = [] if rnaseq_dir.exists(): for d in rnaseq_dir.iterdir(): if d.is_dir() and d.name.startswith("SRR"): samples.append({ "name": d.name, "has_fastq": list(d.glob("*.fastq")) or list(d.glob("*.fq")), "has_gtf": list(d.glob("*.gtf")) }) return {"success": True, "samples": samples} @app.get("/api/rnaseq/results") async def get_rnaseq_results(): """获取RNA-seq分析结果""" project_root = PROJECT_ROOT de_output = project_root / "rnaseq" / "de_output" results = {} if de_output.exists(): results = { "de_results": str(de_output / "DE_results.txt"), "volcano_plot": str(de_output / "volcano_plot.png"), "heatmap": str(de_output / "heatmap.png"), "sig_genes": str(de_output / "sig_genes.DE_genes.txt") } return {"success": True, "results": results} # ==================== LearningAgent API ==================== class LearningRequest(BaseModel): """学习请求""" action: str = "recommend" # record, recommend, analyze stage: str = "" prefix: str = "" params: dict = Field(default_factory=dict) status: str = "" elapsed_sec: float = 0.0 final_members: int = 0 @app.post("/api/agent/learn") async def learning(request: LearningRequest): """ 学习优化接口 记录执行历史、推荐参数、分析趋势 """ try: from gfa_core.mas.agents.learning_v2 import LearningAgentV2 from gfa_core.mas.config import config as mas_config from gfa_core.mas.llm import LLMManager from gfa_core.mas.agents.base import AgentMessage, AgentState # 初始化组件 try: llm_config = mas_config.load_llm_config() llm_manager = LLMManager(config=llm_config) except: llm_manager = None # 创建LearningAgent learning_agent = LearningAgentV2(llm_manager) # 构建消息 message = AgentMessage( sender="user", receiver="learning", message_type=request.action, payload={ "action": request.action, "stage": request.stage, "prefix": request.prefix, "params": request.params, "status": request.status, "elapsed_sec": request.elapsed_sec, "final_members": request.final_members } ) # 构建状态 state = AgentState( session_id=f"learn_{int(time.time())}", results={}, messages=[] ) # 执行学习 result_state = await learning_agent.execute(state, message) if request.action == "recommend": recommendations = result_state.results.get("recommendations", {}) return { "success": True, "action": request.action, "recommendations": recommendations } elif request.action == "analyze": analysis = result_state.results.get("analysis", {}) return { "success": True, "action": request.action, "analysis": analysis } else: # record record_result = result_state.results.get("record_result", {}) return { "success": True, "action": request.action, "result": record_result } except Exception as e: import traceback return { "success": False, "message": f"学习优化失败: {str(e)}", "error_detail": traceback.format_exc() } # ==================== ValidatorAgent API ==================== class ValidateRequest(BaseModel): """验证请求""" stage: str prefix: str workspace: str manifest_path: str @app.post("/api/agent/validate") async def validate_result(request: ValidateRequest): """ 结果验证接口 验证分析结果质量 """ try: from gfa_core.mas.agents.validator_v2 import ValidatorAgentV2 from gfa_core.mas.config import config as mas_config from gfa_core.mas.llm import LLMManager from gfa_core.mas.agents.base import AgentMessage, AgentState # 初始化组件 try: llm_config = mas_config.load_llm_config() llm_manager = LLMManager(config=llm_config) except: llm_manager = None # 创建ValidatorAgent validator_agent = ValidatorAgentV2(llm_manager) # 读取manifest import json manifest_path = Path(request.manifest_path) if not manifest_path.exists(): return { "success": False, "message": f"Manifest文件不存在: {request.manifest_path}" } with open(manifest_path) as f: manifest = json.load(f) # 构建execution_result execution_result = { "status": manifest.get("status", "unknown"), "manifest": manifest } # 构建消息 message = AgentMessage( sender="user", receiver="validator", message_type="validate", payload={ "stage": request.stage, "prefix": request.prefix, "workspace": request.workspace } ) # 构建状态 state = AgentState( session_id=f"validate_{int(time.time())}", results={"execution_result": execution_result}, messages=[] ) # 执行验证 result_state = await validator_agent.execute(state, message) report = result_state.results.get("validation_report", {}) return { "success": True, "report": report, "overall_passed": report.get("overall_passed", False), "score": report.get("score", 0), "summary": report.get("summary", "") } except Exception as e: import traceback return { "success": False, "message": f"验证失败: {str(e)}", "error_detail": traceback.format_exc() } # ==================== ResultParserAgent API ==================== class ParseResultRequest(BaseModel): """结果解析请求""" stage: str manifest_path: str job_id: str = "" @app.post("/api/agent/parse_result") async def parse_result(request: ParseResultRequest): """ 结果解析接口 解析分析结果并生成报告 """ try: from gfa_core.mas.agents.result_parser_v2 import ResultParserAgentV2 from gfa_core.mas.config import config as mas_config from gfa_core.mas.llm import LLMManager from gfa_core.mas.agents.base import AgentMessage, AgentState # 初始化组件 try: llm_config = mas_config.load_llm_config() llm_manager = LLMManager(config=llm_config) except: llm_manager = None # 创建ResultParserAgent parser_agent = ResultParserAgentV2(llm_manager) # 读取manifest import json manifest_path = Path(request.manifest_path) if not manifest_path.exists(): return { "success": False, "message": f"Manifest文件不存在: {request.manifest_path}" } with open(manifest_path) as f: manifest = json.load(f) # 构建execution_result execution_result = { "status": manifest.get("status", "unknown"), "manifest": manifest, "stage": request.stage } # 构建消息 message = AgentMessage( sender="user", receiver="result_parser", message_type="parse", payload={"stage": request.stage} ) # 构建状态 state = AgentState( session_id=request.job_id or f"parse_{int(time.time())}", results={"execution_result": execution_result}, messages=[] ) # 执行解析 result_state = await parser_agent.execute(state, message) report = result_state.results.get("parsed_report", {}) return { "success": True, "report": report, "markdown": report.get("markdown", ""), "summary": report.get("summary", {}), "metrics": report.get("metrics", {}) } except Exception as e: import traceback return { "success": False, "message": f"结果解析失败: {str(e)}", "error_detail": traceback.format_exc() } # ==================== Agent执行状态反馈 API ==================== # Agent任务存储 agent_tasks: Dict[str, dict] = {} class AgentExecuteRequest(BaseModel): """Agent执行请求""" message: str session_id: str = "" job_id: str = "" # 当前选中的 job_id，用于 auto_continue prefix: str = "" model: str = "kimi" history: list = [] auto_execute: bool = True # 是否自动执行分析 class AgentStatusResponse(BaseModel): """Agent状态响应""" task_id: str status: str # pending, running, completed, failed current_stage: str = "" progress: int = 0 # 0-100 message: str = "" logs: list = [] result: dict = {} @app.post("/api/agent/execute") async def agent_execute(request: AgentExecuteRequest, background_tasks: BackgroundTasks): """ Agent执行接口 - 使用Celery异步执行分析任务 与/agent/chat不同，此接口会实际执行分析流程 """ task_id = f"agent_task_{int(time.time())}_{uuid.uuid4().hex[:8]}" # 初始化任务状态 agent_tasks[task_id] = { "task_id": task_id, "status": "pending", "message": request.message, "job_id": request.job_id, "prefix": request.prefix, "current_stage": "", "progress": 0, "logs": [], "result": {}, "start_time": datetime.now().isoformat(), "session_id": request.session_id or task_id, "stages_completed": [], "stages_pending": [] } # 使用Celery异步执行 if CELERY_AVAILABLE and execute_agent_task is not None: agent_queue = _default_celery_queue() queue_health, restart_result = _ensure_celery_queue_health(agent_queue, reason="submit_agent") agent_tasks[task_id]["celery_queue"] = agent_queue agent_tasks[task_id]["celery_queue_health"] = queue_health if restart_result: agent_tasks[task_id]["celery_restart"] = restart_result if not queue_health.get("available"): payload = _queue_unavailable_payload("agent_execute", agent_queue, queue_health, submitted=False, restart_result=restart_result) agent_tasks[task_id]["status"] = "queue_unavailable" agent_tasks[task_id]["message"] = payload["message"] agent_tasks[task_id]["result"] = payload agent_tasks[task_id]["action_required"] = True agent_tasks[task_id]["retryable"] = True agent_tasks[task_id]["recommended_action"] = "restart_worker" agent_tasks[task_id]["logs"].append({ "time": datetime.now().isoformat(), "level": "error", "message": payload["message"], }) else: try: # 提交任务到Celery队列 celery_task = execute_agent_task.delay( task_id=task_id, message=request.message, session_id=request.session_id or task_id, job_id=request.job_id, # 传入选中的 job_id history=request.history if request.history else [], prefix=request.prefix, ) print(f"[DEBUG] Celery task submitted: {celery_task.id} for agent task {task_id}") # 存储Celery任务ID映射 agent_tasks[task_id]["celery_task_id"] = celery_task.id agent_tasks[task_id]["logs"].append({ "time": datetime.now().isoformat(), "level": "info", "message": f"任务已提交到Celery队列 (celery_task_id: {celery_task.id})", }) except Exception as e: print(f"[ERROR] Failed to submit Celery task: {e}") agent_tasks[task_id]["status"] = "failed" agent_tasks[task_id]["logs"].append({ "time": datetime.now().isoformat(), "level": "error", "message": f"Celery任务提交失败: {str(e)}", }) else: # Celery不可用，使用降级方案（直接执行） print(f"[WARN] Celery不可用，使用降级方案执行") agent_tasks[task_id]["logs"].append({ "time": datetime.now().isoformat(), "level": "warn", "message": "Celery不可用，使用本地异步执行" }) async def execute_agent_task_async(): """异步执行Agent任务（降级方案）""" try: agent_tasks[task_id]["status"] = "running" if orchestrator is not None and MAS_AVAILABLE: from gfa_core.mas.core.state import GraphState initial_state: GraphState = { "query": request.message, "session_id": request.session_id or task_id, "job_id": request.job_id, # 传入选中的 job_id "files": [], "current_agent": "router", "status": "running", "errors": [], "final_output": {}, "history": request.history if request.history else [] } result = await orchestrator.run(initial_state) if result.get("status") == "completed": agent_tasks[task_id]["status"] = "completed" agent_tasks[task_id]["progress"] = 100 agent_tasks[task_id]["result"] = result.get("final_output", {}) else: agent_tasks[task_id]["status"] = "failed" else: agent_tasks[task_id]["status"] = "failed" agent_tasks[task_id]["logs"].append({ "time": datetime.now().isoformat(), "level": "error", "message": "Orchestrator不可用" }) except Exception as e: agent_tasks[task_id]["status"] = "failed" agent_tasks[task_id]["logs"].append({ "time": datetime.now().isoformat(), "level": "error", "message": f"执行异常: {str(e)}" }) finally: agent_tasks[task_id]["end_time"] = datetime.now().isoformat() asyncio.create_task(execute_agent_task_async()) return { "success": agent_tasks[task_id].get("status") != "queue_unavailable", "task_id": task_id, "message": agent_tasks[task_id].get("message", "Agent任务已提交"), "status": agent_tasks[task_id].get("status", "pending"), "celery_queue": agent_tasks[task_id].get("celery_queue"), "celery_queue_health": agent_tasks[task_id].get("celery_queue_health"), "celery_restart": agent_tasks[task_id].get("celery_restart"), "action_required": agent_tasks[task_id].get("action_required", False), "retryable": agent_tasks[task_id].get("retryable", False), "recommended_action": agent_tasks[task_id].get("recommended_action"), } @app.get("/api/agent/status/{task_id}") async def get_agent_status(task_id: str): """获取Agent任务执行状态""" if task_id not in agent_tasks: return { "success": False, "message": "任务不存在", "task_id": task_id } task = agent_tasks[task_id] # 如果有Celery任务ID，查询Celery状态 if CELERY_AVAILABLE and "celery_task_id" in task: try: from celery_app import app as celery_app celery_task = celery_app.AsyncResult(task["celery_task_id"]) if celery_task.state == 'PENDING': task["status"] = "pending" _apply_pending_watchdog(task, celery_task.state) elif celery_task.state == 'RUNNING': task["status"] = "running" # 更新进度信息 if celery_task.info: task["current_stage"] = celery_task.info.get('current_stage', task["current_stage"]) task["progress"] = celery_task.info.get('progress', task["progress"]) if 'logs' in celery_task.info: task["logs"] = celery_task.info['logs'] elif celery_task.state == 'SUCCESS': result = celery_task.result result_status = result.get("status") if isinstance(result, dict) else None result_success = result.get("success") if isinstance(result, dict) else None if result_status == "failed" or result_success is False: task["status"] = "failed" task["progress"] = 0 else: task["status"] = "success" task["progress"] = 100 if isinstance(result, dict): task["result"] = result.get('result', {}) if 'logs' in result: task["logs"] = result['logs'] task["current_stage"] = result.get("current_stage", task["current_stage"]) task["message"] = result.get("message", task["message"]) task["end_time"] = datetime.now().isoformat() elif celery_task.state == 'FAILURE': task["status"] = "failed" task["logs"].append({ "time": datetime.now().isoformat(), "level": "error", "message": f"Celery任务失败: {str(celery_task.result)}" }) task["end_time"] = datetime.now().isoformat() except Exception as e: print(f"[ERROR] Failed to query Celery status: {e}") _attach_agent_manifest_status(task) return { "success": True, "task": { "task_id": task["task_id"], "status": task["status"], "current_stage": task["current_stage"], "progress": task["progress"], "message": task["message"], "logs": task["logs"][-50:], # 返回最近50条日志 "result": task["result"], "completed_stages": task.get("completed_stages", task.get("stages_completed", [])), "stages_completed": task.get("stages_completed", []), "start_time": task.get("start_time"), "end_time": task.get("end_time"), "celery_queue": task.get("celery_queue"), "pending_seconds": task.get("pending_seconds"), "celery_queue_health": task.get("celery_queue_health"), "celery_restart": task.get("celery_restart"), "action_required": task.get("action_required", False), "retryable": task.get("retryable", False), "recommended_action": task.get("recommended_action") } } @app.get("/api/agent/tasks") async def list_agent_tasks(): """列出所有Agent任务""" tasks = [] for task_id, task in agent_tasks.items(): tasks.append({ "task_id": task["task_id"], "status": task["status"], "current_stage": task["current_stage"], "progress": task["progress"], "message": task["message"][:50] + "..." if len(task["message"]) > 50 else task["message"], "start_time": task.get("start_time") }) # 按时间倒序 tasks.sort(key=lambda x: x.get("start_time", ""), reverse=True) return { "success": True, "tasks": tasks[:20] # 返回最近20个任务 } @app.post("/api/agent/cancel/{task_id}") async def cancel_agent_task(task_id: str): """取消Agent任务（标记为取消，实际后台任务可能仍在运行）""" if task_id not in agent_tasks: return {"success": False, "message": "任务不存在"} if agent_tasks[task_id]["status"] in ["completed", "failed"]: return {"success": False, "message": "任务已结束，无法取消"} agent_tasks[task_id]["status"] = "cancelled" agent_tasks[task_id]["logs"].append({ "time": datetime.now().isoformat(), "level": "warning", "message": "任务已取消" }) return {"success": True, "message": "任务已标记为取消"} class AgentPlanRequest(BaseModel): """Agent规划请求""" message: str session_id: str = "" job_id: str = "" # 添加job_id字段，用于查询已完成阶段 prefix: str = "" history: list = [] intent: dict = {} # 前端可传入已有的intent，避免重复调用IntentAgentV2 SUBFAMILY_PREDICTION_PLAN_STAGES = [ "identify", "phylogeny", "motif", "domain", "structure", "combo", "reference_classification", ] SUBFAMILY_PREDICTION_STAGE_DEFAULTS = { "identify": { "name": "基因家族鉴定", "dependencies": [], "can_parallel": False, "estimated_time": 5, "params": {}, }, "phylogeny": { "name": "亚家族证据系统发育分析", "dependencies": ["identify"], "can_parallel": False, "estimated_time": 30, "params": { "analysisRoute": "subfamily_evidence", "subfamilyPipeline": True, }, }, "motif": { "name": "Motif分析", "dependencies": ["phylogeny"], "can_parallel": True, "estimated_time": 30, "params": { "analysisRoute": "subfamily_evidence", "subfamilyPipeline": True, }, }, "domain": { "name": "结构域分析", "dependencies": ["phylogeny"], "can_parallel": True, "estimated_time": 20, "params": { "analysisRoute": "subfamily_evidence", "subfamilyPipeline": True, }, }, "structure": { "name": "基因结构分析", "dependencies": ["phylogeny"], "can_parallel": True, "estimated_time": 20, "params": { "analysisRoute": "subfamily_evidence", "subfamilyPipeline": True, }, }, "combo": { "name": "亚家族证据组合图", "dependencies": ["motif", "domain", "structure"], "can_parallel": False, "estimated_time": 10, "params": { "analysisRoute": "subfamily_evidence", "subfamilyPipeline": True, }, }, "reference_classification": { "name": "参考亚家族分类", "dependencies": ["combo"], "can_parallel": False, "estimated_time": 5, "params": { "analysisRoute": "subfamily_prediction", "subfamilyPipeline": True, "perTargetEvidenceOnly": True, }, }, } def _normalize_subfamily_prediction_plan(plan: dict, intent: dict, raw_stages: list, stages_detail: list) -> tuple[list, list]: """Expose the real evidence chain for target-species subfamily classification.""" existing_details = { (item.get("id") or item.get("name")): item for item in (stages_detail or []) if isinstance(item, dict) } existing_stage_ids = [ item.get("id", item.get("name", "")) if isinstance(item, dict) else item for item in (raw_stages or []) ] stage_ids = list(SUBFAMILY_PREDICTION_PLAN_STAGES) aggregate_evidence_stages = {"genome_prepare", "genome_prepare_v2"} for sid in existing_stage_ids: if sid in aggregate_evidence_stages: continue if sid and sid not in stage_ids: stage_ids.append(sid) normalized_details = [] for order, sid in enumerate(stage_ids, start=1): default = SUBFAMILY_PREDICTION_STAGE_DEFAULTS.get(sid, {}) detail = dict(existing_details.get(sid) or {}) detail["id"] = sid detail["name"] = detail.get("name") or default.get("name") or sid detail["order"] = order detail["dependencies"] = list(default.get("dependencies", detail.get("dependencies", []))) detail["can_parallel"] = bool(default.get("can_parallel", detail.get("can_parallel", False))) detail["estimated_time"] = int(default.get("estimated_time", detail.get("estimated_time", 10))) detail_params = dict(detail.get("params") or {}) for key, value in (default.get("params") or {}).items(): detail_params.setdefault(key, value) detail["params"] = detail_params normalized_details.append(detail) estimated_minutes = sum(int(item.get("estimated_time", 10)) for item in normalized_details) plan["stages"] = stage_ids plan["stages_detail"] = normalized_details plan["total_stages"] = len(stage_ids) plan["estimated_time"] = f"{estimated_minutes}分钟" plan["estimated_total_time"] = f"{estimated_minutes}分钟" plan["description"] = "按每个待分类物种分别执行物种+参考亚家族预测，并生成对应树图和联合证据图" intent["stages"] = stage_ids intent["primary_stage"] = "reference_classification" intent["description"] = "按每个 target/comparative 待分类物种生成物种+参考亚家族树和联合证据图" return stage_ids, normalized_details def _build_contextual_gene_family_plan_response(request: Any, session_id: str, contextual_request: Dict[str, Any]) -> Dict[str, Any]: context_gate = contextual_request.get("contextGate") or {} context_issues = list(context_gate.get("issues") or []) prerequisite_stages, confirm_species = _contextual_prerequisite_stages(contextual_request) requested_stage_ids = list(contextual_request.get("stages") or []) subfamily_prerequisite_mode = any(stage in prerequisite_stages for stage in {"reference_classification", "subfamily_confirm"}) needs_subfamily_evidence_chain = "reference_classification" in prerequisite_stages if needs_subfamily_evidence_chain: stage_seed: list[str] = [] if "identify" in prerequisite_stages: stage_seed.append("identify") stage_seed.extend(["phylogeny", "motif", "domain", "structure", "combo", "reference_classification"]) if "subfamily_confirm" in prerequisite_stages: stage_seed.append("subfamily_confirm") stage_ids = list(dict.fromkeys(stage_seed)) elif subfamily_prerequisite_mode: stage_ids = list(dict.fromkeys(prerequisite_stages)) else: stage_ids = list(dict.fromkeys([*prerequisite_stages, *requested_stage_ids])) inserted_prerequisites = list(stage_ids) if subfamily_prerequisite_mode else [sid for sid in stage_ids if sid in prerequisite_stages] workflow_phase = "subfamily_prerequisite" if subfamily_prerequisite_mode else ("post_subfamily_confirm" if contextual_request.get("requiresConfirmedSubfamily") else "custom_scope") plan_params = { "prefix": contextual_request.get("prefix"), "geneFamily": contextual_request.get("family"), "targetSpecies": contextual_request.get("targetSpecies", []), "referenceSpecies": contextual_request.get("referenceSpecies", []), "comparativeSpecies": contextual_request.get("comparativeSpecies", []), "scopeMode": contextual_request.get("scopeMode", "target_only"), "contextRoute": contextual_request.get("context_route", ""), "subfamilyLabelSource": contextual_request.get("subfamily_label_source", ""), "requiresConfirmedSubfamily": bool(contextual_request.get("requiresConfirmedSubfamily")), "contextGate": contextual_request.get("contextGate", {}), "subfamilyConfirmSpecies": confirm_species, "workflowPhase": workflow_phase, "postConfirmReplan": bool(subfamily_prerequisite_mode and "subfamily_confirm" in stage_ids), "postConfirmMessage": getattr(request, "message", "") or "", "cpu": 8, } request_intent = getattr(request, "intent", {}) or {} request_intent_params = request_intent.get("params", {}) if isinstance(request_intent, dict) else {} if isinstance(request_intent_params, dict): contextual_identity_keys = ( "referenceSpecies", "reference_species", "refSpeciesList", "ppiReferenceSpecies", "referenceBundleSpecies", "ppiKaksReferenceSpecies", "scopeId", "scope_id", "scopeFile", "scope_file", "memberSetId", "member_set_id", "forceRerun", "force_rerun", "forceRerunStages", "force_rerun_stages", ) for key in contextual_identity_keys: if key in request_intent_params: plan_params[key] = request_intent_params[key] plan_prefix = str(plan_params.get("prefix") or request.prefix or "") family = str(plan_params.get("geneFamily") or "") classifiable_species = [] for sp in [*(contextual_request.get("targetSpecies") or []), *(contextual_request.get("comparativeSpecies") or [])]: if sp and sp not in classifiable_species: classifiable_species.append(sp) subfamily_by_species = (contextual_request.get("subfamilyState") or {}).get("by_species") or {} missing_identify_species_for_plan = [ sp for sp, data in subfamily_by_species.items() if str((data or {}).get("status") or "") == "missing_identify" ] classification_species_for_plan = [ sp for sp, data in subfamily_by_species.items() if str((data or {}).get("status") or "") in {"missing_identify", "not_classified", "confirmed_mismatch"} ] stages_detail = [] for stage_id in stage_ids: detail = _stage_detail_defaults(stage_id) detail_params = dict(plan_params) if stage_id == "identify" and missing_identify_species_for_plan: detail_params["targetSpecies"] = missing_identify_species_for_plan if needs_subfamily_evidence_chain and stage_id in {"phylogeny", "motif", "domain", "structure", "combo"}: default = SUBFAMILY_PREDICTION_STAGE_DEFAULTS.get(stage_id, {}) detail.update({ "name": default.get("name") or detail.get("name") or stage_id, "dependencies": list(default.get("dependencies") or detail.get("dependencies") or []), "estimated_time": default.get("estimated_time", detail.get("estimated_time", "10分钟")), }) detail_params["requiresConfirmedSubfamily"] = False detail_params["analysisRoute"] = "subfamily_evidence" detail_params["analysis_route"] = "subfamily_evidence" detail_params["subfamilyPipeline"] = True detail_params["referenceClassificationRequiresEvidence"] = True if stage_id == "combo": detail_params["comboPanels"] = ["motif", "domain", "structure"] detail_params["comboPanelSource"] = "subfamily_evidence" if stage_id in {"identify", "reference_classification", "subfamily_confirm"}: detail_params["requiresConfirmedSubfamily"] = False if stage_id == "reference_classification": detail.update({ "name": "单目标+参考亚家族分类与证据图", "dependencies": ["combo"] if needs_subfamily_evidence_chain else ["identify"], "estimated_time": "60分钟", }) if classification_species_for_plan: selected_set_for_plan = set(classification_species_for_plan) role_targets_for_plan = [ sp for sp in (contextual_request.get("targetSpecies") or []) if sp in selected_set_for_plan ] role_comps_for_plan = [ sp for sp in (contextual_request.get("comparativeSpecies") or []) if sp in selected_set_for_plan and sp not in role_targets_for_plan ] if not role_targets_for_plan: role_targets_for_plan = [sp for sp in classification_species_for_plan if sp not in role_comps_for_plan] detail_params["targetSpecies"] = role_targets_for_plan detail_params["comparativeSpecies"] = role_comps_for_plan detail_params["speciesQueue"] = classification_species_for_plan detail_params["scopeMode"] = _subfamily_scope_mode_for_roles(detail_params.get("referenceSpecies") or [], role_comps_for_plan) detail_params.update({ "analysisRoute": "subfamily_prediction", "subfamilyPipeline": True, "perTargetEvidenceOnly": True, }) if stage_id == "subfamily_confirm": detail.update({ "name": "亚家族标签确认", "dependencies": ["reference_classification"] if "reference_classification" in stage_ids else [], "estimated_time": "人工确认", }) detail_params.update({ "analysisRoute": "confirmed_subfamily", "analysis_route": "confirmed_subfamily", "sourcePredictionRoute": "subfamily_prediction", "source_prediction_route": "subfamily_prediction", "interactiveSubfamilyReview": True, "subfamilyConfirmSpecies": confirm_species or contextual_request.get("targetSpecies", []), "geneFamily": family, }) if stage_id == "phylogeny": # Phylogeny never depends on subfamily prediction/confirmation. # Subfamily review happens after reference_classification. if "identify" in stage_ids: detail["dependencies"] = ["identify"] if not needs_subfamily_evidence_chain and contextual_request.get("scopeMode") != "target_only" and contextual_request.get("context_route") in {"custom_phylogeny_scope", "plain_phylogeny"}: detail_params["analysisRoute"] = "custom_phylogeny" detail["params"] = enrich_stage_params_with_context(stage_id, plan_prefix, detail_params) stages_detail.append(detail) if _looks_like_gene_family_workflow_query(getattr(request, "message", "") or "") and not subfamily_prerequisite_mode: stage_ids, stages_detail = _normalize_full_gene_family_workflow_plan( message=getattr(request, "message", "") or "", stage_ids=stage_ids, stages_detail=stages_detail, plan_params=plan_params, plan_prefix=plan_prefix, ) completed_stages: list[str] = [] missing_identify_species = { sp for sp, data in ((contextual_request.get("subfamilyState") or {}).get("by_species") or {}).items() if str((data or {}).get("status") or "") == "missing_identify" } if (context_gate.get("evidence") or {}).get("family_members") and not missing_identify_species and "identify" not in inserted_prerequisites: completed_stages.append("identify") if request.job_id and stage_ids: force_rerun_stage_ids = set(_param_list( plan_params.get("forceRerunStages") or plan_params.get("force_rerun_stages") )) if plan_params.get("forceRerun") or plan_params.get("force_rerun"): force_rerun_stage_ids.update(stage_ids) completed_stages.extend(_identity_completed_stages_for_plan( request.job_id, stage_ids, plan_params, stages_detail, effective_prefix=plan_prefix, request_prefix=request.prefix, force_rerun_stage_ids=force_rerun_stage_ids, )) completed_stages = list(dict.fromkeys(completed_stages)) completed_set = set(completed_stages) # Contextual/free-form gene-family plans do not have a separate blocked-stage # filter like saved-methodology plans. Treat the normalized stage list as the # executable candidate set and remove completed stages below. executable_stage_ids = list(stage_ids) executable_stage_set = set(executable_stage_ids) remaining_stage_ids = [stage_id for stage_id in stage_ids if stage_id not in completed_set and stage_id in executable_stage_set] visible_remaining_stage_ids = [stage_id for stage_id in stage_ids if stage_id not in completed_set] remaining_stages_detail = [ item for item in stages_detail if isinstance(item, dict) and (item.get("id") or item.get("name")) in remaining_stage_ids ] remaining_prerequisite_stages = [sid for sid in inserted_prerequisites if sid in remaining_stage_ids] if not remaining_stage_ids and completed_stages: description = "所有阶段已完成" elif remaining_prerequisite_stages: description = "已把缺失的基因家族成员鉴定、亚家族分类/确认接入执行计划" elif inserted_prerequisites: description = "前置阶段已在同一分析范围完成，继续执行请求阶段" elif contextual_request.get("requiresConfirmedSubfamily"): description = "基于已确认亚家族标签的传统基因家族分析" else: description = "自定义范围基因家族分析" estimated_minutes = sum(int(str(item.get("estimated_time", "10分钟")).replace("分钟", "") or 10) if str(item.get("estimated_time", "")).replace("分钟", "").isdigit() else 10 for item in remaining_stages_detail) intent = { "gene_family": contextual_request.get("family"), "species": "Multi" if len(contextual_request.get("targetSpecies", [])) != 1 or contextual_request.get("referenceSpecies") else (contextual_request.get("targetSpecies") or [""])[0], "target_species": contextual_request.get("targetSpecies", []), "reference_species": contextual_request.get("referenceSpecies", []), "comparative_species": contextual_request.get("comparativeSpecies", []), "scopeMode": contextual_request.get("scopeMode", "target_only"), "stages": stage_ids, "workflowPhase": workflow_phase, "description": description, "confidence": 0.99, "prefix": plan_prefix, "params": dict(plan_params), } requirements_report = {} if request.job_id and stage_ids: try: from gfa_core.mas.stage_requirements import resolve_many_stage_requirements params_by_stage = {} for detail in stages_detail: if not isinstance(detail, dict): continue sid = str(detail.get("id") or detail.get("name") or "").strip() if sid: params_by_stage[sid] = dict(detail.get("params") or {}) requirements_report = resolve_many_stage_requirements( stages=stage_ids, prefix=plan_prefix, job_id=request.job_id, params_by_stage=params_by_stage, shared_params=plan_params, project_root=PROJECT_ROOT, shadow_only=True, ) if requirements_report: print(f"[Agent Plan] Contextual requirements shadow report stages: {list(requirements_report.keys())}") except Exception as exc: requirements_report = {"error": str(exc), "shadow_only": True} print(f"[Agent Plan] Contextual requirements shadow report skipped: {exc}") return { "success": True, "mode": "deterministic_context", "status": "needs_prerequisite_execution" if remaining_prerequisite_stages else "ready", "message": description, "context_gate": context_gate, "issues": context_issues, "suggested_stages": remaining_prerequisite_stages, "plain_phylogeny_hint": "如确认不需要亚家族标注，可输入：不需要亚家族，直接绘制普通进化树。" if remaining_prerequisite_stages and _looks_like_phylogeny_query(request.message or "") else "", "plan": { "analysis_type": "gene_family", "gene_family": contextual_request.get("family"), "species": intent["species"], "target_species": contextual_request.get("targetSpecies", []), "reference_species": contextual_request.get("referenceSpecies", []), "comparative_species": contextual_request.get("comparativeSpecies", []), "scopeMode": contextual_request.get("scopeMode", "target_only"), "workflowPhase": workflow_phase, "postConfirmReplan": bool(subfamily_prerequisite_mode and "subfamily_confirm" in stage_ids), "postConfirmMessage": getattr(request, "message", "") or "", "prefix": plan_prefix, "description": description, "estimated_time": f"{estimated_minutes}分钟", "estimated_total_time": f"{estimated_minutes}分钟", "execution_strategy": "parallel", "max_parallel": 4, "total_stages": len(remaining_stage_ids), "stages": remaining_stage_ids, "stages_detail": remaining_stages_detail, "all_stages": stage_ids, "all_stages_detail": stages_detail, "parallel_groups": [], "params": plan_params, "warnings": context_issues, "prerequisite_stages": remaining_prerequisite_stages, "subfamily_confirm_species": confirm_species, "requirements_report": requirements_report, }, "intent": intent, "completed_stages": list(dict.fromkeys(completed_stages)), "session_id": session_id, } def _load_methodology_plan_for_agent_request(request: AgentPlanRequest) -> tuple[dict[str, Any] | None, dict[str, Any], dict[str, Any]]: intent = request.intent if isinstance(request.intent, dict) else {} params = intent.get("params") if isinstance(intent.get("params"), dict) else {} wants_methodology = bool(params.get("methodology") or intent.get("methodology")) message = str(request.message or "") methodology_message_intent = is_methodology_execution_intent(message) if not wants_methodology and not methodology_message_intent: return None, intent, params effective_job_id = str(request.job_id or "").strip() if not effective_job_id and current_job_id and current_job_id != "default_job": effective_job_id = current_job_id try: request.job_id = effective_job_id except Exception: pass plan = None plan_path = str(params.get("methodologyPlanPath") or params.get("methodology_plan_path") or "").strip() if plan_path: try: path = Path(plan_path).expanduser() if not path.is_absolute(): path = PROJECT_ROOT / path if path.exists(): plan = json.loads(path.read_text(encoding="utf-8")) except Exception as exc: print(f"[Agent Plan] Failed to load methodology plan path {plan_path}: {exc}") if plan is None and effective_job_id: try: plan = load_bound_methodology_plan(INPUT_ROOT, effective_job_id) except Exception as exc: print(f"[Agent Plan] Failed to load bound methodology plan: {exc}") if plan is None: return {}, {**intent, "methodology": True, "source": "methodology"}, params return plan, intent, params def _methodology_species_alias_map(job_id: str) -> dict[str, str]: aliases: dict[str, str] = dict(_species_alias_map_for_job(job_id)) config_path = INPUT_ROOT / job_id / "config.xlsx" if job_id else None if not config_path or not config_path.exists(): return aliases try: import pandas as pd df = pd.read_excel(config_path).fillna("") except Exception as exc: print(f"[Agent Plan] Methodology config alias read skipped: {exc}") return aliases def add(alias: str, code: str) -> None: alias = re.sub(r"[^A-Za-z0-9]+", " ", str(alias or "")).strip().lower() code = str(code or "").strip() if alias and code: aliases[alias] = code for _, row in df.iterrows(): code = str(row.get("Species_Code") or row.get("Species") or "").strip() if not code: continue add(code, code) for key in ("Species_Name", "Scientific_Name", "Latin_Name", "species_name", "scientific_name", "latin_name"): add(row.get(key), code) filename = str(row.get("Original_File_Name") or "").strip() if filename: stem = Path(filename).name stem = re.sub(r"\.(?:gz|zip|fa|fasta|faa|fna|ffn|gff3?|gtf|cds|pep|all|chr|toplevel|dna|protein)+$", "", stem, flags=re.I) parts = re.split(r"[._-]+", stem) if len(parts) >= 2: add(" ".join(parts[:2]), code) add("_".join(parts[:2]), code) return aliases def _canonicalize_methodology_species(job_id: str, values: Any) -> list[str]: if values is None: raw_values = [] elif isinstance(values, str): raw_values = [item.strip() for item in re.split(r"[,;，；、]+", values) if item.strip()] elif isinstance(values, (list, tuple, set)): raw_values = [str(item).strip() for item in values if str(item).strip()] else: raw_values = [str(values).strip()] if str(values).strip() else [] if not raw_values: return [] aliases = _methodology_species_alias_map(job_id) result: list[str] = [] for raw in raw_values: key = re.sub(r"[^A-Za-z0-9]+", " ", str(raw or "")).strip().lower() mapped = aliases.get(key) if not mapped: mapped_list = _canonicalize_species_codes_for_job(job_id, [str(raw or "")]) mapped = mapped_list[0] if mapped_list else str(raw or "").strip() if mapped and mapped not in result: result.append(mapped) return result def _methodology_execution_prefix(job_id: str, request_prefix: str, targets: list[str], family: str) -> str: """Build the execution prefix from config Species_Code, not display/Latin names.""" family = _nonempty_prefix(family) raw_prefix = str(request_prefix or "").strip() if raw_prefix and "." in raw_prefix: raw_species, raw_family = _prefix_species_family(raw_prefix) mapped = _canonicalize_methodology_species(job_id, [raw_species]) if raw_species else [] if mapped and raw_family: return f"{mapped[0]}.{_nonempty_prefix(raw_family)}" if len(targets) == 1 and targets[0] and family: return f"{targets[0]}.{family}" return f"Target.{family}" if family else raw_prefix def _methodology_unresolved_species(values: list[str]) -> list[str]: """Species labels containing whitespace are display names, not executable config codes.""" return [str(item).strip() for item in values or [] if re.search(r"\s", str(item or "").strip())] METHODOLOGY_SAVED_PLAN_STAGE_ORDER = [ "genome_prepare", "identify", "phylogeny", "motif", "domain", "structure", "combo", "reference_classification", "subfamily_confirm", "pep", "synteny", "interspecies_synteny", "karyotype", "promoter", "tfbs", "expression", "rnaseq_qc", "rnaseq_align", "rnaseq_quant", "rnaseq_sample_qc", "rnaseq_de", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea", "rnaseq_wgcna", "ortholog", "kaks", "ppi", ] METHODOLOGY_SAVED_PLAN_SUPPORTED_STAGES = set(METHODOLOGY_SAVED_PLAN_STAGE_ORDER) METHODOLOGY_SUBFAMILY_EVIDENCE_STAGES = {"phylogeny", "motif", "domain", "structure", "combo"} METHODOLOGY_SUBFAMILY_COMBO_PANELS = ["motif", "domain", "structure"] METHODOLOGY_ARTICLE_COMBO_PANEL_ORDER = ["motif", "domain", "structure"] METHODOLOGY_SUBFAMILY_GATE_STAGES = {"identify", *METHODOLOGY_SUBFAMILY_EVIDENCE_STAGES, "reference_classification", "subfamily_confirm"} METHODOLOGY_PREREQUISITE_STAGES = {"genome_prepare", "identify", "reference_classification", "subfamily_confirm"} METHODOLOGY_SUBFAMILY_INFERRED_LITERATURE_PREFIXES = ("inferred_subfamily_route", "methodology_subfamily_route") METHODOLOGY_RAW_RNASEQ_STAGES = { "rnaseq_qc", "rnaseq_align", "rnaseq_quant", "rnaseq_sample_qc", "rnaseq_de", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea", "rnaseq_wgcna", } METHODOLOGY_RAW_RNASEQ_BLOCK_REASON = "raw_rnaseq_requires_user_consent" METHODOLOGY_DISPLAY_STAGE_LABELS = { "genome_prepare": "数据准备：基因组文件标准化", "identify": "数据准备：基因家族鉴定", "reference_classification": "亚家族分类预测（Si+At 证据链）", "subfamily_confirm": "亚家族分类人工确认", "phylogeny": "系统发育分析（Si+At，多物种；使用确认亚家族标签）", "motif": "Motif分析（使用确认亚家族标签）", "domain": "结构域分析（使用确认亚家族标签）", "structure": "基因结构分析（使用确认亚家族标签）", "combo": "组合图（使用确认亚家族标签）", "synteny": "种内共线性分析", "interspecies_synteny": "种间共线性分析（Si vs At；含种间进化树）", "karyotype": "核型/染色体分布分析", "expression": "组织表达分析（TPM/SesameFG；缺矩阵则提示）", } METHODOLOGY_WORKFLOW_PHASES = [ ("data_preparation", "数据准备", ["genome_prepare", "identify"]), ("subfamily_classification", "亚家族分类确认", ["phylogeny", "motif", "domain", "structure", "combo", "reference_classification", "subfamily_confirm"]), ("post_subfamily_family_analysis", "确认后常规分析", ["phylogeny", "motif", "domain", "structure", "combo", "synteny", "interspecies_synteny", "karyotype"]), ("expression_rnaseq", "表达/RNA-seq 数据门控", ["expression"]), ] METHODOLOGY_DISPLAY_STAGE_ORDER = ["genome_prepare", "identify", "reference_classification", "subfamily_confirm", "phylogeny", "motif", "domain", "structure", "combo", "synteny", "interspecies_synteny", "karyotype", "expression"] METHODOLOGY_POST_CONFIRM_CORE_STAGES = ["phylogeny", "motif", "domain", "structure", "combo"] def _methodology_execution_route_stage_order(plan: dict[str, Any]) -> list[str]: route = plan.get("execution_routes") if isinstance(route, dict): return [str(stage).strip() for stage in (route.get("stage_order") or []) if str(stage).strip()] if isinstance(route, list): for item in route: if isinstance(item, dict) and isinstance(item.get("stage_order"), list): return [str(stage).strip() for stage in item.get("stage_order") if str(stage).strip()] return [] def _methodology_stage_is_inferred_subfamily(stage: dict[str, Any]) -> bool: literature = str(stage.get("literature_stage") or stage.get("observation_id") or "").strip().lower() data_type = str(stage.get("data_source_type") or "").strip().lower() route = str(stage.get("analysis_route") or stage.get("analysisRoute") or "").strip() return ( any(literature.startswith(prefix) for prefix in METHODOLOGY_SUBFAMILY_INFERRED_LITERATURE_PREFIXES) or data_type == "subfamily_evidence_chain" or route in {"subfamily_prediction", "subfamily_evidence"} ) def _methodology_independent_stage_ids(plan: dict[str, Any]) -> set[str]: independent: set[str] = set() for stage in plan.get("stages") or []: if not isinstance(stage, dict) or _methodology_stage_is_inferred_subfamily(stage): continue stage_id = str(stage.get("bbav5_stage_id") or "").strip() if stage_id in METHODOLOGY_SAVED_PLAN_SUPPORTED_STAGES: independent.add(stage_id) preview = _methodology_preview_from_saved_plan(plan) for stage in preview.get("suggested_stages", []) or []: if not isinstance(stage, dict): continue if _methodology_stage_is_inferred_subfamily(stage): continue stage_id = str(stage.get("bbav5_stage_id") or stage.get("stage_id") or "").strip() if stage_id and stage_id != "external_or_manual" and stage_id in METHODOLOGY_SAVED_PLAN_SUPPORTED_STAGES: independent.add(stage_id) return independent def _methodology_plan_has_explicit_subfamily(plan: dict[str, Any]) -> bool: haystack: list[Any] = [plan.get("stages") or [], _methodology_preview_from_saved_plan(plan).get("suggested_stages") or []] text = json.dumps(haystack, ensure_ascii=False).lower() return any(token in text for token in ["subfamily", "亚家族", "reference_classification", "subfamily_classification"]) def _methodology_sanitize_saved_route_stage_ids(plan: dict[str, Any], stage_ids: list[str]) -> list[str]: independent = _methodology_independent_stage_ids(plan) explicit_subfamily = _methodology_plan_has_explicit_subfamily(plan) sanitized: list[str] = [] for stage_id in stage_ids or []: if stage_id == "domain" and stage_id not in independent: continue if stage_id in {"reference_classification", "subfamily_confirm"} and stage_id not in independent and not explicit_subfamily: continue sanitized.append(stage_id) if "combo" in sanitized: panel_count = len([panel for panel in ("motif", "domain", "structure") if panel in sanitized]) if panel_count < 2: sanitized = [stage for stage in sanitized if stage != "combo"] return _methodology_order_stages(sanitized) def _methodology_combo_panels_from_saved_plan(plan: dict[str, Any], stage_ids: list[str]) -> list[str]: """Resolve article-methodology combo panels only; subfamily evidence uses a fixed 3-panel contract.""" independent = _methodology_independent_stage_ids(plan) if "combo" not in independent: return [] panels = [panel for panel in METHODOLOGY_ARTICLE_COMBO_PANEL_ORDER if panel in independent] if len(panels) >= 2: return panels metadata = _methodology_stage_metadata_from_preview(plan) for source in (metadata.get("combo") or {}, plan): if not isinstance(source, dict): continue params = source.get("method_execution_parameters") or source.get("bbav5_default_parameters") or source.get("params") or source if not isinstance(params, dict): continue raw = params.get("comboPanels") or params.get("combo_panels") if not raw: continue values = raw if isinstance(raw, list) else re.split(r"[,;，；\s]+", str(raw)) panels = [panel for panel in METHODOLOGY_ARTICLE_COMBO_PANEL_ORDER if panel in {str(item).strip() for item in values}] if len(panels) >= 2: return panels return [] def _methodology_target_stage_ids(plan: dict[str, Any]) -> list[str]: candidates: list[str] = [] for stage in plan.get("stages") or []: if not isinstance(stage, dict) or _methodology_stage_is_inferred_subfamily(stage): continue stage_id = str(stage.get("bbav5_stage_id") or "").strip() if stage_id and stage_id in METHODOLOGY_SAVED_PLAN_SUPPORTED_STAGES and stage_id not in METHODOLOGY_PREREQUISITE_STAGES: candidates.append(stage_id) preview = _methodology_preview_from_saved_plan(plan) for stage in preview.get("suggested_stages", []) or []: if not isinstance(stage, dict): continue if _methodology_stage_is_inferred_subfamily(stage): continue stage_id = str(stage.get("bbav5_stage_id") or stage.get("stage_id") or "").strip() if not stage_id or stage_id == "external_or_manual" or stage_id not in METHODOLOGY_SAVED_PLAN_SUPPORTED_STAGES: continue candidates.append(stage_id) for fallback in stage.get("fallback_stages") or []: fallback_id = str(fallback or "").strip() if fallback_id in METHODOLOGY_SAVED_PLAN_SUPPORTED_STAGES: candidates.append(fallback_id) independent = _methodology_independent_stage_ids(plan) for stage_id in _methodology_execution_route_stage_order(plan): if stage_id not in METHODOLOGY_SAVED_PLAN_SUPPORTED_STAGES or stage_id in METHODOLOGY_PREREQUISITE_STAGES: continue # Domain may appear in the fixed subfamily-evidence route. Do not promote # that inferred gate stage into the post-confirmation article workflow # unless the saved methodology has an independent domain analysis. if stage_id == "domain" and stage_id not in independent: continue candidates.append(stage_id) return _methodology_order_stages(candidates) def _methodology_expression_blocked_warnings(deferred: list[dict[str, Any]], expression_status: dict[str, Any], stage_metadata: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: warnings: list[dict[str, Any]] = [] if not deferred: return warnings for item in deferred: stage_id = str(item.get("stage") or "").strip() metadata = stage_metadata.get(stage_id) or {} warnings.append({ "code": "expression_data_missing" if item.get("reason") == "missing_expression_matrix_or_fastq_samples_yaml" else "expression_route_deferred", "stage": stage_id, "stage_label": _stage_detail_defaults(stage_id).get("name", stage_id), "reason": item.get("reason", ""), "analysis_context": metadata.get("analysis_context", ""), "analysis_route": metadata.get("analysis_route", ""), "data_source_type": metadata.get("data_source_type", ""), "accessions": metadata.get("accessions", []), "required_uploads": ( ["expression matrix (FPKM/TPM/count)", "or FASTQ/FQ files plus samples.yaml"] if item.get("reason") == "missing_expression_matrix_or_fastq_samples_yaml" else (["explicit user consent for raw RNA-seq download", "FASTQ/FQ files plus samples.yaml before execution"] if item.get("reason") == "raw_rnaseq_requires_user_consent" else []) ), "expression_data_status": expression_status, }) return warnings def _methodology_order_stages(stage_ids: list[str]) -> list[str]: seen: set[str] = set() cleaned = [str(stage or "").strip() for stage in stage_ids or [] if str(stage or "").strip()] unique = [stage for stage in cleaned if stage not in seen and not seen.add(stage)] return sorted( unique, key=lambda stage: ( METHODOLOGY_SAVED_PLAN_STAGE_ORDER.index(stage) if stage in METHODOLOGY_SAVED_PLAN_STAGE_ORDER else len(METHODOLOGY_SAVED_PLAN_STAGE_ORDER), stage, ), ) def _methodology_preview_from_saved_plan(plan: dict[str, Any]) -> dict[str, Any]: preview = plan.get("reproduction_plan_preview") if isinstance(plan.get("reproduction_plan_preview"), dict) else {} if preview: return preview batch_dir = str(plan.get("source_batch_dir") or "").strip() if not batch_dir: return {} try: path = Path(batch_dir).expanduser() / "reproduction_plan_preview.json" if path.exists(): data = json.loads(path.read_text(encoding="utf-8")) return data if isinstance(data, dict) else {} except Exception as exc: print(f"[Agent Plan] Failed to load methodology reproduction preview: {exc}") return {} def _methodology_stage_metadata_from_preview(plan: dict[str, Any]) -> dict[str, dict[str, Any]]: metadata: dict[str, dict[str, Any]] = {} preview = _methodology_preview_from_saved_plan(plan) for preview_stage in preview.get("suggested_stages", []) or []: if not isinstance(preview_stage, dict): continue stage_id = str(preview_stage.get("bbav5_stage_id") or preview_stage.get("stage_id") or "").strip() if not stage_id or stage_id == "external_or_manual" or stage_id not in METHODOLOGY_SAVED_PLAN_SUPPORTED_STAGES: continue metadata.setdefault(stage_id, preview_stage) for fallback in preview_stage.get("fallback_stages", []) or []: fallback_id = str(fallback or "").strip() if fallback_id in METHODOLOGY_SAVED_PLAN_SUPPORTED_STAGES: metadata.setdefault(fallback_id, {**preview_stage, "bbav5_stage_id": fallback_id, "stage_id": fallback_id, "source_methodology_stage": stage_id}) for stage in [*(plan.get("stages", []) or []), *(plan.get("methodology_route_stages", []) or [])]: if not isinstance(stage, dict): continue stage_id = str(stage.get("bbav5_stage_id") or "").strip() if stage_id and stage_id in METHODOLOGY_SAVED_PLAN_SUPPORTED_STAGES: metadata.setdefault(stage_id, stage) if isinstance(stage.get("bbav5_default_parameters"), dict): metadata[stage_id].setdefault("method_execution_parameters", stage.get("bbav5_default_parameters")) return metadata def _methodology_saved_plan_needs_subfamily(plan: dict[str, Any], stage_ids: list[str], reference_species: list[str]) -> bool: return True def _methodology_saved_plan_stage_ids(plan: dict[str, Any], method_params: dict[str, Any], reference_species: list[str]) -> tuple[list[str], dict[str, dict[str, Any]], bool]: route_stages = [str(stage).strip() for stage in (method_params.get("requestedMethodologyStages") or []) if str(stage).strip()] metadata = _methodology_stage_metadata_from_preview(plan) preview_stages = list(metadata.keys()) paper_stages = _methodology_sanitize_saved_route_stage_ids(plan, _methodology_order_stages([*route_stages, *preview_stages])) stage_ids = [ "identify", "phylogeny", "motif", "domain", "structure", "combo", "reference_classification", "subfamily_confirm", *paper_stages, ] if "genome_prepare" in paper_stages: stage_ids.insert(0, "genome_prepare") stage_ids = _methodology_order_stages(stage_ids) return stage_ids, metadata, True def _methodology_expression_data_status(job_id: str) -> dict[str, Any]: input_dir = INPUT_ROOT / job_id if job_id else INPUT_ROOT matrix_patterns = [ "*fpkm*", "*FPKM*", "*tpm*", "*TPM*", "*count*matrix*", "*counts*", "*expression*matrix*", "*expr*matrix*" ] fastq_exts = (".fastq", ".fastq.gz", ".fq", ".fq.gz") matrix_files: list[str] = [] fastq_files: list[str] = [] if input_dir.exists(): for pattern in matrix_patterns: for path in input_dir.rglob(pattern): if path.is_file() and path.name != "config.xlsx" and str(path) not in matrix_files: matrix_files.append(str(path)) for path in input_dir.rglob("*"): if path.is_file() and path.name.lower().endswith(fastq_exts): fastq_files.append(str(path)) samples_yaml = input_dir / "samples.yaml" has_matrix = bool(matrix_files) has_fastq = bool(fastq_files) and samples_yaml.exists() return { "has_matrix": has_matrix, "has_fastq": has_fastq, "matrix_files": matrix_files[:20], "fastq_files": fastq_files[:20], "samples_yaml": str(samples_yaml) if samples_yaml.exists() else "", "route": "matrix_expression" if has_matrix else ("raw_rnaseq" if has_fastq else "missing_expression_data"), } def _methodology_filter_expression_stages_for_data( stage_ids: list[str], expression_status: dict[str, Any], stage_metadata: dict[str, dict[str, Any]], ) -> tuple[list[str], list[dict[str, Any]]]: expression_related = {"expression", *METHODOLOGY_RAW_RNASEQ_STAGES} deferred: list[dict[str, Any]] = [] route = expression_status.get("route") if route == "matrix_expression": keep = [stage for stage in stage_ids if not stage.startswith("rnaseq_")] for stage in stage_ids: if stage.startswith("rnaseq_"): deferred.append({"stage": stage, "reason": "expression_matrix_available_use_expression_stage"}) return keep, deferred if route == "raw_rnaseq": keep = [stage for stage in stage_ids if stage != "expression"] if "expression" in stage_ids: deferred.append({"stage": "expression", "reason": "raw_fastq_available_use_rnaseq_pipeline"}) return keep, deferred keep: list[str] = [] for stage in stage_ids: if stage not in expression_related: keep.append(stage) continue metadata = stage_metadata.get(stage) or stage_metadata.get("expression") or {} metadata_route = str(metadata.get("analysis_route") or metadata.get("analysisRoute") or "").strip() data_source_type = str(metadata.get("data_source_type") or "").strip() accessions = metadata.get("accessions") or metadata.get("geo_accessions") or [] if isinstance(accessions, str): accessions = [accessions] has_geo_expression = stage == "expression" and any(re.match(r"^GS[EM]\d+", str(item).strip(), flags=re.IGNORECASE) for item in accessions) if stage == "expression" and ( (metadata_route == "matrix_reuse" and data_source_type in {"matrix_database", "expression_matrix", "fpkm_matrix", "tpm_matrix", "count_matrix"}) or has_geo_expression ): keep.append(stage) continue if stage == "expression" and (data_source_type == "raw_accession" or metadata_route.startswith("raw_rnaseq")): keep.append(stage) deferred.append({"stage": stage, "reason": "raw_rnaseq_requires_user_consent", "blocking": False}) elif stage == "expression": keep.append(stage) deferred.append({"stage": stage, "reason": "missing_expression_matrix_or_fastq_samples_yaml", "blocking": False}) elif data_source_type == "raw_accession" or metadata_route.startswith("raw_rnaseq") or stage.startswith("rnaseq_"): deferred.append({"stage": stage, "reason": "raw_rnaseq_requires_user_consent", "blocking": True}) else: deferred.append({"stage": stage, "reason": "missing_expression_matrix_or_fastq_samples_yaml", "blocking": True}) return keep, deferred def _methodology_confirmed_subfamily_completed(job_id: str, plan_params: dict[str, Any]) -> bool: if not job_id: return False final_dir = WORKSPACE_ROOT / job_id / "final" if not final_dir.exists(): return False family = str((plan_params or {}).get("geneFamily") or "").strip() targets = set(_param_list((plan_params or {}).get("targetSpecies"))) def matches_name(path: Path) -> bool: name = path.name if family and family not in name: return False if targets and not any(target in name for target in targets): return False return True for path in final_dir.glob("*confirmed_subfamily.tsv"): if path.is_file() and path.stat().st_size > 0 and matches_name(path): return True for manifest_path in final_dir.glob("*.subfamily_confirm.manifest.json"): if not matches_name(manifest_path): continue try: data = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: continue if str(data.get("status") or "").lower() not in {"success", "partial_success", "completed"}: continue artifacts = data.get("artifacts") or {} if isinstance(artifacts, dict): for artifact in artifacts.values(): if not isinstance(artifact, dict): continue value = artifact.get("full_path") or artifact.get("path") or artifact.get("file") if value and "confirmed_subfamily" in str(value): candidate = Path(str(value)) if not candidate.is_absolute(): candidate = final_dir / candidate.name if candidate.exists() and candidate.stat().st_size > 0: return True return False def _methodology_success_manifest_completed(job_id: str, stage_id: str, plan_params: dict[str, Any]) -> bool: if not job_id or not stage_id: return False final_dir = WORKSPACE_ROOT / job_id / "final" if not final_dir.exists(): return False family = str((plan_params or {}).get("geneFamily") or "").strip() targets = set(_param_list((plan_params or {}).get("targetSpecies"))) refs = set(_param_list((plan_params or {}).get("referenceSpecies"))) for manifest_path in final_dir.glob(f"*.{stage_id}.manifest.json"): try: data = json.loads(manifest_path.read_text(encoding="utf-8")) except Exception: continue status = str(data.get("status") or data.get("state") or "").lower() if status and status not in {"success", "completed"}: continue manifest_stage = str(data.get("stage") or stage_id) if manifest_stage != stage_id: continue if stage_id == "genome_prepare": return True scope_id = str(data.get("scope_id") or data.get("scopeId") or "") manifest_prefix = str(data.get("prefix") or "") name = manifest_path.name haystack = " ".join([scope_id, manifest_prefix, name]) if family and family not in haystack: continue if targets and not any(target in haystack for target in targets): if stage_id != "genome_prepare": continue if refs and stage_id not in {"identify", "genome_prepare"} and not any(ref in haystack for ref in refs): continue return True return False def _methodology_blast_evalue_from_metadata(metadata: dict[str, Any]) -> float | None: candidates = [ metadata.get("method_execution_parameters"), metadata.get("bbav5_default_parameters"), metadata.get("parameters"), ] for params in candidates: if not isinstance(params, dict): continue blast = params.get("blast") if isinstance(params.get("blast"), dict) else {} value = blast.get("evalue") or blast.get("e_value") or params.get("blastEvalue") or params.get("blast_evalue") if value in (None, ""): continue try: return float(str(value).strip()) except ValueError: continue return None def _methodology_blast_identity_from_metadata(metadata: dict[str, Any]) -> float | None: candidates = [ metadata.get("method_execution_parameters"), metadata.get("bbav5_default_parameters"), metadata.get("parameters"), ] for params in candidates: if not isinstance(params, dict): continue blast = params.get("blast") if isinstance(params.get("blast"), dict) else {} value = blast.get("identityThreshold") or blast.get("identity_threshold") or params.get("blastIdentity") or params.get("blast_identity") if value in (None, ""): continue text = str(value).strip().rstrip("%") try: number = float(text) except ValueError: continue return number * 100.0 if 0 < number <= 1 else number return None def _methodology_saved_plan_dependencies(stage_id: str, stage_ids: list[str], *, needs_subfamily: bool = False, combo_panels: list[str] | None = None) -> list[str]: present = set(stage_ids or []) combo_dependency_panels = [str(panel).strip() for panel in (combo_panels or METHODOLOGY_SUBFAMILY_COMBO_PANELS) if str(panel).strip()] deps_by_stage = { "genome_prepare": [], "identify": ["genome_prepare"] if "genome_prepare" in present else [], "phylogeny": ["identify"], "motif": ["phylogeny"], "domain": ["phylogeny"], "structure": ["phylogeny"], "combo": [stage for stage in combo_dependency_panels if stage in present], "reference_classification": ["combo"] if "combo" in present else [stage for stage in ["phylogeny", "motif", "domain", "structure"] if stage in present], "subfamily_confirm": ["reference_classification"], "pep": ["identify"], "synteny": ["identify"], "interspecies_synteny": ["identify"], "karyotype": [stage for stage in ["identify", "synteny"] if stage in present], "promoter": ["identify", "phylogeny"], "tfbs": [stage for stage in ["genome_prepare", "identify"] if stage in present], "expression": ["identify"], "rnaseq_qc": [], "rnaseq_align": ["rnaseq_qc"], "rnaseq_quant": ["rnaseq_align"], "rnaseq_sample_qc": ["rnaseq_quant"], "rnaseq_de": ["rnaseq_sample_qc"], "rnaseq_go_enrich": ["rnaseq_de"], "rnaseq_kegg_enrich": ["rnaseq_de"], "rnaseq_fgsea": [stage for stage in ["rnaseq_de", "rnaseq_kegg_enrich"] if stage in present], "rnaseq_wgcna": ["rnaseq_sample_qc"], "ortholog": ["identify"], "kaks": ["identify", "ortholog"], "ppi": ["identify"], } deps = [dep for dep in deps_by_stage.get(stage_id, []) if dep in present] return deps def _methodology_downstream_route(target_species: list[str], reference_species: list[str], comparative_species: list[str], shared_scope: str) -> str: if reference_species or comparative_species or len(target_species or []) > 1 or str(shared_scope or "").startswith("target_plus"): return "multi_species" return "single_species" def _methodology_confirmed_node_id(stage_id: str, existing_ids: set[str]) -> str: base = f"confirmed_{stage_id}" if base not in existing_ids: return base idx = 2 while f"{base}_{idx}" in existing_ids: idx += 1 return f"{base}_{idx}" def _methodology_append_confirmed_downstream_nodes( stages_detail: list[dict[str, Any]], *, post_confirm_targets: list[str], plan_params: dict[str, Any], prefix: str, target_species: list[str], reference_species: list[str], comparative_species: list[str], shared_scope: str, article_combo_panels: list[str], blocked_stage_ids: list[str], ) -> list[dict[str, Any]]: """Append post-confirm methodology nodes without overloading stage ids. The same bbav5 action (for example phylogeny) may be needed twice: once as subfamily-prediction evidence before reference_classification, and once as a formal downstream plot after user confirmation. The execution scheduler uses node ids for dependency state and the `stage` field for the actual backend action. """ if not post_confirm_targets: return stages_detail existing_ids = {str(item.get("id") or item.get("name") or "").strip() for item in stages_detail if isinstance(item, dict)} node_by_stage: dict[str, str] = {} appended: list[dict[str, Any]] = [] route = _methodology_downstream_route(target_species, reference_species, comparative_species, shared_scope) blocked = set(blocked_stage_ids or []) target_set = {stage for stage in post_confirm_targets if stage and stage not in blocked} def deps_for(stage_id: str) -> list[str]: deps = ["subfamily_confirm"] if stage_id in {"motif", "domain", "structure"} and "phylogeny" in node_by_stage: deps.append(node_by_stage["phylogeny"]) elif stage_id == "combo": panels = [panel for panel in (article_combo_panels if len(article_combo_panels) >= 2 else METHODOLOGY_SUBFAMILY_COMBO_PANELS) if panel in target_set] deps.extend(node_by_stage[panel] for panel in panels if panel in node_by_stage) elif stage_id == "karyotype" and "synteny" in node_by_stage: deps.append(node_by_stage["synteny"]) elif stage_id == "kaks" and "ortholog" in node_by_stage: deps.append(node_by_stage["ortholog"]) elif stage_id in {"rnaseq_align", "rnaseq_quant", "rnaseq_sample_qc", "rnaseq_de", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea", "rnaseq_wgcna"}: rnaseq_deps = _methodology_saved_plan_dependencies(stage_id, post_confirm_targets) deps = [dep for dep in rnaseq_deps if dep in node_by_stage] return list(dict.fromkeys(deps)) for stage_id in _methodology_order_stages(post_confirm_targets): if stage_id in blocked: continue node_id = _methodology_confirmed_node_id(stage_id, existing_ids) existing_ids.add(node_id) node_by_stage[stage_id] = node_id detail = _stage_detail_defaults(stage_id) detail.update({ "id": node_id, "stage": stage_id, "action": stage_id, "name": f"{detail.get('name') or stage_id}（确认亚家族后）", "dependencies": deps_for(stage_id), "methodology_phase": "post_subfamily_confirm", "requires_subfamily_confirm": True, }) item_params = dict(plan_params) item_params.update({ "scopeMode": "target_plus_reference" if reference_species else shared_scope, "targetSpecies": target_species, "referenceSpecies": reference_species, "comparativeSpecies": comparative_species, "refSpeciesList": list(dict.fromkeys([*reference_species, *comparative_species])), "analysisRoute": route, "analysis_route": route, "subfamilyPipeline": False, "requiresConfirmedSubfamily": True, "subfamilyLabelSource": "confirmed_or_uploaded_subfamily", }) item_params.pop("memberSetId", None) item_params.pop("member_set_id", None) if stage_id == "combo": candidate_panels = list(article_combo_panels) if len(article_combo_panels) >= 2 else list(METHODOLOGY_SUBFAMILY_COMBO_PANELS) panels = [panel for panel in candidate_panels if panel in target_set] if len(panels) < 2: panels = [panel for panel in METHODOLOGY_ARTICLE_COMBO_PANEL_ORDER if panel in target_set] item_params["comboPanels"] = panels item_params["comboPanelSource"] = "article_methodology" if len(article_combo_panels) >= 2 else "regular_gene_family" detail["params"] = enrich_stage_params_with_context(stage_id, prefix, item_params) appended.append(detail) return [*stages_detail, *appended] def _methodology_post_confirm_member_set(params: dict[str, Any]) -> str: scope_id = str(params.get("scopeId") or params.get("scope_id") or "").strip() if not scope_id: return "" return f"{scope_id}:confirmed_subfamily:all_members" def _methodology_normalize_post_confirm_stage_details( stages_detail: list[dict[str, Any]], *, post_confirm_targets: list[str], plan_params: dict[str, Any], prefix: str, target_species: list[str], reference_species: list[str], comparative_species: list[str], deferred_methodology_stages: list[dict[str, Any]], expression_status: dict[str, Any], stage_metadata: dict[str, dict[str, Any]], article_combo_panels: list[str], ) -> list[dict[str, Any]]: if not post_confirm_targets: return [] existing_by_stage: dict[str, dict[str, Any]] = {} for item in stages_detail or []: if not isinstance(item, dict): continue stage_id = str(item.get("stage") or item.get("action") or item.get("id") or item.get("name") or "").strip() if stage_id and stage_id not in existing_by_stage: existing_by_stage[stage_id] = item ordered_targets = _methodology_order_stages(post_confirm_targets) combo_panels = list(article_combo_panels) if len(article_combo_panels) >= 2 else list(METHODOLOGY_SUBFAMILY_COMBO_PANELS) expression_gate = next( (item for item in deferred_methodology_stages or [] if item.get("stage") == "expression" and not item.get("blocking", True)), {}, ) normalized: list[dict[str, Any]] = [] route = _methodology_downstream_route(target_species, reference_species, comparative_species, "target_plus_reference" if reference_species else "target_only") for order, stage_id in enumerate(ordered_targets, start=1): source_detail = dict(existing_by_stage.get(stage_id) or _stage_detail_defaults(stage_id)) detail = dict(source_detail) detail.update({ "id": stage_id, "stage": stage_id, "action": stage_id, "order": order, "methodology_phase": "post_subfamily_confirm", }) detail["dependencies"] = _methodology_saved_plan_dependencies( stage_id, ordered_targets, combo_panels=combo_panels, ) detail_params = {**plan_params, **(source_detail.get("params") or {})} detail_params.update({ "prefix": prefix, "geneFamily": plan_params.get("geneFamily") or (prefix.split(".")[-1] if "." in prefix else ""), "targetSpecies": target_species, "referenceSpecies": reference_species, "comparativeSpecies": comparative_species, "refSpeciesList": list(dict.fromkeys([*reference_species, *comparative_species])), "subfamilyPipeline": False, "subfamilyLabelSource": "confirmed_or_uploaded_subfamily", }) if stage_id in FULL_GENE_FAMILY_TARGET_ONLY_STAGES: _normalize_full_workflow_stage_scope(stage_id, detail_params, plan_params, prefix, "single_species") elif stage_id in FULL_GENE_FAMILY_REFERENCE_STAGES: _normalize_full_workflow_stage_scope(stage_id, detail_params, plan_params, prefix, route) elif stage_id == "expression": detail_params.update({ "scopeMode": "target_only", "referenceSpecies": [], "comparativeSpecies": [], "refSpeciesList": [], "analysisRoute": "single_species", "analysis_route": "single_species", "requiresConfirmedSubfamily": False, "expressionDataStatus": expression_status, }) _drop_scope_identity_params(detail_params) if expression_gate: metadata = stage_metadata.get("expression") or {} accessions = metadata.get("accessions") or metadata.get("geo_accessions") or [] detail_params.update({ "requiresDataSourceConfirmation": True, "expressionDataSourceReason": expression_gate.get("reason", ""), "expressionAccessions": accessions if isinstance(accessions, list) else [accessions], "expressionDataSourceType": metadata.get("data_source_type", ""), "expressionAnalysisRoute": metadata.get("analysis_route") or metadata.get("analysisRoute") or "", }) detail["status"] = "needs_data_source_confirmation" detail["note"] = "表达分析需要先确认数据来源；不会自动下载或重分析 raw RNA-seq。" if stage_id in FORMAL_GENE_FAMILY_CORE_STAGES: detail_params["requiresConfirmedSubfamily"] = True detail_params["forceRerun"] = False if stage_id == "combo": panels = [panel for panel in combo_panels if panel in ordered_targets] if len(panels) < 2: panels = [panel for panel in METHODOLOGY_SUBFAMILY_COMBO_PANELS if panel in ordered_targets] detail_params["comboPanels"] = panels detail_params["comboPanelSource"] = "regular_gene_family" detail["name"] = f"{_stage_detail_defaults(stage_id).get('name', stage_id)}（确认亚家族后）" if stage_id in GENE_FAMILY_SCOPE_STAGES: enriched = enrich_stage_params_with_context(stage_id, prefix, detail_params) if stage_id in FORMAL_GENE_FAMILY_CORE_STAGES: member_set = _methodology_post_confirm_member_set(enriched) if member_set: enriched["memberSetId"] = member_set enriched["member_set_id"] = member_set for ctx_key in ("stageContext", "stage_context"): ctx = enriched.get(ctx_key) if isinstance(ctx, dict): ctx["member_set_id"] = member_set ctx["subfamily_source"] = "confirmed" detail["params"] = enriched else: detail["params"] = detail_params normalized.append(detail) return normalized def _methodology_saved_plan_display_targets( stages_detail: list[dict[str, Any]], methodology_target_stage_ids: list[str], blocked_methodology_stage_ids: list[str], deferred_methodology_stages: list[dict[str, Any]], expression_status: dict[str, Any], ) -> tuple[list[str], list[dict[str, Any]], dict[str, Any]]: detail_by_id = {str((item.get("id") or item.get("name") or "")).strip(): item for item in stages_detail or [] if isinstance(item, dict)} target_set = {stage for stage in methodology_target_stage_ids or [] if stage not in METHODOLOGY_RAW_RNASEQ_STAGES and stage != "external_or_manual"} blocked_set = set(blocked_methodology_stage_ids or []) display_ids = [stage for stage in METHODOLOGY_DISPLAY_STAGE_ORDER if stage in target_set or stage in blocked_set or stage in {"genome_prepare", "identify", "reference_classification", "subfamily_confirm"} or stage in detail_by_id] display_details: list[dict[str, Any]] = [] for order, stage_id in enumerate(display_ids, start=1): base = dict(detail_by_id.get(stage_id) or _stage_detail_defaults(stage_id)) base["id"] = stage_id base["name"] = METHODOLOGY_DISPLAY_STAGE_LABELS.get(stage_id, base.get("name") or stage_id) base["display_label"] = base["name"] base["description"] = base["name"] base["order"] = order base["display_as_primary_target"] = True base["methodologyRequired"] = True params = dict(base.get("params") or {}) params["displayLabel"] = base["name"] if stage_id == "phylogeny": params["scopeMode"] = "target_plus_reference" if params.get("referenceSpecies") else params.get("scopeMode", "target_plus_reference") params["analysisRoute"] = "multi_species" params["analysis_route"] = "multi_species" elif stage_id == "combo": panels = params.get("comboPanels") or params.get("combo_panels") or list(METHODOLOGY_SUBFAMILY_COMBO_PANELS) if not isinstance(panels, list): panels = [item.strip() for item in re.split(r"[,;，；\s]+", str(panels)) if item.strip()] panels = [panel for panel in METHODOLOGY_ARTICLE_COMBO_PANEL_ORDER if panel in set(panels)] if len(panels) < 2: panels = list(METHODOLOGY_SUBFAMILY_COMBO_PANELS) params["comboPanels"] = panels params.setdefault("comboPanelSource", "subfamily_evidence") panel_label = " + ".join({"motif": "Motif", "domain": "Domain", "structure": "Structure"}.get(panel, panel) for panel in panels) base["name"] = f"{panel_label} 组合图（使用确认亚家族标签）" base["display_label"] = base["name"] base["description"] = base["name"] params["displayLabel"] = base["name"] base["dependencies"] = [stage for stage in panels if stage in target_set or stage in detail_by_id] elif stage_id == "expression": base["note"] = "组织表达 TPM/SesameFG 矩阵复现；本地无矩阵时需确认或上传矩阵。" params["expressionDataStatus"] = expression_status if stage_id in blocked_set: base["status"] = "blocked" base["blocked"] = True base["block_reason"] = next((item.get("reason", "") for item in deferred_methodology_stages if item.get("stage") == stage_id), base.get("block_reason", "")) if stage_id == "expression" and not expression_status.get("has_matrix"): base["block_reason"] = "missing_expression_matrix_or_fastq_samples_yaml" params["executionBlocked"] = True params["blockReason"] = base.get("block_reason", "") base["params"] = params display_details.append(base) raw_children = [str(item.get("stage") or "").strip() for item in deferred_methodology_stages or [] if item.get("reason") == METHODOLOGY_RAW_RNASEQ_BLOCK_REASON and str(item.get("stage") or "").strip() in METHODOLOGY_RAW_RNASEQ_STAGES] if raw_children: raw_item = { "id": "raw_rnaseq_reanalysis", "name": "干旱胁迫 RNA-seq 原始数据重分析（需用户授权，暂不执行）", "display_label": "干旱胁迫 RNA-seq 原始数据重分析（需用户授权，暂不执行）", "description": "干旱胁迫 RNA-seq 原始数据重分析（需用户授权，暂不执行）", "order": len(display_details) + 1, "dependencies": [], "status": "blocked", "blocked": True, "block_reason": METHODOLOGY_RAW_RNASEQ_BLOCK_REASON, "display_as_primary_target": True, "methodologyRequired": True, "child_stages": list(dict.fromkeys(raw_children)), "params": { "executionBlocked": True, "blockReason": METHODOLOGY_RAW_RNASEQ_BLOCK_REASON, "displayLabel": "干旱胁迫 RNA-seq 原始数据重分析（需用户授权，暂不执行）", "childStages": list(dict.fromkeys(raw_children)), }, } display_ids.append("raw_rnaseq_reanalysis") display_details.append(raw_item) summary = { "workflow_contract_version": "methodology_fixed_workflow.v1", "target_stages": display_details, "blocked_child_stages": list(dict.fromkeys(raw_children)), } return display_ids, display_details, summary async def _methodology_saved_plan_response(request: AgentPlanRequest, session_id: str, plan: dict[str, Any], source_params: dict[str, Any]) -> dict[str, Any]: """Translate a saved methodology into target stages and let MAS own dependencies.""" readiness = methodology_plan_readiness(plan, job_id=request.job_id, input_root=INPUT_ROOT, workspace_root=WORKSPACE_ROOT) if not readiness.get("config_exists"): return { "success": False, "mode": "methodology_saved_plan", "status": "methodology_data_not_ready", "message": "方法学方案数据未就绪，请先确认 config.xlsx。", "readiness": readiness, "issues": readiness.get("missing_requirements", []), "session_id": session_id, } method_params = methodology_agent_params(plan) family = source_params.get("geneFamily") or method_params.get("geneFamily") or _gene_family_from_config(INPUT_ROOT / request.job_id / "config.xlsx") or _prefix_species_family(request.prefix or "")[1] family = _nonempty_prefix(family) raw_target_species = source_params.get("targetSpecies") or method_params.get("targetSpecies") raw_reference_species = source_params.get("referenceSpecies") or method_params.get("referenceSpecies") raw_comparative_species = source_params.get("comparativeSpecies") or method_params.get("comparativeSpecies") target_species = _canonicalize_methodology_species(request.job_id, raw_target_species) reference_species = _canonicalize_methodology_species(request.job_id, raw_reference_species) comparative_species = _canonicalize_methodology_species(request.job_id, raw_comparative_species) identify_refs = _canonicalize_methodology_species( request.job_id, source_params.get("identifyBlastReferenceSpecies") or method_params.get("identifyBlastReferenceSpecies") or reference_species, ) primary_refs = _canonicalize_methodology_species( request.job_id, source_params.get("primaryReferenceSpecies") or method_params.get("primaryReferenceSpecies") or reference_species, ) unresolved_species = _methodology_unresolved_species([*target_species, *reference_species, *comparative_species, *identify_refs, *primary_refs]) if unresolved_species: return { "success": False, "mode": "methodology_saved_plan", "status": "methodology_species_mapping_missing", "message": "方法学方案中的物种名称未能映射到 config.xlsx 的 Species_Code，已阻止生成执行计划。", "unresolved_species": unresolved_species, "session_id": session_id, } required_pfams = str(source_params.get("requiredPfams") or method_params.get("requiredPfams") or "").strip() raw_stage_ids, stage_metadata, needs_subfamily = _methodology_saved_plan_stage_ids(plan, method_params, reference_species) identify_metadata = stage_metadata.get("identify") or {} methodology_blast_evalue = _methodology_blast_evalue_from_metadata(identify_metadata) methodology_blast_identity = _methodology_blast_identity_from_metadata(identify_metadata) needs_subfamily = True if not raw_stage_ids: return {"success": False, "mode": "methodology_saved_plan", "status": "methodology_plan_invalid", "message": "methodology_plan.json 缺少可转发的分析目标。", "session_id": session_id} if not family or not target_species: return {"success": False, "mode": "methodology_saved_plan", "status": "methodology_plan_invalid", "message": f"方法学方案无法解析可执行 family/targetSpecies: family={family or ''}, targetSpecies={target_species}", "session_id": session_id} article_combo_panels = _methodology_combo_panels_from_saved_plan(plan, raw_stage_ids) subfamily_combo_panels = list(METHODOLOGY_SUBFAMILY_COMBO_PANELS) methodology_target_stage_ids = [ stage for stage in _methodology_target_stage_ids(plan) if stage != "combo" or len(article_combo_panels) >= 2 ] expression_data_status = _methodology_expression_data_status(request.job_id) executable_stage_ids, deferred_methodology_stages = _methodology_filter_expression_stages_for_data(raw_stage_ids, expression_data_status, stage_metadata) blocked_methodology_stage_ids = [ str(item.get("stage") or "").strip() for item in deferred_methodology_stages if str(item.get("stage") or "").strip() and item.get("blocking", True) ] expression_warnings = _methodology_expression_blocked_warnings(deferred_methodology_stages, expression_data_status, stage_metadata) prefix = _methodology_execution_prefix(request.job_id, request.prefix or "", target_species, family) shared_scope = "target_plus_reference" if reference_species or comparative_species else "target_only" plan_params = { **method_params, **source_params, "source": "methodology", "methodology": True, "prefix": prefix, "geneFamily": family, "targetSpecies": target_species, "referenceSpecies": reference_species, "comparativeSpecies": comparative_species, "identifyBlastReferenceSpecies": identify_refs, "primaryReferenceSpecies": primary_refs, "refSpeciesList": list(dict.fromkeys([*reference_species, *comparative_species])), "requiredPfams": required_pfams, "scopeMode": "target_only", "requestedMethodologyStages": methodology_target_stage_ids, "methodologyTargetStages": methodology_target_stage_ids, "blockedMethodologyStages": blocked_methodology_stage_ids, "methodologyArticleComboPanels": article_combo_panels, "methodologySubfamilyComboPanels": subfamily_combo_panels, "methodologyPlanId": plan.get("plan_id", ""), "sourcePaperId": plan.get("paper_id", ""), "expressionDataStatus": expression_data_status, "deferredMethodologyStages": deferred_methodology_stages, "methodologyRoutes": [ { "stage": stage_id, "context": (stage_metadata.get(stage_id) or {}).get("analysis_context", ""), "route": (stage_metadata.get(stage_id) or {}).get("analysis_route", ""), "data_source_type": (stage_metadata.get(stage_id) or {}).get("data_source_type", ""), "fallback_stages": (stage_metadata.get(stage_id) or {}).get("fallback_stages", []), "accessions": (stage_metadata.get(stage_id) or {}).get("accessions", []), } for stage_id in raw_stage_ids ], } if methodology_blast_evalue is not None: plan_params["blastEvalue"] = methodology_blast_evalue if methodology_blast_identity is not None: plan_params["blastIdentity"] = methodology_blast_identity confirmed_subfamily_ready = _methodology_confirmed_subfamily_completed(request.job_id, plan_params) genome_prepare_requested = "genome_prepare" in set(method_params.get("requestedMethodologyStages") or []) or "genome_prepare" in set(readiness.get("suggested_stages") or []) genome_prepare_completed = _methodology_success_manifest_completed(request.job_id, "genome_prepare", plan_params) genome_prepare_pending = bool(genome_prepare_requested and not genome_prepare_completed) data_preparation_completed = not genome_prepare_pending if not genome_prepare_pending and "identify" in raw_stage_ids and not required_pfams: return {"success": False, "mode": "methodology_saved_plan", "status": "methodology_required_pfams_missing", "message": "方法学 identify 缺少 requiredPfams，未启动普通 Pfam 推断。", "session_id": session_id} post_confirm_targets = [ *METHODOLOGY_POST_CONFIRM_CORE_STAGES, *[ stage for stage in methodology_target_stage_ids if stage in executable_stage_ids and stage not in blocked_methodology_stage_ids ], ] post_confirm_targets = _methodology_order_stages(post_confirm_targets) post_confirm_targets.extend(stage for stage in executable_stage_ids if stage.startswith("rnaseq_") and stage not in post_confirm_targets) if genome_prepare_pending: routed_stage_targets = ["genome_prepare"] plan_params.update({ "scopeMode": "target_only", "subfamilyPipeline": False, "requiresConfirmedSubfamily": False, }) plan_params.pop("analysisRoute", None) plan_params.pop("analysis_route", None) plan_params.pop("memberSetId", None) plan_params.pop("member_set_id", None) methodology_current_phase = "data_preparation" methodology_gate = None methodology_phase = "data_preparation" elif needs_subfamily and not confirmed_subfamily_ready: evidence_targets = [ "identify", "phylogeny", "motif", "domain", "structure", "combo", "reference_classification", "subfamily_confirm", ] routed_stage_targets = _methodology_order_stages([ *(stage for stage in evidence_targets if stage in raw_stage_ids or stage in {"reference_classification", "subfamily_confirm"}), ]) plan_params.update({ "scopeMode": "target_plus_reference" if reference_species else shared_scope, "comboPanels": list(subfamily_combo_panels), "comboPanelSource": "subfamily_evidence", "subfamilyPipeline": True, "referenceClassificationRequiresEvidence": True, "requiresConfirmedSubfamily": False, "subfamilyLabelSource": "confirmed_or_uploaded_subfamily", }) methodology_current_phase = "subfamily_classification" methodology_gate = "subfamily_confirm_required" methodology_phase = "subfamily_gate" else: routed_stage_targets = _methodology_order_stages(post_confirm_targets) plan_params.update({ "scopeMode": "target_plus_reference" if reference_species else "target_only", "comboPanels": list(article_combo_panels) if len(article_combo_panels) >= 2 else list(subfamily_combo_panels), "comboPanelSource": "article_methodology" if len(article_combo_panels) >= 2 else "regular_gene_family", "subfamilyPipeline": False, "requiresConfirmedSubfamily": True, "subfamilyLabelSource": "confirmed_or_uploaded_subfamily", }) plan_params.pop("requires_confirmed_subfamily", None) plan_params.pop("analysisRoute", None) plan_params.pop("analysis_route", None) plan_params.pop("memberSetId", None) plan_params.pop("member_set_id", None) methodology_current_phase = "post_subfamily_family_analysis" methodology_gate = None methodology_phase = "post_subfamily_confirm" if blocked_methodology_stage_ids and all(stage == "expression" for stage in blocked_methodology_stage_ids): methodology_gate = methodology_gate or "expression_matrix_missing" if any(item.get("reason") == METHODOLOGY_RAW_RNASEQ_BLOCK_REASON for item in deferred_methodology_stages): methodology_gate = methodology_gate or "raw_rnaseq_requires_user_consent" routed_intent = { "gene_family": family, "species": target_species[0] if len(target_species) == 1 else "Multi", "target_species": target_species, "reference_species": reference_species, "comparative_species": comparative_species, "scopeMode": plan_params.get("scopeMode", shared_scope), "stages": routed_stage_targets, "description": "按已保存方法学方案复现文献流程", "confidence": 0.99, "prefix": prefix, "methodology": True, "source": "methodology", "params": plan_params, } mas_plan: dict[str, Any] = {} mas_intent: dict[str, Any] = routed_intent if routed_stage_targets: if orchestrator is not None and MAS_AVAILABLE: initial_state = { "query": request.message, "session_id": session_id, "job_id": request.job_id, "files": [], "current_agent": "planning", "status": "running", "errors": [], "final_output": {}, "history": request.history if request.history else [], "results": {"completed_stages": [], "intent": routed_intent}, } result = await orchestrator.plan(initial_state) if result.get("status") != "completed": error_msg = result.get("error") or result.get("final_output", {}).get("error") or "多智能体规划失败" return {"success": False, "mode": "methodology_saved_plan", "status": "mas_plan_failed", "message": f"方法学目标转发后规划失败: {error_msg}", "session_id": session_id} mas_plan = result.get("final_output", {}).get("analysis_plan", {}) or {} mas_intent = result.get("final_output", {}).get("intent", {}) or routed_intent else: try: from gfa_core.mas.agents.planning_v2 import PlanningAgentV2 fallback_planner = PlanningAgentV2(llm_manager=None, project_root=str(PROJECT_ROOT)) mas_plan = await fallback_planner._generate_plan_with_llm( query=request.message, intent=routed_intent, completed_stages=[], job_id=request.job_id or "", workspace=str(WORKSPACE_ROOT / request.job_id) if request.job_id else str(WORKSPACE_ROOT), ) mas_intent = routed_intent print("[Agent Plan] Methodology used PlanningAgentV2 fallback because orchestrator is unavailable") except Exception as exc: return {"success": False, "mode": "methodology_saved_plan", "status": "mas_unavailable", "message": f"多智能体规划器不可用，且普通规划 fallback 失败: {exc}", "session_id": session_id} else: mas_plan = { "analysis_type": "methodology_reproduction", "gene_family": family, "species": routed_intent["species"], "target_species": target_species, "reference_species": reference_species, "comparative_species": comparative_species, "scopeMode": plan_params.get("scopeMode", shared_scope), "prefix": prefix, "description": "按已保存方法学方案复现文献流程；当前无可执行阶段。", "estimated_time": "0分钟", "estimated_total_time": "0分钟", "execution_strategy": "single", "max_parallel": 1, "total_stages": 0, "stages": [], "stages_detail": [], "parallel_groups": [], "params": plan_params, } mas_stages_detail = [item for item in (mas_plan.get("stages_detail") or []) if isinstance(item, dict)] for item in mas_stages_detail: stage_id = str(item.get("id") or item.get("name") or "").strip() if stage_id != "identify": continue item_params = dict(item.get("params") or {}) item_params.update({ "targetSpecies": target_species, "referenceSpecies": [], "comparativeSpecies": [], "identifyBlastReferenceSpecies": identify_refs, "primaryReferenceSpecies": primary_refs, "refSpeciesList": [], "requiredPfams": required_pfams, "scopeMode": "target_only", }) for stale_key in ("analysisRoute", "analysis_route", "memberSetId", "member_set_id", "requiresConfirmedSubfamily", "requires_confirmed_subfamily"): item_params.pop(stale_key, None) if methodology_blast_evalue is not None: item_params["blastEvalue"] = methodology_blast_evalue if methodology_blast_identity is not None: item_params["blastIdentity"] = methodology_blast_identity item["params"] = enrich_stage_params_with_context(stage_id, prefix, item_params) for item in mas_stages_detail: stage_id = str(item.get("id") or item.get("name") or "").strip() if stage_id != "subfamily_confirm": continue item_params = dict(item.get("params") or {}) item_params.update({ "analysisRoute": "confirmed_subfamily", "analysis_route": "confirmed_subfamily", "sourcePredictionRoute": "subfamily_prediction", "source_prediction_route": "subfamily_prediction", "interactiveSubfamilyReview": True, "subfamilyConfirmSpecies": target_species, "targetSpecies": target_species, "referenceSpecies": reference_species, "comparativeSpecies": comparative_species, "geneFamily": family, }) item["params"] = enrich_stage_params_with_context(stage_id, prefix, item_params) if needs_subfamily and not confirmed_subfamily_ready and not genome_prepare_pending: existing_detail_ids = {str(item.get("id") or item.get("name") or "").strip() for item in mas_stages_detail if isinstance(item, dict)} if "subfamily_confirm" not in existing_detail_ids: confirm_detail = _stage_detail_defaults("subfamily_confirm") confirm_detail.update({ "id": "subfamily_confirm", "name": "亚家族标签确认", "dependencies": ["reference_classification"], "estimated_time": "人工确认", "can_parallel": False, }) confirm_params = dict(plan_params) confirm_params.update({ "analysisRoute": "confirmed_subfamily", "analysis_route": "confirmed_subfamily", "sourcePredictionRoute": "subfamily_prediction", "source_prediction_route": "subfamily_prediction", "interactiveSubfamilyReview": True, "subfamilyConfirmSpecies": target_species, "targetSpecies": target_species, "referenceSpecies": reference_species, "comparativeSpecies": comparative_species, "geneFamily": family, }) confirm_detail["params"] = enrich_stage_params_with_context("subfamily_confirm", prefix, confirm_params) mas_stages_detail.append(confirm_detail) target_species = _canonicalize_methodology_species(request.job_id, target_species) reference_species = _canonicalize_methodology_species(request.job_id, reference_species) comparative_species = _canonicalize_methodology_species(request.job_id, comparative_species) identify_refs = _canonicalize_methodology_species(request.job_id, identify_refs) primary_refs = _canonicalize_methodology_species(request.job_id, primary_refs) unresolved_species = _methodology_unresolved_species([*target_species, *reference_species, *comparative_species, *identify_refs, *primary_refs]) if unresolved_species: return { "success": False, "mode": "methodology_saved_plan", "status": "methodology_species_mapping_missing", "message": "方法学方案中的物种名称未能映射到 config.xlsx 的 Species_Code，已阻止生成执行计划。", "unresolved_species": unresolved_species, "session_id": session_id, } prefix = _methodology_execution_prefix(request.job_id, request.prefix or "", target_species, family) plan_params.update({ "prefix": prefix, "targetSpecies": target_species, "referenceSpecies": reference_species, "comparativeSpecies": comparative_species, "identifyBlastReferenceSpecies": identify_refs, "primaryReferenceSpecies": primary_refs, "refSpeciesList": list(dict.fromkeys([*reference_species, *comparative_species])), }) for item in mas_stages_detail: stage_id = str(item.get("stage") or item.get("id") or item.get("name") or "").strip() if not stage_id: continue item_params = dict(item.get("params") or {}) uses_reference_scope = ( stage_id in {"reference_classification", "subfamily_confirm", "interspecies_synteny", "ortholog", "kaks", "ppi"} or (methodology_phase == "subfamily_gate" and stage_id in METHODOLOGY_SUBFAMILY_EVIDENCE_STAGES) or (methodology_phase == "post_subfamily_confirm" and stage_id in FULL_GENE_FAMILY_REFERENCE_STAGES) ) item_params.update({ "targetSpecies": target_species, "referenceSpecies": reference_species, "comparativeSpecies": comparative_species, "identifyBlastReferenceSpecies": identify_refs, "primaryReferenceSpecies": primary_refs, "refSpeciesList": list(dict.fromkeys([*reference_species, *comparative_species])), "geneFamily": family, }) if uses_reference_scope: item_params["scopeMode"] = "target_plus_reference" if reference_species else plan_params.get("scopeMode", shared_scope) item_params["referenceSpecies"] = reference_species item_params["comparativeSpecies"] = comparative_species item_params["refSpeciesList"] = list(dict.fromkeys([*reference_species, *comparative_species])) if methodology_phase == "subfamily_gate" and stage_id in METHODOLOGY_SUBFAMILY_EVIDENCE_STAGES: item_params["analysisRoute"] = "subfamily_evidence" item_params["analysis_route"] = "subfamily_evidence" else: item_params["scopeMode"] = "target_only" item_params["referenceSpecies"] = [] item_params["comparativeSpecies"] = [] item_params["refSpeciesList"] = [] for stale_key in ("analysisRoute", "analysis_route", "memberSetId", "member_set_id", "requiresConfirmedSubfamily", "requires_confirmed_subfamily"): item_params.pop(stale_key, None) item["params"] = enrich_stage_params_with_context(stage_id, prefix, item_params) if methodology_phase == "post_subfamily_confirm": mas_stages_detail = _methodology_normalize_post_confirm_stage_details( mas_stages_detail, post_confirm_targets=routed_stage_targets, plan_params=plan_params, prefix=prefix, target_species=target_species, reference_species=reference_species, comparative_species=comparative_species, deferred_methodology_stages=deferred_methodology_stages, expression_status=expression_data_status, stage_metadata=stage_metadata, article_combo_panels=article_combo_panels, ) mas_stage_ids = [str(item.get("id") or item.get("name") or "").strip() for item in mas_stages_detail] if not mas_stage_ids: mas_stage_ids = [str(stage.get("id") or stage.get("name") or "").strip() if isinstance(stage, dict) else str(stage).strip() for stage in (mas_plan.get("stages") or []) if str(stage).strip()] all_stage_ids = list(dict.fromkeys([*routed_stage_targets, *mas_stage_ids])) completed_stage_ids: list[str] = [] if request.job_id and mas_stages_detail: executable_stage_names = { "identify", "phylogeny", "motif", "domain", "structure", "combo", "reference_classification", "subfamily_confirm", "pep", "synteny", "interspecies_synteny", "karyotype", "promoter", "tfbs", "expression", "rnaseq_qc", "rnaseq_align", "rnaseq_quant", "rnaseq_sample_qc", "rnaseq_de", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea", "rnaseq_wgcna", "ortholog", "kaks", "ppi", } post_confirm_identity_stage_ids = { str(item.get("id") or item.get("name") or "").strip() for item in mas_stages_detail if isinstance(item, dict) and str(item.get("methodology_phase") or "") == "post_subfamily_confirm" } normal_stage_ids = [ stage_id for stage_id in mas_stage_ids if stage_id in executable_stage_names ] completed_stage_ids = _identity_completed_stages_for_plan( request.job_id, normal_stage_ids, plan_params, mas_stages_detail, effective_prefix=prefix, request_prefix=prefix, ) completed_set = set(completed_stage_ids) for stage_id in normal_stage_ids: if stage_id in completed_set: continue if stage_id == "subfamily_confirm": if _methodology_confirmed_subfamily_completed(request.job_id, plan_params): completed_stage_ids.append(stage_id) completed_set.add(stage_id) continue if stage_id in post_confirm_identity_stage_ids: continue if _methodology_success_manifest_completed(request.job_id, stage_id, plan_params): completed_stage_ids.append(stage_id) completed_set.add(stage_id) if completed_stage_ids: print(f"[Agent Plan] Methodology completed stages filtered from execution: {completed_stage_ids}") display_seed_details = list(mas_stages_detail) detail_ids = {str(item.get("id") or item.get("name") or "") for item in display_seed_details} for blocked_stage in blocked_methodology_stage_ids: if blocked_stage in detail_ids: continue detail = _stage_detail_defaults(blocked_stage) detail["id"] = blocked_stage detail["status"] = "blocked" detail["blocked"] = True detail["block_reason"] = next((item.get("reason", "") for item in deferred_methodology_stages if item.get("stage") == blocked_stage), "") params_for_blocked = dict(plan_params) params_for_blocked.update({"executionBlocked": True, "blockReason": detail.get("block_reason", "")}) detail["params"] = params_for_blocked display_seed_details.append(detail) methodology_display_stage_ids, methodology_display_details, methodology_display_summary = _methodology_saved_plan_display_targets( display_seed_details, methodology_target_stage_ids, blocked_methodology_stage_ids, deferred_methodology_stages, expression_data_status, ) methodology_workflow_phases = [] for phase_id, label, phase_stages in METHODOLOGY_WORKFLOW_PHASES: phase_detail = [item for item in display_seed_details if (item.get("id") or item.get("name")) in phase_stages] if phase_id == "data_preparation": phase_status = "completed" if data_preparation_completed else "current" elif phase_id == "subfamily_classification": phase_status = "completed" if confirmed_subfamily_ready else ("planned" if genome_prepare_pending else "current") elif phase_id == "post_subfamily_family_analysis": phase_status = "current" if confirmed_subfamily_ready and routed_stage_targets else "planned" else: phase_status = "blocked" if blocked_methodology_stage_ids else "planned" methodology_workflow_phases.append({ "phase_id": phase_id, "label": label, "status": phase_status, "required": phase_id != "expression_rnaseq", "stage_order": phase_stages, "stages_detail": phase_detail, }) methodology_display_summary["phases"] = methodology_workflow_phases methodology_display_summary["current_phase"] = methodology_current_phase methodology_display_summary["gate"] = methodology_gate methodology_display_summary["routed_target_stages"] = routed_stage_targets completed_stage_set = set(completed_stage_ids) executable_details = [ item for item in mas_stages_detail if str(item.get("id") or item.get("name") or "").strip() not in completed_stage_set ] executable_ids = [str(item.get("id") or item.get("name") or "").strip() for item in executable_details if str(item.get("id") or item.get("name") or "").strip()] estimated_time = mas_plan.get("estimated_total_time") or mas_plan.get("estimated_time") or "0分钟" plan_payload = { **mas_plan, "analysis_type": "methodology_reproduction", "gene_family": family, "species": routed_intent["species"], "target_species": target_species, "reference_species": reference_species, "comparative_species": comparative_species, "scopeMode": plan_params.get("scopeMode", shared_scope), "prefix": prefix, "description": "按已保存方法学方案复现文献流程", "estimated_time": estimated_time, "estimated_total_time": estimated_time, "total_stages": len(executable_ids), "stages": executable_ids, "stages_detail": executable_details, "executable_stages": executable_ids, "executable_stages_detail": executable_details, "visible_stages": all_stage_ids, "visible_stages_detail": display_seed_details, "methodology_target_stages": methodology_display_stage_ids, "methodology_target_stages_detail": methodology_display_details, "methodology_execution_target_stages": methodology_target_stage_ids, "methodology_execution_target_stages_detail": [item for item in methodology_display_details if (item.get("id") or item.get("name")) in methodology_target_stage_ids], "methodology_workflow_phases": methodology_workflow_phases, "methodology_current_phase": methodology_current_phase, "methodology_gate": methodology_gate, "methodology_display_summary": methodology_display_summary, "blocked_methodology_stages": blocked_methodology_stage_ids, "blocked_methodology_stages_detail": [item for item in display_seed_details if (item.get("id") or item.get("name")) in blocked_methodology_stage_ids], "all_stages": all_stage_ids, "all_stages_detail": display_seed_details, "warnings": expression_warnings, "params": plan_params, "methodology_plan_id": plan.get("plan_id", ""), "methodology_source": plan.get("source_batch_dir", ""), "methodology_subfamily_pipeline": needs_subfamily, "methodology_phase": methodology_phase, "methodology_workflow_contract_version": "methodology_target_forwarder.v1", "expression_data_status": expression_data_status, "deferred_methodology_stages": deferred_methodology_stages, "routed_target_stages": routed_stage_targets, } return { "success": True, "mode": "methodology_saved_plan", "status": "ready", "message": "按已保存方法学目标生成多智能体执行计划", "readiness": readiness, "plan": plan_payload, "intent": mas_intent, "completed_stages": completed_stage_ids, "methodology_phase": methodology_phase, "deferred_methodology_stages": deferred_methodology_stages, "blocked_methodology_stages": blocked_methodology_stage_ids, "warnings": expression_warnings, "session_id": session_id, } @app.post("/api/agent/plan") async def agent_plan(request: AgentPlanRequest): """ Agent任务规划接口 V2 - 使用LLM智能规划 返回详细的分析计划，包括： - 识别的意图（基因家族、物种、阶段） - 详细的阶段列表（含依赖、并行信息） - 执行策略（并行/串行） - 预估时间 """ try: session_id = request.session_id or f"session_{int(time.time())}" methodology_plan, methodology_intent, methodology_params = _load_methodology_plan_for_agent_request(request) if methodology_plan == {}: return { "success": False, "mode": "methodology_saved_plan", "status": "methodology_plan_missing", "message": "当前请求未传入有效 job_id，或当前 job 未绑定 methodology_plan.json；不能按已保存方法学方案执行，也不会退回普通基因家族规划。", "session_id": session_id, } if methodology_plan: return await _methodology_saved_plan_response(request, session_id, methodology_plan, methodology_params) extension_intent = _match_extension_stage_intent(request.message, request.job_id or "") if extension_intent: return _extension_stage_plan_response(extension_intent, session_id, request.job_id or "") plan_role_resolution = ( _reconcile_confirmed_gene_family_species( request.job_id, request.message, request.intent, ) if _is_gene_family_analysis_intent(request.intent) else {"applicable": False, "conflicts": []} ) if plan_role_resolution.get("conflicts"): return { "success": False, "status": "species_role_conflict", "message": _species_role_conflict_message(plan_role_resolution["conflicts"]), "issues": plan_role_resolution["conflicts"], "suggested_command": "打开 config.xlsx", "session_id": session_id, } if plan_role_resolution.get("applicable"): request.intent = dict(request.intent or {}) request.intent["params"] = dict(request.intent.get("params") or {}) _apply_reconciled_species_to_intent(request.intent, plan_role_resolution) query_lower_for_context = (request.message or "").lower() subfamily_confirm_query_for_context = any( kw in query_lower_for_context for kw in ["确认亚家族", "亚家族确认", "确认亚家族标签", "人工确认亚家族", "subfamily confirm", "confirmed subfamily"] ) subfamily_prediction_query_for_context = ( any(kw in query_lower_for_context for kw in ["亚家族", "subfamily"]) and not _skip_subfamily_label_query(query_lower_for_context) and not subfamily_confirm_query_for_context and not _looks_like_gene_family_workflow_query(query_lower_for_context) and not _requires_confirmed_subfamily_query(query_lower_for_context) ) if subfamily_prediction_query_for_context: requested_prefix_for_subfamily = request.prefix or "" prefix_species, prefix_family = _prefix_species_family(requested_prefix_for_subfamily) family_for_subfamily = _resolve_authoritative_gene_family( request.job_id, prefix_family, requested_prefix_for_subfamily, {}, request.message or "", subfamily=True, ) request_roles_for_subfamily = _resolve_subfamily_request_species_roles( request.job_id, request.message or "", request.intent, ) targets_for_subfamily = request_roles_for_subfamily["classification_species"] or _target_species_fallback_from_prefix(requested_prefix_for_subfamily) if not targets_for_subfamily and prefix_species and prefix_species not in {"Target", "Multi"}: targets_for_subfamily = [prefix_species] scope_defaults_for_subfamily = _latest_gene_family_scope_defaults(request.job_id, family_for_subfamily, requested_prefix_for_subfamily) if not targets_for_subfamily: targets_for_subfamily = scope_defaults_for_subfamily.get("target_species") or [] identify_selection_for_subfamily = _load_identify_species_selection(request.job_id) if request.job_id else {} if _looks_like_all_species_query(request.message or "") or _looks_like_all_subfamily_species_query(request.message or ""): selected_for_subfamily, _selection_mode_for_subfamily = _select_subfamily_classification_species( request.job_id, family_for_subfamily, identify_selection_for_subfamily or {}, request.message or "", targets_for_subfamily, ) if selected_for_subfamily: targets_for_subfamily = selected_for_subfamily else: targets_for_subfamily = list(dict.fromkeys([ *(scope_defaults_for_subfamily.get("target_species") or []), *(scope_defaults_for_subfamily.get("comparative_species") or []), ])) or targets_for_subfamily refs_for_subfamily = request_roles_for_subfamily["reference_species"] or scope_defaults_for_subfamily.get("reference_species") or [ item.get("species") for item in ((identify_selection_for_subfamily.get("groups") or {}).get("reference") or []) if item.get("species") ] if not refs_for_subfamily: refs_for_subfamily = _reference_species_from_config(request.job_id, family_for_subfamily, targets_for_subfamily) if not refs_for_subfamily and request.job_id and family_for_subfamily: input_dir_for_subfamily = INPUT_ROOT / request.job_id uploaded_refs = [] for path in sorted(input_dir_for_subfamily.glob(f"*.{family_for_subfamily}.subfamily")): species_token = path.name.split(".", 1)[0] if species_token and species_token not in targets_for_subfamily and species_token not in uploaded_refs: uploaded_refs.append(species_token) refs_for_subfamily = uploaded_refs if family_for_subfamily and targets_for_subfamily and not refs_for_subfamily: return { "success": False, "mode": "deterministic_subfamily_pre_mas", "message": f"亚家族分类需要参考物种；已解析目标物种={targets_for_subfamily}、Family={family_for_subfamily}，但未从 config.xlsx 找到 seed_reference/known_family reference。", "issues": [{"code": "subfamily_missing_reference_species", "severity": "blocking"}], "session_id": session_id, } if family_for_subfamily and targets_for_subfamily: prefix_for_subfamily = f"{targets_for_subfamily[0]}.{family_for_subfamily}" if len(targets_for_subfamily) == 1 else f"Target.{family_for_subfamily}" deterministic_stages = ["identify", "phylogeny", "motif", "domain", "structure", "combo", "reference_classification"] classification_species_for_subfamily = list(targets_for_subfamily) classification_species_for_subfamily, role_targets_for_subfamily, role_comps_for_subfamily = _split_subfamily_selected_species_by_config_roles( request.job_id, classification_species_for_subfamily, identify_selection_for_subfamily or {}, scope_defaults_for_subfamily.get("target_species") or targets_for_subfamily, ) confirm_prefix_by_species = {sp: f"{sp}.{family_for_subfamily}" for sp in classification_species_for_subfamily} subfamily_state_for_plan = _subfamily_state_for_targets(request.job_id, classification_species_for_subfamily, family_for_subfamily) plan_params = { "prefix": prefix_for_subfamily, "geneFamily": family_for_subfamily, "targetSpecies": role_targets_for_subfamily, "referenceSpecies": refs_for_subfamily, "comparativeSpecies": role_comps_for_subfamily, "scopeMode": _subfamily_scope_mode_for_roles(refs_for_subfamily, role_comps_for_subfamily), "analysisMode": "joint_multi_species" if len(classification_species_for_subfamily) > 1 else "per_species", "speciesQueue": classification_species_for_subfamily, "confirmPrefixBySpecies": confirm_prefix_by_species, "subfamilyState": subfamily_state_for_plan, } if identify_selection_for_subfamily: plan_params["identifySpeciesSelection"] = _identify_selection_with_default( identify_selection_for_subfamily, targets_for_subfamily, "已根据本次亚家族分类请求确定待分类物种；reference 仅作为参考来源。", ) stages_detail = [] for order, stage_id in enumerate(deterministic_stages, start=1): detail = _stage_detail_defaults(stage_id) detail["order"] = order detail_params = dict(plan_params) if stage_id == "identify": detail_params["scopeMode"] = "target_only" detail_params["targetSpecies"] = list(classification_species_for_subfamily) detail_params["referenceSpecies"] = [] detail_params["comparativeSpecies"] = [] elif stage_id in {"phylogeny", "motif", "domain", "structure", "combo"}: detail_params["analysisRoute"] = "subfamily_evidence" detail_params["subfamilyPipeline"] = True if stage_id == "combo": detail_params["comboPanels"] = list(METHODOLOGY_SUBFAMILY_COMBO_PANELS) detail_params["comboPanelSource"] = "subfamily_evidence" elif stage_id == "reference_classification": detail_params["analysisRoute"] = "subfamily_prediction" detail_params["subfamilyPipeline"] = True detail_params["perTargetEvidenceOnly"] = True detail["params"] = enrich_stage_params_with_context(stage_id, prefix_for_subfamily, detail_params) stages_detail.append(detail) requirements_report = {} try: from gfa_core.mas.stage_requirements import resolve_many_stage_requirements requirements_report = resolve_many_stage_requirements( stages=deterministic_stages, prefix=prefix_for_subfamily, job_id=request.job_id, params_by_stage={item["id"]: item.get("params", {}) for item in stages_detail if item.get("id")}, shared_params=plan_params, project_root=PROJECT_ROOT, shadow_only=True, ) except Exception as exc: requirements_report = {"error": str(exc), "shadow_only": True} completed_stages = _identity_completed_stages_for_plan( request.job_id, deterministic_stages, plan_params, stages_detail, effective_prefix=prefix_for_subfamily, request_prefix=request.prefix or "", ) remaining_stage_ids = [stage_id for stage_id in deterministic_stages if stage_id not in set(completed_stages)] estimated_minutes = sum( int(str(item.get("estimated_time", "10")).replace("分钟", "") or 10) if str(item.get("estimated_time", "")).replace("分钟", "").isdigit() else 10 for item in stages_detail if (item.get("id") or item.get("name")) in remaining_stage_ids ) response = { "success": True, "mode": "deterministic_subfamily_pre_mas", "plan": { "analysis_type": "gene_family", "gene_family": family_for_subfamily, "species": classification_species_for_subfamily[0] if len(classification_species_for_subfamily) == 1 else "Multi", "target_species": role_targets_for_subfamily, "reference_species": refs_for_subfamily, "comparative_species": role_comps_for_subfamily, "scopeMode": plan_params["scopeMode"], "prefix": prefix_for_subfamily, "description": "执行 target/comparative + reference 亚家族分类", "estimated_time": f"{estimated_minutes}分钟", "estimated_total_time": f"{estimated_minutes}分钟", "execution_strategy": "parallel", "max_parallel": 4, "total_stages": len(remaining_stage_ids), "stages": remaining_stage_ids, "stages_detail": [item for item in stages_detail if (item.get("id") or item.get("name")) in remaining_stage_ids], "all_stages": deterministic_stages, "all_stages_detail": stages_detail, "parallel_groups": [], "params": plan_params, "requirements_report": requirements_report, }, "intent": { "gene_family": family_for_subfamily, "species": classification_species_for_subfamily[0] if len(classification_species_for_subfamily) == 1 else "Multi", "target_species": role_targets_for_subfamily, "reference_species": refs_for_subfamily, "comparative_species": role_comps_for_subfamily, "scopeMode": plan_params["scopeMode"], "stages": deterministic_stages, "description": "执行目标物种+参考物种亚家族分类", "confidence": 0.98, "prefix": prefix_for_subfamily, "params": plan_params, }, "completed_stages": completed_stages, "session_id": session_id, } print(f"[Agent Plan] Deterministic subfamily plan (pre-MAS): targets={targets_for_subfamily}, refs={refs_for_subfamily}, remaining={remaining_stage_ids}") return response contextual_request_pre_mas = _resolve_contextual_gene_family_request( request.job_id, request.message, request.prefix, requested_stages=list((request.intent or {}).get("stages") or []), request_intent=request.intent, ) if contextual_request_pre_mas and not subfamily_prediction_query_for_context and not subfamily_confirm_query_for_context: response = _build_contextual_gene_family_plan_response(request, session_id, contextual_request_pre_mas) plan_for_log = response.get("plan") or {} if response.get("success") and plan_for_log: print(f"[Agent Plan] Deterministic contextual plan (pre-MAS): {plan_for_log.get('stages')}, prefix={plan_for_log.get('prefix')}") else: print(f"[Agent Plan] Deterministic contextual plan blocked (pre-MAS): {response.get('message')}; issues={response.get('issues')}") return response # 使用Orchestrator V2进行规划 if orchestrator is not None and MAS_AVAILABLE: from gfa_core.mas.core.state import GraphState try: from gfa_core.mas.utils.intent_parser import parse_ncbi_fetch_request, parse_promoter_plot_revise_request, parse_sequence_extract_request from gfa_core.mas.utility_tasks import utility_display_name utility_params = parse_ncbi_fetch_request(request.message or "") utility_stage = "ncbi_fetch_sequences" if utility_params else "" if not utility_params: utility_params = parse_sequence_extract_request(request.message or "") utility_stage = "sequence_extract_by_id" if utility_params else "" if not utility_params: utility_params = parse_promoter_plot_revise_request(request.message or "") utility_stage = "promoter_plot_revise" if utility_params else "" if utility_stage: utility_prefix = utility_params.get("prefix") or request.prefix or "utility" stage_params = {"prefix": utility_prefix, "cpu": 1, **utility_params} stage_name = "启动子自定义绘图" if utility_stage == "promoter_plot_revise" else utility_display_name(utility_stage, utility_params) if utility_stage == "ncbi_fetch_sequences": utility_message = f"我理解你想从 NCBI 下载序列；输入为 {utility_params.get('ncbiGeneIds') or utility_params.get('ncbiQuery') or '未指定'}。" elif utility_stage == "promoter_plot_revise": utility_message = "我理解你想基于已有 PlantCARE/promoter 结果进行启动子自定义筛选绘图，不重新跑启动子预测。" else: utility_message = "我理解你想按ID从本地FASTA提取/处理序列。" utility_family = "NCBI.fetch" if utility_stage == "ncbi_fetch_sequences" else ("Promoter.custom" if utility_stage == "promoter_plot_revise" else "Sequence.extract") utility_species = utility_params.get("ncbiSpecies") or utility_params.get("ncbiTaxid") or "" return { "success": True, "mode": "utility_rule", "plan": { "analysis_type": "utility", "gene_family": utility_family, "species": utility_species, "prefix": utility_prefix, "description": utility_message, "estimated_time": "5分钟", "estimated_total_time": "5分钟", "execution_strategy": "single", "max_parallel": 1, "total_stages": 1, "stages": [utility_stage], "stages_detail": [{ "id": utility_stage, "name": stage_name, "order": 1, "dependencies": [], "can_parallel": False, "estimated_time": 5, "params": stage_params, }], "parallel_groups": [], "params": stage_params, }, "intent": { "stage": utility_stage, "stages": [utility_stage], "gene_family": utility_family, "species": utility_species, "prefix": utility_prefix, "task_kind": "utility", "taskType": "utility", "params": stage_params, "description": utility_message, "confidence": 0.98, }, "completed_stages": [], "session_id": session_id, } except Exception as exc: print(f"[Agent Plan] Utility rule parse skipped: {exc}") incoming_intent = dict(request.intent or {}) from gfa_core.mas.utils.intent_parser import resolve_gwas_intent resolved_gwas_request = resolve_gwas_intent(request.message or "") authoritative_full_gwas_stages = ( list(resolved_gwas_request.get("stages") or []) if resolved_gwas_request and resolved_gwas_request.get("full_chain") else [] ) if authoritative_full_gwas_stages: gwas_params = dict(incoming_intent.get("params") or {}) gwas_prefix = incoming_intent.get("prefix") or gwas_params.get("prefix") or request.prefix or "cohort" gwas_params["prefix"] = gwas_prefix incoming_intent.update({ "stages": authoritative_full_gwas_stages, "primary_stage": resolved_gwas_request["target"], "task_kind": "pipeline", "taskType": "pipeline", "prefix": gwas_prefix, "params": gwas_params, "description": "执行完整 GWAS 分析链", "confidence": 0.98, }) incoming_params = dict(incoming_intent.get("params") or {}) def _request_list(*keys): for key in keys: value = incoming_intent.get(key) if value is None: value = incoming_params.get(key) if isinstance(value, str): return [item.strip() for item in re.split(r"[,;，；\s]+", value) if item.strip()] if isinstance(value, list): return [str(item).strip() for item in value if str(item).strip()] return [] requested_target_species = _canonicalize_species_codes_for_job(request.job_id, _request_list("targetSpecies", "target_species")) requested_comparative_species = _canonicalize_species_codes_for_job(request.job_id, _request_list("comparativeSpecies", "comparative_species")) requested_reference_species = _canonicalize_species_codes_for_job(request.job_id, _request_list("referenceSpecies", "reference_species")) requested_scope_mode = incoming_intent.get("scopeMode") or incoming_params.get("scopeMode") or "" requested_stage = ( incoming_intent.get("primary_stage") or incoming_intent.get("stage") or ((incoming_intent.get("stages") or ["agent"])[-1] if incoming_intent.get("stages") else "agent") ) requested_prefix = incoming_intent.get("prefix") or incoming_params.get("prefix") or request.prefix or "" effective_prefix_params = {**incoming_params, "prefix": requested_prefix} effective_plan_prefix = resolve_effective_prefix( request.job_id, effective_prefix_params, requested_stage, ) if effective_plan_prefix: incoming_intent["prefix"] = effective_plan_prefix incoming_params["prefix"] = effective_plan_prefix incoming_intent["params"] = incoming_params query_lower_early = (request.message or "").lower() subfamily_confirm_query_early = any( kw in query_lower_early for kw in ["确认亚家族", "亚家族确认", "确认亚家族标签", "人工确认亚家族", "subfamily confirm", "confirmed subfamily"] ) subfamily_prediction_query_early = ( any(kw in query_lower_early for kw in ["亚家族", "subfamily"]) and not _skip_subfamily_label_query(query_lower_early) and not subfamily_confirm_query_early and not _looks_like_gene_family_workflow_query(query_lower_early) and not _requires_confirmed_subfamily_query(query_lower_early) ) if subfamily_prediction_query_early: request_roles_for_subfamily = _resolve_subfamily_request_species_roles( request.job_id, request.message or "", incoming_intent, ) current_targets = _canonicalize_species_codes_for_job( request.job_id, _param_list(incoming_params.get("targetSpecies")) or _param_list(incoming_intent.get("target_species")), ) target_species_for_subfamily = request_roles_for_subfamily["classification_species"] or current_targets family_for_subfamily = _resolve_authoritative_gene_family( request.job_id, incoming_params.get("geneFamily") or incoming_intent.get("gene_family"), effective_plan_prefix or requested_prefix, incoming_params, request.message or "", subfamily=True, ) scope_defaults_for_subfamily = _latest_gene_family_scope_defaults(request.job_id, family_for_subfamily, effective_plan_prefix or requested_prefix) if not target_species_for_subfamily: target_species_for_subfamily = scope_defaults_for_subfamily.get("target_species") or _target_species_fallback_from_prefix(effective_plan_prefix or requested_prefix) identify_selection_for_subfamily = _load_identify_species_selection(request.job_id) if request.job_id else {} reference_defaults_for_subfamily = request_roles_for_subfamily["reference_species"] or _primary_reference_species_from_scope(request.job_id, family_for_subfamily, effective_plan_prefix or requested_prefix) if not reference_defaults_for_subfamily: reference_defaults_for_subfamily = scope_defaults_for_subfamily.get("reference_species") or [] if not reference_defaults_for_subfamily: reference_defaults_for_subfamily = _reference_species_from_config(request.job_id, family_for_subfamily, target_species_for_subfamily) if not reference_defaults_for_subfamily: input_dir = INPUT_ROOT / request.job_id if request.job_id else None if input_dir and (input_dir / f"At.{family_for_subfamily}.subfamily").exists(): reference_defaults_for_subfamily = ["At"] if target_species_for_subfamily: incoming_params["targetSpecies"] = target_species_for_subfamily incoming_intent["target_species"] = target_species_for_subfamily incoming_intent["species"] = target_species_for_subfamily[0] if len(target_species_for_subfamily) == 1 else "Multi" if reference_defaults_for_subfamily: incoming_params["referenceSpecies"] = reference_defaults_for_subfamily incoming_intent["reference_species"] = reference_defaults_for_subfamily incoming_params["scopeMode"] = "target_plus_reference" incoming_intent["scopeMode"] = "target_plus_reference" incoming_params["comparativeSpecies"] = request_roles_for_subfamily["comparative_species"] incoming_intent["comparative_species"] = request_roles_for_subfamily["comparative_species"] incoming_params["geneFamily"] = family_for_subfamily incoming_intent["gene_family"] = family_for_subfamily if target_species_for_subfamily and family_for_subfamily: incoming_params["prefix"] = f"{target_species_for_subfamily[0]}.{family_for_subfamily}" if len(target_species_for_subfamily) == 1 else f"Target.{family_for_subfamily}" incoming_intent["prefix"] = incoming_params["prefix"] incoming_intent["params"] = incoming_params if subfamily_prediction_query_early and incoming_intent.get("stages") and "reference_classification" in incoming_intent.get("stages", []): requested_deterministic = [str(stage).strip() for stage in incoming_intent.get("stages", []) if str(stage).strip()] evidence_chain = ["identify", "phylogeny", "motif", "domain", "structure", "combo", "reference_classification"] deterministic_stages = [stage for stage in evidence_chain if stage in set(requested_deterministic) or stage in evidence_chain] deterministic_stages.extend(stage for stage in requested_deterministic if stage not in deterministic_stages) deterministic_stages = list(dict.fromkeys(deterministic_stages)) family_for_subfamily = _resolve_authoritative_gene_family( request.job_id, incoming_params.get("geneFamily") or incoming_intent.get("gene_family"), effective_plan_prefix or requested_prefix, incoming_params, request.message or "", subfamily=True, ) request_roles_for_subfamily = _resolve_subfamily_request_species_roles( request.job_id, request.message or "", incoming_intent, ) targets_for_subfamily = request_roles_for_subfamily["classification_species"] or _canonicalize_species_codes_for_job( request.job_id, _param_list(incoming_params.get("targetSpecies")) or _param_list(incoming_intent.get("target_species")), ) scope_defaults_for_subfamily = _latest_gene_family_scope_defaults(request.job_id, family_for_subfamily, effective_plan_prefix or requested_prefix) if not targets_for_subfamily: targets_for_subfamily = scope_defaults_for_subfamily.get("target_species") or _target_species_fallback_from_prefix(effective_plan_prefix or requested_prefix) refs_for_subfamily = request_roles_for_subfamily["reference_species"] or _canonicalize_species_codes_for_job( request.job_id, _param_list(incoming_params.get("referenceSpecies")) or _param_list(incoming_intent.get("reference_species")) or scope_defaults_for_subfamily.get("reference_species"), ) if not refs_for_subfamily: selection = _load_identify_species_selection(request.job_id) if request.job_id else {} refs_for_subfamily = [ item.get("species") for item in ((selection.get("groups") or {}).get("reference") or []) if item.get("species") ] if not refs_for_subfamily: refs_for_subfamily = _reference_species_from_config(request.job_id, family_for_subfamily, targets_for_subfamily) if not refs_for_subfamily: return { "success": False, "mode": "deterministic_subfamily_from_intent", "message": f"亚家族分类需要参考物种；已解析目标物种={targets_for_subfamily}、Family={family_for_subfamily}，但未从 config.xlsx 找到 seed_reference/known_family reference。", "issues": [{"code": "subfamily_missing_reference_species", "severity": "blocking"}], "session_id": session_id, } prefix_for_subfamily = f"{targets_for_subfamily[0]}.{family_for_subfamily}" if len(targets_for_subfamily) == 1 else f"Target.{family_for_subfamily}" identify_selection_for_plan = _load_identify_species_selection(request.job_id) if request.job_id else {} analysis_mode_for_subfamily = str(incoming_params.get("analysisMode") or incoming_params.get("analysis_mode") or "").strip() if analysis_mode_for_subfamily not in {"per_species", "joint_multi_species"}: analysis_mode_for_subfamily = "joint_multi_species" if len(targets_for_subfamily) > 1 else "per_species" classification_species_for_subfamily = list(targets_for_subfamily) classification_species_for_subfamily, role_targets_for_subfamily, role_comps_for_subfamily = _split_subfamily_selected_species_by_config_roles( request.job_id, classification_species_for_subfamily, identify_selection_for_plan or {}, scope_defaults_for_subfamily.get("target_species") or targets_for_subfamily, ) confirm_prefix_by_species = {sp: f"{sp}.{family_for_subfamily}" for sp in classification_species_for_subfamily if sp and family_for_subfamily} subfamily_state_for_plan = _subfamily_state_for_targets(request.job_id, classification_species_for_subfamily, family_for_subfamily) if classification_species_for_subfamily and family_for_subfamily else {} subfamily_statuses_for_plan = { str((data or {}).get("status") or "") for data in (subfamily_state_for_plan.get("by_species") or {}).values() } # reference_classification owns the review popup. Do not add # subfamily_confirm as a normal batch stage unless the user # explicitly asked for confirmation. if subfamily_confirm_query_early and ( "predicted_unconfirmed" in subfamily_statuses_for_plan and not (subfamily_statuses_for_plan & {"missing_identify", "not_classified", "confirmed_mismatch"}) ): deterministic_stages = ["subfamily_confirm"] plan_params = { **incoming_params, "prefix": prefix_for_subfamily, "geneFamily": family_for_subfamily, "targetSpecies": role_targets_for_subfamily, "referenceSpecies": refs_for_subfamily, "comparativeSpecies": role_comps_for_subfamily, "scopeMode": _subfamily_scope_mode_for_roles(refs_for_subfamily, role_comps_for_subfamily), "analysisMode": analysis_mode_for_subfamily, "speciesQueue": classification_species_for_subfamily, "confirmPrefixBySpecies": confirm_prefix_by_species, "subfamilyState": subfamily_state_for_plan, } if identify_selection_for_plan: selected_for_subfamily, selection_mode = _select_subfamily_classification_species( request.job_id, family_for_subfamily, identify_selection_for_plan, request.message or "", targets_for_subfamily, ) if selected_for_subfamily: classification_species_for_subfamily, role_targets_for_subfamily, role_comps_for_subfamily = _split_subfamily_selected_species_by_config_roles( request.job_id, selected_for_subfamily, identify_selection_for_plan or {}, scope_defaults_for_subfamily.get("target_species") or targets_for_subfamily, ) targets_for_subfamily = classification_species_for_subfamily plan_params["targetSpecies"] = role_targets_for_subfamily plan_params["comparativeSpecies"] = role_comps_for_subfamily plan_params["scopeMode"] = _subfamily_scope_mode_for_roles(refs_for_subfamily, role_comps_for_subfamily) plan_params["speciesQueue"] = classification_species_for_subfamily plan_params["confirmPrefixBySpecies"] = {sp: f"{sp}.{family_for_subfamily}" for sp in classification_species_for_subfamily if sp and family_for_subfamily} plan_params["subfamilyState"] = _subfamily_state_for_targets(request.job_id, classification_species_for_subfamily, family_for_subfamily) if classification_species_for_subfamily and family_for_subfamily else {} subfamily_statuses_for_plan = { str((data or {}).get("status") or "") for data in ((plan_params.get("subfamilyState") or {}).get("by_species") or {}).values() } if subfamily_confirm_query_early and ( "predicted_unconfirmed" in subfamily_statuses_for_plan and not (subfamily_statuses_for_plan & {"missing_identify", "not_classified", "confirmed_mismatch"}) ): deterministic_stages = ["subfamily_confirm"] plan_params["subfamilySelectionMode"] = selection_mode identify_selection_for_plan = _identify_selection_with_default( identify_selection_for_plan, targets_for_subfamily, "已根据本次亚家族分类请求确定待分类物种；reference 仅作为参考来源。", ) plan_params["identifySpeciesSelection"] = identify_selection_for_plan stages_detail = [] for order, stage_id in enumerate(deterministic_stages, start=1): detail = _stage_detail_defaults(stage_id) detail["order"] = order detail_params = dict(plan_params) if stage_id == "identify": detail_params["scopeMode"] = "target_only" detail_params["targetSpecies"] = list(plan_params.get("speciesQueue") or []) detail_params["referenceSpecies"] = [] detail_params["comparativeSpecies"] = [] detail_params.pop("analysisRoute", None) elif stage_id in {"phylogeny", "reference_classification", "motif", "domain", "structure", "combo"}: detail_params["scopeMode"] = plan_params.get("scopeMode") or _subfamily_scope_mode_for_roles(refs_for_subfamily, plan_params.get("comparativeSpecies") or []) detail_params["targetSpecies"] = plan_params.get("targetSpecies") or [] detail_params["referenceSpecies"] = refs_for_subfamily detail_params["comparativeSpecies"] = plan_params.get("comparativeSpecies") or [] detail_params["subfamilyPipeline"] = True if stage_id in {"phylogeny", "motif", "domain", "structure", "combo"}: detail_params["analysisRoute"] = "subfamily_evidence" if stage_id == "phylogeny": detail["dependencies"] = ["identify"] if "identify" in deterministic_stages else [] elif stage_id in {"motif", "domain", "structure"}: detail["dependencies"] = ["phylogeny"] elif stage_id == "combo": detail["dependencies"] = ["motif", "domain", "structure"] if stage_id == "reference_classification": detail["dependencies"] = ["combo"] if "combo" in deterministic_stages else ["phylogeny"] detail_params["analysisRoute"] = "subfamily_prediction" elif stage_id == "subfamily_confirm": detail.update({ "name": "亚家族标签确认", "dependencies": ["reference_classification"] if "reference_classification" in deterministic_stages else [], "estimated_time": "人工确认", }) detail_params.update({ "analysisRoute": "confirmed_subfamily", "analysis_route": "confirmed_subfamily", "sourcePredictionRoute": "subfamily_prediction", "source_prediction_route": "subfamily_prediction", "interactiveSubfamilyReview": True, "subfamilyConfirmSpecies": targets_for_subfamily, "geneFamily": family_for_subfamily, }) detail["params"] = enrich_stage_params_with_context(stage_id, prefix_for_subfamily, detail_params) stages_detail.append(detail) requirements_report = {} try: from gfa_core.mas.stage_requirements import resolve_many_stage_requirements requirements_report = resolve_many_stage_requirements( stages=deterministic_stages, prefix=prefix_for_subfamily, job_id=request.job_id, params_by_stage={item["id"]: item.get("params", {}) for item in stages_detail if item.get("id")}, shared_params=plan_params, project_root=PROJECT_ROOT, shadow_only=True, ) except Exception as exc: requirements_report = {"error": str(exc), "shadow_only": True} estimated_minutes = sum( int(str(item.get("estimated_time", "10")).replace("分钟", "") or 10) if str(item.get("estimated_time", "")).replace("分钟", "").isdigit() else 10 for item in stages_detail ) response = { "success": True, "mode": "deterministic_subfamily_from_intent", "plan": { "analysis_type": "gene_family", "gene_family": family_for_subfamily, "species": (plan_params.get("speciesQueue") or [""])[0] if len(plan_params.get("speciesQueue") or []) == 1 else "Multi", "target_species": plan_params.get("targetSpecies", []), "reference_species": refs_for_subfamily, "comparative_species": plan_params.get("comparativeSpecies", []), "scopeMode": plan_params.get("scopeMode", "target_plus_reference"), "prefix": prefix_for_subfamily, "description": "已有预测文件时先进行亚家族标签确认；缺失预测时再执行 target/comparative + reference 亚家族分类", "estimated_time": f"{estimated_minutes}分钟", "estimated_total_time": f"{estimated_minutes}分钟", "execution_strategy": "parallel", "max_parallel": 4, "total_stages": len(deterministic_stages), "stages": deterministic_stages, "stages_detail": stages_detail, "parallel_groups": [], "params": plan_params, "requirements_report": requirements_report, }, "intent": { **incoming_intent, "gene_family": family_for_subfamily, "species": (plan_params.get("speciesQueue") or [""])[0] if len(plan_params.get("speciesQueue") or []) == 1 else "Multi", "target_species": plan_params.get("targetSpecies", []), "reference_species": refs_for_subfamily, "comparative_species": plan_params.get("comparativeSpecies", []), "scopeMode": plan_params.get("scopeMode", "target_plus_reference"), "stages": deterministic_stages, "params": plan_params, }, "completed_stages": [], "session_id": session_id, } print(f"[Agent Plan] Deterministic subfamily intent plan: targets={targets_for_subfamily}, refs={refs_for_subfamily}, stages={deterministic_stages}") return response contextual_request_early = _resolve_contextual_gene_family_request( request.job_id, request.message, request.prefix, requested_stages=list(incoming_intent.get("stages") or []), request_intent=incoming_intent, ) if contextual_request_early and not subfamily_prediction_query_early and not subfamily_confirm_query_early: response = _build_contextual_gene_family_plan_response(request, session_id, contextual_request_early) plan_for_log = response.get('plan') or {} if response.get('success') and plan_for_log: print(f"[Agent Plan] Deterministic contextual plan: {plan_for_log.get('stages')}, prefix={plan_for_log.get('prefix')}") else: print(f"[Agent Plan] Deterministic contextual plan blocked: {response.get('message')}; issues={response.get('issues')}") return response # WGS follows the deterministic Manifest-driven planning path used # by RNA-seq. The current message is authoritative; stale intent # from a previous chat turn must not select the stage. from gfa_core.mas.utils.intent_parser import parse_intent_from_query current_wgs_intent = parse_intent_from_query(request.message or "") current_wgs_stages = [ str(stage).strip() for stage in (current_wgs_intent.get("stages") or []) if str(stage).strip().startswith("wgs_") ] effective_wgs_job_id = str( request.job_id or incoming_intent.get("job_id") or current_job_id or "" ).strip() if effective_wgs_job_id == "default_job": effective_wgs_job_id = "" if current_wgs_stages and effective_wgs_job_id: from gfa_core.mas.stage_catalog import complete_dependencies, STAGE_DEPENDENCIES from gfa_core.mas.tools.stage_status import get_stage_status wgs_stages = complete_dependencies(current_wgs_stages) incoming_wgs_params = incoming_intent.get("params") or {} wgs_params = { key: incoming_wgs_params[key] for key in ("wgsMethod", "wgs_method", "method") if key in incoming_wgs_params } wgs_params.update(current_wgs_intent.get("params") or {}) requested_wgs_prefix = ( request.prefix or wgs_params.get("prefix") or incoming_intent.get("prefix") or "" ) try: wgs_prefix = _wgs_prefix_for_plan( effective_wgs_job_id, wgs_stages, requested_wgs_prefix ) or "cohort" except ValueError as exc: return { "success": False, "mode": "deterministic_wgs_plan", "message": str(exc), "session_id": session_id, } completed_wgs_stages = [] completed_wgs_authority_stages = [] remaining_wgs_stages = [] wgs_status_summary = {} completed_wgs_methods = {} for stage_id in wgs_stages: try: identity_params = _wgs_manifest_identity_for_plan( effective_wgs_job_id, stage_id, wgs_prefix ) except ValueError as exc: return { "success": False, "mode": "deterministic_wgs_plan", "message": str(exc), "session_id": session_id, } status = get_stage_status( job_id=effective_wgs_job_id, prefix=wgs_prefix, stage=stage_id, wgs_sample_set_id=identity_params.get("wgsSampleSetId", ""), wgs_reference_id=identity_params.get("wgsReferenceId", ""), wgs_analysis_id=identity_params.get("wgsAnalysisId", ""), validate_wgs_upstream_authority=False, ) wgs_status_summary[stage_id] = { "status": status.status, "completed": bool(status.completed), "manifest_path": str(status.manifest_path) if status.manifest_path else "", "missing_count": len(status.missing_artifacts or []) + len(status.errors or []), } if status.completed: completed_wgs_stages.append(stage_id) if status.manifest_path and re.fullmatch( r"wgs-analysis:[0-9a-f]{64}", str(identity_params.get("wgsAnalysisId") or ""), ): completed_wgs_authority_stages.append(stage_id) if stage_id in {"wgs_variant_select", "wgs_variant_filter"}: manifest = getattr(status, "manifest", None) if not isinstance(manifest, dict) and status.manifest_path: try: manifest = json.loads( Path(status.manifest_path).read_text(encoding="utf-8") ) except (OSError, json.JSONDecodeError): manifest = None if isinstance(manifest, dict): method = str( manifest.get("method") or (manifest.get("parameters") or {}).get("method") or "" ).strip().lower() if method in {"gatk", "bcftools"}: completed_wgs_methods[stage_id] = method else: remaining_wgs_stages.append(stage_id) explicit_wgs_method = str( wgs_params.get("wgsMethod") or wgs_params.get("wgs_method") or wgs_params.get("method") or "" ).strip().lower() for consumer, producer in ( ("wgs_variant_filter", "wgs_variant_select"), ("wgs_impute", "wgs_variant_filter"), ): if consumer not in remaining_wgs_stages: continue authority_method = completed_wgs_methods.get(producer, "") if not authority_method: continue if explicit_wgs_method and explicit_wgs_method != authority_method: return { "success": False, "mode": "deterministic_wgs_plan", "message": ( f"WGS method conflict: requested {explicit_wgs_method}, " f"but completed {producer} authority uses {authority_method}" ), "session_id": session_id, } wgs_params["wgsMethod"] = authority_method explicit_wgs_method = authority_method wgs_names = { "wgs_qc": "WGS质控", "wgs_index": "WGS参考索引", "wgs_align": "WGS序列比对", "wgs_markdup": "WGS去重复", "wgs_bam_qc": "WGS BAM质控", "wgs_haplotype_calling": "WGS单样本变异检测", "wgs_gvcf_combine": "WGS GVCF合并", "wgs_joint_genotyping": "WGS联合分型", "wgs_variant_select": "WGS变异类型选择", "wgs_variant_filter": "WGS变异硬过滤", "wgs_impute": "WGS基因型填充", "wgs_maf_missing_filter": "WGS MAF与缺失率过滤", } wgs_details = [] for order, stage_id in enumerate(remaining_wgs_stages, start=1): wgs_details.append({ "id": stage_id, "name": wgs_names.get(stage_id, stage_id), "description": wgs_names.get(stage_id, stage_id), "order": order, "dependencies": list(STAGE_DEPENDENCIES.get(stage_id, [])), "can_parallel": False, "estimated_time": 30, "params": {"prefix": wgs_prefix, "cpu": 8, **wgs_params}, }) wgs_plan_params = {"prefix": wgs_prefix, **wgs_params} _clear_planned_wgs_upstream_bindings( remaining_wgs_stages, wgs_plan_params, wgs_details ) try: from gfa_core.wgs_upstream_manifest import ( WgsUpstreamManifestError, materialize_completed_wgs_upstream_params, ) for detail in wgs_details: stage_id = str(detail.get("id") or "") if stage_id in {"wgs_qc", "wgs_index"}: continue detail["params"].update( materialize_completed_wgs_upstream_params( stage=stage_id, job_id=effective_wgs_job_id, prefix=wgs_prefix, params=detail["params"], completed_stages=completed_wgs_authority_stages, workspace=WORKSPACE_ROOT / effective_wgs_job_id, ) ) except WgsUpstreamManifestError as exc: return { "success": False, "mode": "deterministic_wgs_plan", "message": str(exc), "session_id": session_id, } wgs_plan_intent = { **current_wgs_intent, "prefix": wgs_prefix, "params": dict(wgs_plan_params), "stages": current_wgs_stages, } return { "success": True, "mode": "deterministic_wgs_plan", "plan": { "analysis_type": "wgs", "gene_family": "", "species": "", "prefix": wgs_prefix, "description": "所有阶段已完成" if not remaining_wgs_stages else "执行WGS分析", "estimated_time": f"{len(wgs_details) * 30}分钟", "estimated_total_time": f"{len(wgs_details) * 30}分钟", "execution_strategy": "parallel", "max_parallel": 4, "total_stages": len(remaining_wgs_stages), "stages": remaining_wgs_stages, "stages_detail": wgs_details, "parallel_groups": [], "params": wgs_plan_params, "stage_status_summary": wgs_status_summary, }, "intent": wgs_plan_intent, "completed_stages": completed_wgs_stages, "session_id": session_id, } print( f"[Agent Plan] Deterministic WGS plan: job_id={effective_wgs_job_id}, " f"prefix={wgs_prefix}, remaining={remaining_wgs_stages}" ) rnaseq_intent_stages = [str(stage).strip() for stage in (incoming_intent.get("stages") or []) if str(stage).strip()] if rnaseq_intent_stages and all(stage.startswith("rnaseq_") for stage in rnaseq_intent_stages) and request.job_id: try: from gfa_core.mas.stage_catalog import complete_dependencies from gfa_core.mas.tools.stage_status import get_stage_status expanded_rnaseq_stages = complete_dependencies(rnaseq_intent_stages) completed_rnaseq_stages = [] remaining_rnaseq_stages = [] stage_status_summary = {} for stage_id in expanded_rnaseq_stages: status = get_stage_status( job_id=request.job_id, prefix=effective_plan_prefix or request.prefix or incoming_intent.get("prefix", ""), stage=stage_id, ) stage_status_summary[stage_id] = { "status": status.status, "completed": bool(status.completed), "manifest_path": str(status.manifest_path) if status.manifest_path else "", "missing_count": len(status.missing_artifacts or []) + len(status.errors or []), } if status.completed: completed_rnaseq_stages.append(stage_id) else: remaining_rnaseq_stages.append(stage_id) rnaseq_names = { "rnaseq_qc": "RNA-seq QC", "rnaseq_align": "RNA-seq Align", "rnaseq_quant": "RNA-seq Quant", "rnaseq_sample_qc": "RNA-seq Sample QC", "rnaseq_de": "RNA-seq DE", "rnaseq_go_enrich": "RNA-seq GO Enrich", "rnaseq_kegg_enrich": "RNA-seq KEGG Enrich", "rnaseq_fgsea": "RNA-seq fgsea", "rnaseq_wgcna": "RNA-seq WGCNA", "rnaseq_candidate_family": "RNA-seq Candidate Family", } plan_prefix = effective_plan_prefix or request.prefix or incoming_intent.get("prefix", "") final_dir_for_rnaseq = PROJECT_ROOT / "workspace" / request.job_id / "final" gene2ko_candidates = [] if final_dir_for_rnaseq.exists(): gene2ko_candidates.extend(final_dir_for_rnaseq.glob("*gene2ko*.txt")) gene2ko_candidates.extend(final_dir_for_rnaseq.glob("*gene2ko*.tsv")) has_gene2ko_background = any(path.exists() and path.stat().st_size > 0 for path in gene2ko_candidates) rnaseq_deps = { "rnaseq_align": ["rnaseq_qc"], "rnaseq_quant": ["rnaseq_align"], "rnaseq_sample_qc": ["rnaseq_quant"], "rnaseq_de": ["rnaseq_sample_qc"], "rnaseq_go_enrich": ["rnaseq_de"], "rnaseq_kegg_enrich": ["rnaseq_de"], "rnaseq_fgsea": ["rnaseq_de"], "rnaseq_wgcna": ["rnaseq_sample_qc"], "rnaseq_candidate_family": ["rnaseq_de", "rnaseq_go_enrich", "rnaseq_kegg_enrich", "rnaseq_fgsea"], } if ( "rnaseq_fgsea" in remaining_rnaseq_stages and "rnaseq_kegg_enrich" in expanded_rnaseq_stages and not has_gene2ko_background ): # fgsea consumes gene2ko, not KEGG enrichment results. In an # automatic batch, KEGG enrichment/annotation may be the task # that creates gene2ko, so wait for it to avoid a foreground # race and keep direct fgsea runs fast when gene2ko exists. rnaseq_deps["rnaseq_fgsea"] = ["rnaseq_de", "rnaseq_kegg_enrich"] rnaseq_estimates = { "rnaseq_qc": 10, "rnaseq_align": 30, "rnaseq_quant": 15, "rnaseq_sample_qc": 10, "rnaseq_de": 15, "rnaseq_go_enrich": 10, "rnaseq_kegg_enrich": 10, "rnaseq_fgsea": 10, "rnaseq_wgcna": 20, "rnaseq_candidate_family": 10, } stages_detail = [] for order, stage_id in enumerate(remaining_rnaseq_stages, start=1): stages_detail.append({ "id": stage_id, "name": rnaseq_names.get(stage_id, stage_id), "description": rnaseq_names.get(stage_id, stage_id), "order": order, "dependencies": rnaseq_deps.get(stage_id, []), "can_parallel": False, "estimated_time": rnaseq_estimates.get(stage_id, 10), "params": {"prefix": plan_prefix, "cpu": 8, **incoming_params}, }) estimated_minutes = sum(int(item.get("estimated_time", 10)) for item in stages_detail) description = "所有阶段已完成" if not remaining_rnaseq_stages else f"执行RNA-seq的{', '.join(remaining_rnaseq_stages)}分析" response = { "success": True, "mode": "deterministic_rnaseq_plan", "plan": { "analysis_type": "rnaseq", "gene_family": incoming_intent.get("gene_family", ""), "species": incoming_intent.get("species", ""), "prefix": plan_prefix, "description": description, "estimated_time": f"{estimated_minutes}分钟", "estimated_total_time": f"{estimated_minutes}分钟", "execution_strategy": "parallel", "max_parallel": 4, "total_stages": len(remaining_rnaseq_stages), "stages": remaining_rnaseq_stages, "stages_detail": stages_detail, "parallel_groups": [], "params": {"prefix": plan_prefix, "cpu": 8, **incoming_params}, "stage_status_summary": stage_status_summary, }, "intent": {**incoming_intent, "stages": rnaseq_intent_stages, "prefix": plan_prefix}, "completed_stages": completed_rnaseq_stages, "session_id": session_id, } print(f"[Agent Plan] Deterministic RNA-seq plan: remaining={remaining_rnaseq_stages}, completed={completed_rnaseq_stages}") return response except Exception as exc: print(f"[Agent Plan] Deterministic RNA-seq plan skipped: {exc}") # 构建初始状态 initial_state: GraphState = { "query": request.message, "session_id": session_id, "job_id": request.job_id, # 添加 job_id 用于 manifest 查询 "files": [], "current_agent": "planning", "status": "running", "errors": [], "final_output": {}, "history": request.history if request.history else [], "results": {} # 用于存储已完成阶段 } # 不再把 prefix 级粗扫 completed_stages 注入规划器； # 具体阶段是否完成必须在 plan/stages_detail 生成后按 identity 逐项判断。 workspace_name = request.job_id if request.job_id else request.session_id if workspace_name: initial_state["results"]["completed_stages"] = [] print(f"[Agent Plan] Job_id: {workspace_name}, prefix={effective_plan_prefix or request.prefix}, Completed stages deferred to identity-aware check") # 如果前端传入了 intent，直接放入 state（避免 orchestrator.plan 重复调用 IntentAgentV2） if incoming_intent and incoming_intent.get("stages"): initial_state["results"]["intent"] = incoming_intent print(f"[Agent Plan] 使用前端传入的 intent: {incoming_intent.get('stages', [])}, prefix={incoming_intent.get('prefix', '')}") # Fast deterministic path for explicit combo requests. A direct combo # request can be resolved from current job artifacts; sending it through # PlanningAgentV2 first may scan all stage contracts/manifests and block # the single web process while the frontend only shows "AI thinking". direct_combo_chain_query_early = ( (bool(re.search(r"combo", request.message or "", flags=re.IGNORECASE)) or "组合图" in (request.message or "")) and "reference_classification" not in (incoming_intent.get("stages") or []) ) if direct_combo_chain_query_early: combo_targets = incoming_params.get("targetSpecies") or incoming_intent.get("target_species") or requested_target_species or [] if isinstance(combo_targets, str): combo_targets = [item.strip() for item in re.split(r"[,;，；\s]+", combo_targets) if item.strip()] combo_family = incoming_params.get("geneFamily") or incoming_intent.get("gene_family") or "" combo_prefix = incoming_params.get("prefix") or incoming_intent.get("prefix") or effective_plan_prefix or request.prefix or "" if not combo_family and combo_prefix and "." in combo_prefix: combo_family = combo_prefix.split(".", 1)[1] if not combo_targets and combo_prefix and "." in combo_prefix: prefix_species = combo_prefix.split(".", 1)[0] if prefix_species not in {"Target", "Multi"}: combo_targets = [prefix_species] if combo_targets and combo_family: member_ready = True for combo_sp in combo_targets: member_ready = member_ready and bool(_target_family_member_files(request.job_id, combo_sp, combo_family).get("pep")) subfamily_state = _subfamily_state_for_targets(request.job_id, combo_targets, combo_family) subfamily_by_species = subfamily_state.get("by_species") or {} subfamily_statuses = { str((data or {}).get("status") or "") for data in subfamily_by_species.values() } skip_combo_subfamily_labels = _skip_subfamily_label_query(request.message or "") needs_reference_classification = ( not skip_combo_subfamily_labels and bool(subfamily_statuses & {"missing_identify", "not_classified", "confirmed_mismatch"}) ) needs_subfamily_confirm = ( not skip_combo_subfamily_labels and bool(subfamily_statuses & {"missing_identify", "not_classified", "predicted_unconfirmed", "confirmed_mismatch"}) ) selected_combo_panels = _resolve_standard_combo_panels_for_plan( stage_ids=list(incoming_intent.get("stages") or []), plan_params=incoming_params, job_id=request.job_id, prefix=combo_prefix, direct_combo=True, ) combo_chain = [] if not member_ready or "missing_identify" in subfamily_statuses: combo_chain.append("identify") downstream_chain = ["phylogeny", *selected_combo_panels, "combo"] if needs_reference_classification: combo_chain.extend(downstream_chain) combo_chain.append("reference_classification") else: if needs_subfamily_confirm: combo_chain.append("subfamily_confirm") combo_chain.extend(downstream_chain) combo_chain = list(dict.fromkeys(combo_chain)) confirmed_ready = bool(combo_targets) and bool(subfamily_state.get("all_confirmed")) explicit_combo_rerun = _explicit_force_rerun_requested(request.message or "", incoming_params) force_rerun_stage_ids = set(combo_chain) if explicit_combo_rerun else set() plan_params = { **incoming_params, "prefix": combo_prefix, "targetSpecies": combo_targets, "geneFamily": combo_family, "scopeMode": incoming_params.get("scopeMode") or "target_only", "referenceSpecies": incoming_params.get("referenceSpecies") or [], "comparativeSpecies": incoming_params.get("comparativeSpecies") or [], "forceRerunStages": sorted(force_rerun_stage_ids), "subfamilyState": subfamily_state, "comboPanels": list(selected_combo_panels), } stages_detail = [] for order, sid in enumerate(combo_chain, start=1): detail = _stage_detail_defaults(sid) detail["order"] = order detail_params = {**plan_params} detail_params["forceRerun"] = sid in force_rerun_stage_ids detail_params["forceRerunStages"] = sorted(force_rerun_stage_ids) if sid == "reference_classification": detail["name"] = "单目标+参考亚家族分类与证据图" detail["dependencies"] = ["combo"] if "combo" in combo_chain else ["phylogeny"] detail_params.update({ "analysisRoute": "subfamily_prediction", "subfamilyPipeline": True, "perTargetEvidenceOnly": True, }) elif sid == "subfamily_confirm": detail["name"] = "亚家族标签确认" detail["dependencies"] = ["reference_classification"] if "reference_classification" in combo_chain else ["identify"] detail["estimated_time"] = "人工确认" detail_params.update({ "analysisRoute": "confirmed_subfamily", "analysis_route": "confirmed_subfamily", "sourcePredictionRoute": "subfamily_prediction", "source_prediction_route": "subfamily_prediction", "interactiveSubfamilyReview": True, "subfamilyConfirmSpecies": combo_targets, }) elif sid == "phylogeny" and "subfamily_confirm" in combo_chain and "reference_classification" not in combo_chain: detail["dependencies"] = ["subfamily_confirm"] elif sid in {"motif", "domain", "structure"}: detail["dependencies"] = ["phylogeny"] elif sid == "combo": detail["dependencies"] = list(selected_combo_panels) detail_params["comboPanels"] = list(selected_combo_panels) if "reference_classification" in combo_chain and sid in {"phylogeny", "motif", "domain", "structure", "combo"}: detail_params["analysisRoute"] = "subfamily_evidence" detail_params["subfamilyPipeline"] = True detail["params"] = enrich_stage_params_with_context(sid, combo_prefix, detail_params) stages_detail.append(detail) completed_stages = _identity_completed_stages_for_plan( request.job_id, combo_chain, plan_params, stages_detail, effective_prefix=effective_plan_prefix or "", request_prefix=request.prefix or "", force_rerun_stage_ids=force_rerun_stage_ids, ) remaining_stage_ids = [stage_id for stage_id in combo_chain if stage_id not in set(completed_stages)] estimated_minutes = sum( int(str(item.get("estimated_time", "10分钟")).replace("分钟", "") or 10) if str(item.get("estimated_time", "")).replace("分钟", "").isdigit() else 10 for item in stages_detail if (item.get("id") or item.get("name")) in remaining_stage_ids ) estimated_time = f"{estimated_minutes}分钟" response = { "success": True, "mode": "deterministic_direct_combo", "plan": { "analysis_type": "gene_family", "gene_family": combo_family, "species": combo_targets[0] if len(combo_targets) == 1 else "Multi", "target_species": combo_targets, "reference_species": plan_params.get("referenceSpecies", []), "comparative_species": plan_params.get("comparativeSpecies", []), "scopeMode": plan_params.get("scopeMode", "target_only"), "prefix": combo_prefix, "description": "执行combo分析", "estimated_time": estimated_time, "estimated_total_time": estimated_time, "execution_strategy": "parallel", "max_parallel": 4, "total_stages": len(remaining_stage_ids), "stages": remaining_stage_ids, "stages_detail": [ item for item in stages_detail if (item.get("id") or item.get("name")) in remaining_stage_ids ], "all_stages": combo_chain, "all_stages_detail": stages_detail, "parallel_groups": [], "params": plan_params, }, "intent": { "gene_family": combo_family, "species": combo_targets[0] if len(combo_targets) == 1 else "Multi", "target_species": combo_targets, "reference_species": plan_params.get("referenceSpecies", []), "comparative_species": plan_params.get("comparativeSpecies", []), "scopeMode": plan_params.get("scopeMode", "target_only"), "stages": combo_chain, "description": "执行combo分析", "confidence": 0.95, "params": { "prefix": combo_prefix, "geneFamily": combo_family, "targetSpecies": combo_targets, "referenceSpecies": plan_params.get("referenceSpecies", []), "comparativeSpecies": plan_params.get("comparativeSpecies", []), "scopeMode": plan_params.get("scopeMode", "target_only"), }, }, "completed_stages": completed_stages, "session_id": session_id, } print(f"[Agent Plan] Deterministic direct combo plan: {combo_chain}, completed={completed_stages}, force_rerun={sorted(force_rerun_stage_ids)}, statuses={sorted(subfamily_statuses)}") return response # 调用PlanningAgent V2进行规划 print(f"[Agent Plan] Starting LLM planning for: {request.message}") result = await orchestrator.plan(initial_state) if result.get("status") == "completed": plan = result.get("final_output", {}).get("analysis_plan", {}) intent = result.get("final_output", {}).get("intent", {}) if authoritative_full_gwas_stages: gwas_prefix = ( intent.get("prefix") or (intent.get("params") or {}).get("prefix") or incoming_intent.get("prefix") or "cohort" ) intent_params = {**(intent.get("params") or {}), "prefix": gwas_prefix} intent.update({ "stages": authoritative_full_gwas_stages, "primary_stage": resolved_gwas_request["target"], "task_kind": "pipeline", "taskType": "pipeline", "prefix": gwas_prefix, "params": intent_params, "description": "执行完整 GWAS 分析链", "confidence": 0.98, }) existing_details = { str(item.get("id") or item.get("name") or ""): item for item in (plan.get("stages_detail") or []) if isinstance(item, dict) } from gfa_core.mas.stage_catalog import STAGE_DEPENDENCIES normalized_details = [] for order, stage_id in enumerate(authoritative_full_gwas_stages, start=1): detail = _stage_detail_defaults(stage_id) detail["order"] = order detail["dependencies"] = list(STAGE_DEPENDENCIES.get(stage_id, [])) if stage_id in existing_details: detail["params"] = dict(existing_details[stage_id].get("params") or {}) normalized_details.append(detail) plan["analysis_type"] = "gwas" plan["stages"] = authoritative_full_gwas_stages plan["stages_detail"] = normalized_details plan["total_stages"] = len(authoritative_full_gwas_stages) plan["description"] = "执行完整 GWAS 分析链" plan["params"] = {**(plan.get("params") or {}), "prefix": gwas_prefix} raw_stages = plan.get("stages", []) stages_detail = plan.get("stages_detail") or ( raw_stages if raw_stages and isinstance(raw_stages[0], dict) else [] ) stage_ids = [ stage.get("id", stage.get("name", "")) if isinstance(stage, dict) else stage for stage in raw_stages ] completed_stages = [] fallback_intent_stages = incoming_intent.get("stages") or intent.get("stages") or [] plan_reports_all_completed = ( not stage_ids and int(plan.get("total_stages") or 0) == 0 and str(plan.get("description") or "") in {"所有阶段已完成", "All stages completed"} ) if not stage_ids and fallback_intent_stages and not plan_reports_all_completed: stage_ids = [str(stage).strip() for stage in fallback_intent_stages if str(stage).strip()] raw_stages = stage_ids stages_detail = [_stage_detail_defaults(stage_id) for stage_id in stage_ids] plan["stages"] = stage_ids plan["stages_detail"] = stages_detail plan["total_stages"] = len(stage_ids) if all(stage_id.startswith("rnaseq_") for stage_id in stage_ids): plan["description"] = plan.get("description") or "执行RNA-seq完整分析流程" query_lower = (request.message or "").lower() subfamily_confirm_query = any( kw in query_lower for kw in ["确认亚家族", "亚家族确认", "确认亚家族标签", "人工确认亚家族", "subfamily confirm", "confirmed subfamily"] ) subfamily_prediction_query = ( any(kw in query_lower for kw in ["亚家族", "subfamily"]) and not _skip_subfamily_label_query(query_lower) and not subfamily_confirm_query ) if subfamily_prediction_query: stage_ids, stages_detail = _normalize_subfamily_prediction_plan( plan, intent, raw_stages, stages_detail, ) raw_stages = stage_ids estimated_time = ( plan.get("estimated_time") or plan.get("estimated_total_time") or ("0分钟" if not raw_stages else "未知") ) plan_params = plan.get("params") or { "prefix": resolve_effective_prefix(request.job_id, {"prefix": intent.get("prefix", request.prefix or "")}, "agent"), "cpu": 8, } _apply_inferred_family_to_intent(intent, request.job_id, request.message) if intent.get("prefix"): plan_params["prefix"] = intent["prefix"] if request.prefix and stage_ids and all(str(stage_id).startswith("rnaseq_") for stage_id in stage_ids): intent["prefix"] = request.prefix intent.setdefault("params", {})["prefix"] = request.prefix plan_params["prefix"] = request.prefix if plan_role_resolution.get("applicable"): _apply_reconciled_species_to_intent(intent, plan_role_resolution) else: _apply_scope_selection_to_intent(intent, request.message) if subfamily_prediction_query: if not plan_role_resolution.get("applicable"): explicit_species = _extract_species_codes_from_text_for_job(request.job_id, request.message or "") if explicit_species: intent["target_species"] = explicit_species intent.setdefault("params", {})["targetSpecies"] = explicit_species intent["species"] = explicit_species[0] if len(explicit_species) == 1 else "Multi" ref_defaults = list(plan_role_resolution.get("reference_species") or []) if not ref_defaults and request.job_id: ref_defaults = [ item.get("species") for item in ((_load_identify_species_selection(request.job_id).get("groups") or {}).get("reference") or []) if item.get("species") ] if ref_defaults: intent["reference_species"] = ref_defaults intent.setdefault("params", {})["referenceSpecies"] = ref_defaults intent["scopeMode"] = "target_plus_reference" intent["params"]["scopeMode"] = "target_plus_reference" scope_params = intent.get("params") or {} for key in ("targetSpecies", "referenceSpecies", "comparativeSpecies", "scopeMode", "geneFamily"): if scope_params.get(key): plan_params[key] = scope_params[key] contextual_request = _resolve_contextual_gene_family_request( request.job_id, request.message, request.prefix, requested_stages=list(intent.get("stages") or []), request_intent=intent, ) if contextual_request and not subfamily_prediction_query and not subfamily_confirm_query: stage_ids = list(contextual_request.get("stages") or []) raw_stages = stage_ids plan["stages"] = stage_ids plan["total_stages"] = len(stage_ids) plan["description"] = "基于已确认亚家族标签的传统基因家族分析" if contextual_request.get("confirmed_subfamily_available") else "自定义范围基因家族分析" stages_detail = [_stage_detail_defaults(stage_id) for stage_id in stage_ids] plan["stages_detail"] = stages_detail plan_params.update({ "geneFamily": contextual_request.get("family"), "prefix": contextual_request.get("prefix"), "targetSpecies": contextual_request.get("targetSpecies", []), "referenceSpecies": contextual_request.get("referenceSpecies", []), "comparativeSpecies": contextual_request.get("comparativeSpecies", []), "scopeMode": contextual_request.get("scopeMode", "target_only"), "contextRoute": contextual_request.get("context_route", ""), "subfamilyLabelSource": contextual_request.get("subfamily_label_source", ""), "requiresConfirmedSubfamily": bool(contextual_request.get("requiresConfirmedSubfamily")), "contextGate": contextual_request.get("contextGate", {}), }) intent["prefix"] = contextual_request.get("prefix") intent["gene_family"] = contextual_request.get("family") intent["target_species"] = contextual_request.get("targetSpecies", []) intent["reference_species"] = contextual_request.get("referenceSpecies", []) intent["comparative_species"] = contextual_request.get("comparativeSpecies", []) intent["scopeMode"] = contextual_request.get("scopeMode", "target_only") intent.setdefault("params", {}).update({ "prefix": contextual_request.get("prefix"), "geneFamily": contextual_request.get("family"), "targetSpecies": contextual_request.get("targetSpecies", []), "referenceSpecies": contextual_request.get("referenceSpecies", []), "comparativeSpecies": contextual_request.get("comparativeSpecies", []), "scopeMode": contextual_request.get("scopeMode", "target_only"), "contextRoute": contextual_request.get("context_route", ""), "subfamilyLabelSource": contextual_request.get("subfamily_label_source", ""), "requiresConfirmedSubfamily": bool(contextual_request.get("requiresConfirmedSubfamily")), }) context_gate = contextual_request.get("contextGate") or {} if context_gate.get("issues"): plan.setdefault("warnings", []).extend(context_gate.get("issues") or []) for detail in stages_detail: if not isinstance(detail, dict): continue detail_params = detail.setdefault("params", {}) detail_params["contextRoute"] = contextual_request.get("context_route", "") detail_params["subfamilyLabelSource"] = contextual_request.get("subfamily_label_source", "") detail_params["requiresConfirmedSubfamily"] = bool(contextual_request.get("requiresConfirmedSubfamily")) if contextual_request.get("context_route") == "custom_phylogeny_scope" and contextual_request.get("scopeMode") != "target_only": for detail in stages_detail: if isinstance(detail, dict) and (detail.get("id") or detail.get("name")) == "phylogeny": detail.setdefault("params", {})["analysisRoute"] = "custom_phylogeny" if intent.get("prefix"): plan_params["prefix"] = intent["prefix"] # Scope-aware gene-family routes use the current job output label. # Do not let a generic multi-species intent prefix such as # Multi.SPL hide already generated Target.SPL artifacts. if request.prefix and "reference_classification" in stage_ids: # 亚家族分类入口必须保留全局分析前缀；单物种前缀只在执行层 per-species 派生。 intent["prefix"] = request.prefix intent.setdefault("params", {})["prefix"] = request.prefix plan_params["prefix"] = request.prefix plan_prefix = plan_params.get("prefix") or effective_plan_prefix or request.prefix or "" traditional_target_only_plan = False if plan_prefix: for detail in stages_detail: if not isinstance(detail, dict): continue stage_id = detail.get("id") or detail.get("name") or "" detail_params = detail.setdefault("params", {}) merged_params = { **plan_params, **detail_params, } if "reference_classification" in stage_ids and stage_id == "identify": merged_params.pop("analysisRoute", None) merged_params.pop("analysis_route", None) merged_params.pop("memberSetId", None) merged_params.pop("member_set_id", None) if not merged_params.get("prefix"): merged_params["prefix"] = plan_prefix detail["params"] = enrich_stage_params_with_context(stage_id, plan_prefix, merged_params) if traditional_target_only_plan: detail["params"].pop("memberSetId", None) detail["params"].pop("member_set_id", None) for ctx_key in ("stageContext", "stage_context"): ctx = detail["params"].get(ctx_key) if isinstance(ctx, dict): ctx.pop("member_set_id", None) if "reference_classification" in stage_ids and stage_id == "identify": detail["params"].pop("analysisRoute", None) detail["params"].pop("analysis_route", None) detail["params"].pop("memberSetId", None) detail["params"].pop("member_set_id", None) identify_species_selection = {} if "identify" in stage_ids and not plan_params.get("targetSpecies") and not intent.get("target_species"): identify_species_selection = _load_identify_species_selection(request.job_id) if identify_species_selection: plan_params["identifySpeciesSelection"] = identify_species_selection plan_params["targetSpecies"] = identify_species_selection.get("defaultSelected", []) if "reference_classification" in stage_ids: reference_defaults = _primary_reference_species_from_scope(request.job_id, plan_params.get("geneFamily") or intent.get("gene_family") or "", effective_plan_prefix or requested_prefix) comparative_defaults = [ item.get("species") for item in (identify_species_selection.get("groups", {}).get("comparative") or []) if item.get("species") ] if reference_defaults: plan_params["referenceSpecies"] = reference_defaults plan_params["scopeMode"] = "target_plus_reference" if comparative_defaults and "comparative" in str(plan_params.get("scopeMode") or ""): plan_params["comparativeSpecies"] = comparative_defaults intent["target_species"] = plan_params["targetSpecies"] intent["reference_species"] = plan_params.get("referenceSpecies", []) intent["comparative_species"] = plan_params.get("comparativeSpecies", []) intent["scopeMode"] = plan_params.get("scopeMode", "target_only") for detail in stages_detail: if isinstance(detail, dict) and (detail.get("id") or detail.get("name")) == "identify": detail_params = detail.setdefault("params", {}) detail_params["identifySpeciesSelection"] = identify_species_selection detail_params["targetSpecies"] = plan_params["targetSpecies"] if not identify_species_selection and request.job_id and any( stage in {"identify", "phylogeny", "reference_classification", "motif", "domain", "structure", "combo", "promoter", "tfbs", "synteny", "karyotype", "pep", "ortholog", "ppi"} for stage in stage_ids ): identify_species_selection = _load_identify_species_selection(request.job_id) if subfamily_prediction_query: if requested_target_species: plan_params["targetSpecies"] = requested_target_species intent["target_species"] = requested_target_species selection_for_explicit_targets = dict(identify_species_selection or _load_identify_species_selection(request.job_id) or {}) if selection_for_explicit_targets: selection_for_explicit_targets["defaultSelected"] = requested_target_species selection_for_explicit_targets["message"] = "已按自然语言明确指定的物种选择本次亚家族分类目标；不会沿用 config 默认 target 或最近确认物种。" plan_params["identifySpeciesSelection"] = selection_for_explicit_targets identify_species_selection = selection_for_explicit_targets elif _looks_like_remaining_query(request.message or ""): remaining_species = _remaining_subfamily_species( request.job_id, intent.get("gene_family") or plan_params.get("geneFamily") or plan_params.get("gene_family") or "", identify_species_selection or _load_identify_species_selection(request.job_id), ) if remaining_species: plan_params["targetSpecies"] = remaining_species plan_params["comparativeSpecies"] = [] selection_for_remaining = dict(identify_species_selection or _load_identify_species_selection(request.job_id) or {}) if selection_for_remaining: selection_for_remaining["defaultSelected"] = remaining_species selection_for_remaining["message"] = "已自动选择尚未完成亚家族分类确认的基因组；reference 仅作为参考来源。" plan_params["identifySpeciesSelection"] = selection_for_remaining identify_species_selection = selection_for_remaining intent["target_species"] = remaining_species intent["comparative_species"] = [] if requested_comparative_species and (not subfamily_prediction_query or "comparative" in str(requested_scope_mode or plan_params.get("scopeMode") or "")): plan_params["comparativeSpecies"] = requested_comparative_species intent["comparative_species"] = requested_comparative_species elif subfamily_prediction_query: plan_params["comparativeSpecies"] = [] intent["comparative_species"] = [] if requested_reference_species: plan_params["referenceSpecies"] = requested_reference_species intent["reference_species"] = requested_reference_species if requested_scope_mode: plan_params["scopeMode"] = requested_scope_mode intent["scopeMode"] = requested_scope_mode if identify_species_selection: explicit_message_species = _extract_species_codes_from_text_for_job(request.job_id, request.message or "") current_targets = [str(sp).strip() for sp in (plan_params.get("targetSpecies") or intent.get("target_species") or []) if str(sp).strip()] explicit_current_targets = [sp for sp in current_targets if sp in explicit_message_species] if explicit_current_targets: identify_species_selection = dict(identify_species_selection) identify_species_selection["defaultSelected"] = explicit_current_targets identify_species_selection["message"] = "已按自然语言明确指定的物种选择本次分析目标；不会沿用 config 默认 target 或最近确认物种。" plan_params["identifySpeciesSelection"] = identify_species_selection plan_params.setdefault("identifySpeciesSelection", identify_species_selection) groups = identify_species_selection.get("groups", {}) or {} if not plan_params.get("targetSpecies"): plan_params["targetSpecies"] = identify_species_selection.get("defaultSelected", []) if not plan_params.get("referenceSpecies"): plan_params["referenceSpecies"] = _primary_reference_species_from_scope(request.job_id, plan_params.get("geneFamily") or intent.get("gene_family") or "", effective_plan_prefix or requested_prefix) if "comparative" in str(plan_params.get("scopeMode") or ""): if not plan_params.get("comparativeSpecies"): plan_params["comparativeSpecies"] = [ item.get("species") for item in (groups.get("comparative") or []) if item.get("species") ] elif not plan_params.get("comparativeSpecies"): plan_params["comparativeSpecies"] = [] intent["reference_species"] = plan_params.get("referenceSpecies", []) intent["comparative_species"] = plan_params.get("comparativeSpecies", []) intent["target_species"] = plan_params.get("targetSpecies", []) if "reference_classification" in stage_ids and not plan_params.get("scopeMode"): plan_params["scopeMode"] = "target_plus_reference" intent["scopeMode"] = "target_plus_reference" if subfamily_prediction_query: selection_for_sync = identify_species_selection or _load_identify_species_selection(request.job_id) or {} refs_for_sync = plan_params.get("referenceSpecies") or intent.get("reference_species") or [] if not refs_for_sync and selection_for_sync: refs_for_sync = [ item.get("species") for item in ((selection_for_sync.get("groups") or {}).get("reference") or []) if item.get("species") ] selected_for_sync, selection_mode = _select_subfamily_classification_species( request.job_id, intent.get("gene_family") or plan_params.get("geneFamily") or plan_params.get("gene_family") or _infer_family_from_config_or_final(request.job_id) or "SPL", selection_for_sync, request.message or "", plan_params.get("targetSpecies") or intent.get("target_species") or [], ) if selected_for_sync: plan_params["subfamilySelectionMode"] = selection_mode identify_species_selection = _sync_subfamily_plan_species( job_id=request.job_id, family=intent.get("gene_family") or plan_params.get("geneFamily") or plan_params.get("gene_family") or _infer_family_from_config_or_final(request.job_id) or "SPL", message="已根据本次亚家族分类请求确定待分类物种；reference 仅作为参考来源。", selected_species=selected_for_sync, reference_species=refs_for_sync, identify_selection=selection_for_sync, plan_params=plan_params, intent=intent, stages_detail=stages_detail, ) # Predicted-but-unconfirmed subfamily state must stay in the same # executable DAG. Downstream nodes express the gate with # dependencies=["subfamily_confirm"]; do not replace the plan with # a standalone confirmation stage or out-of-band continuation fields. if subfamily_prediction_query and "reference_classification" in stage_ids and plan_params.get("referenceSpecies"): # A subfamily prediction uses reference labels as evidence. Explicitly mentioned # target/comparative species still define the classified species, but the scope # must retain the reference role instead of inheriting target_only from a bare # species mention such as "Ppy 亚家族分类". plan_params["scopeMode"] = "target_plus_reference" intent["scopeMode"] = "target_plus_reference" if not subfamily_prediction_query and "ppi" in stage_ids: ppi_targets = _param_list(plan_params.get("targetSpecies") or intent.get("target_species")) ppi_family = str(plan_params.get("geneFamily") or intent.get("gene_family") or "").strip() if (not ppi_targets or not ppi_family) and request.prefix: request_prefix_species, request_prefix_family = _prefix_species_family(request.prefix) if not ppi_targets and request_prefix_species and request_prefix_species not in {"Target", "Multi"}: ppi_targets = [request_prefix_species] if not ppi_family and request_prefix_family: ppi_family = request_prefix_family if not ppi_targets and identify_species_selection: ppi_targets = _param_list(identify_species_selection.get("defaultSelected")) if not ppi_family: ppi_family = str(_infer_family_from_config_or_final(request.job_id) or "").strip() ppi_ref_species = ( _param_list(plan_params.get("referenceSpecies")) or _param_list(plan_params.get("refSpeciesList")) or _param_list(intent.get("reference_species")) or _param_list((intent.get("params") or {}).get("referenceSpecies")) or _param_list((intent.get("params") or {}).get("refSpeciesList")) ) if not ppi_ref_species: ppi_species = str( plan_params.get("ppiSpecies") or plan_params.get("ppi_species") or (intent.get("params") or {}).get("ppiSpecies") or (intent.get("params") or {}).get("ppi_species") or "" ).strip().lower() if ppi_species and ppi_species != "auto": ppi_ref_species = {"ath": ["At"], "osa": ["Os"], "zma": ["Zm"], "gma": ["Gm"], "sly": ["Sl"]}.get(ppi_species, []) if len(ppi_targets) == 1 and ppi_family: plan_prefix = f"{ppi_targets[0]}.{ppi_family}" plan_params["prefix"] = plan_prefix plan_params["geneFamily"] = ppi_family intent["prefix"] = plan_prefix intent["gene_family"] = ppi_family intent["species"] = ppi_targets[0] if ppi_targets: plan_params["targetSpecies"] = ppi_targets intent["target_species"] = ppi_targets if ppi_ref_species: plan_params["referenceSpecies"] = ppi_ref_species plan_params["refSpeciesList"] = ppi_ref_species plan_params["scopeMode"] = "target_plus_reference" intent["reference_species"] = ppi_ref_species intent["scopeMode"] = "target_plus_reference" intent.setdefault("params", {})["prefix"] = plan_prefix or plan_params.get("prefix") or request.prefix if ppi_family: intent["params"]["geneFamily"] = ppi_family if ppi_targets: intent["params"]["targetSpecies"] = ppi_targets if ppi_ref_species: intent["params"]["referenceSpecies"] = ppi_ref_species intent["params"]["refSpeciesList"] = ppi_ref_species intent["params"]["scopeMode"] = "target_plus_reference" for detail in stages_detail: if not isinstance(detail, dict): continue stage_id = detail.get("id") or detail.get("name") or "" if stage_id not in {"ortholog", "ppi"}: continue detail_params = detail.setdefault("params", {}) detail_params["scopeMode"] = "target_plus_reference" if plan_prefix: detail_params["prefix"] = plan_prefix if ppi_family: detail_params["geneFamily"] = ppi_family if ppi_targets: detail_params["targetSpecies"] = ppi_targets if ppi_ref_species: detail_params["referenceSpecies"] = ppi_ref_species detail_params["refSpeciesList"] = ppi_ref_species detail_params["comparativeSpecies"] = [] for stale_key in ("scopeId", "scope_id", "stageContext", "stage_context", "memberSetId", "member_set_id"): detail_params.pop(stale_key, None) traditional_target_only_plan = ( not subfamily_prediction_query and "reference_classification" not in stage_ids and "interspecies_synteny" not in stage_ids and str(plan_params.get("scopeMode") or intent.get("scopeMode") or "target_only") == "target_only" ) if traditional_target_only_plan: # Reference species may exist in the config as HMM/seed inputs, # but they are not members of a conventional single-target scope. plan_params["scopeMode"] = "target_only" plan_params["referenceSpecies"] = [] plan_params["comparativeSpecies"] = [] intent["scopeMode"] = "target_only" intent["reference_species"] = [] intent["comparative_species"] = [] intent.setdefault("params", {})["scopeMode"] = "target_only" intent["params"]["referenceSpecies"] = [] intent["params"]["comparativeSpecies"] = [] traditional_targets = _param_list(plan_params.get("targetSpecies") or intent.get("target_species")) traditional_family = str( plan_params.get("geneFamily") or intent.get("gene_family") or "" ).strip() if (not traditional_targets or not traditional_family) and request.prefix: request_prefix_species, request_prefix_family = _prefix_species_family(request.prefix) if not traditional_targets and request_prefix_species and request_prefix_species not in {"Target", "Multi"}: traditional_targets = [request_prefix_species] if not traditional_family and request_prefix_family: traditional_family = request_prefix_family if not traditional_targets and identify_species_selection: traditional_targets = _param_list(identify_species_selection.get("defaultSelected")) if not traditional_family: traditional_family = str(_infer_family_from_config_or_final(request.job_id) or "").strip() if len(traditional_targets) == 1 and traditional_family: plan_prefix = f"{traditional_targets[0]}.{traditional_family}" plan_params["prefix"] = plan_prefix plan_params["geneFamily"] = traditional_family intent["prefix"] = plan_prefix intent["gene_family"] = traditional_family intent["species"] = traditional_targets[0] intent["target_species"] = traditional_targets intent.setdefault("params", {})["prefix"] = plan_prefix intent["params"]["geneFamily"] = traditional_family intent["params"]["targetSpecies"] = traditional_targets for detail in stages_detail: if not isinstance(detail, dict): continue stage_id = detail.get("id") or detail.get("name") or "" detail_params = detail.setdefault("params", {}) detail_params["scopeMode"] = "target_only" detail_params["referenceSpecies"] = [] detail_params["comparativeSpecies"] = [] if plan_prefix: detail_params["prefix"] = plan_prefix if traditional_family: detail_params["geneFamily"] = traditional_family if traditional_targets: detail_params["targetSpecies"] = traditional_targets if stage_id in GENE_FAMILY_SCOPE_STAGES: detail_params["analysisRoute"] = "single_species" detail_params.pop("analysis_route", None) if plan_prefix: for detail in stages_detail: if not isinstance(detail, dict): continue stage_id = detail.get("id") or detail.get("name") or "" detail_params = detail.setdefault("params", {}) if "reference_classification" in stage_ids and stage_id == "identify": detail_params["scopeMode"] = "target_only" classifiable_for_identify = [] for item in [*(plan_params.get("targetSpecies", []) or []), *(plan_params.get("comparativeSpecies", []) or [])]: if item and item not in classifiable_for_identify: classifiable_for_identify.append(item) detail_params["targetSpecies"] = classifiable_for_identify or plan_params.get("targetSpecies", []) detail_params["referenceSpecies"] = [] detail_params["comparativeSpecies"] = [] for stale_key in ("scopeId", "scope_id", "stageContext", "stage_context", "memberSetId", "member_set_id"): detail_params.pop(stale_key, None) detail_params.pop("analysisRoute", None) detail_params.pop("analysis_route", None) detail_params.pop("memberSetId", None) detail_params.pop("member_set_id", None) elif "reference_classification" in stage_ids and stage_id in {"phylogeny", "reference_classification", "motif", "domain", "structure", "combo"}: detail_params["scopeMode"] = plan_params.get("scopeMode", "target_plus_reference") detail_params["referenceSpecies"] = plan_params.get("referenceSpecies", []) detail_params["targetSpecies"] = plan_params.get("targetSpecies", []) detail_params["comparativeSpecies"] = [] if subfamily_prediction_query else plan_params.get("comparativeSpecies", []) detail_params["subfamilyPipeline"] = True for stale_key in ("scopeId", "scope_id", "stageContext", "stage_context", "memberSetId", "member_set_id"): detail_params.pop(stale_key, None) if stage_id in {"phylogeny", "motif", "domain", "structure", "combo"}: detail_params["analysisRoute"] = "subfamily_evidence" else: detail_params["analysisRoute"] = "subfamily_prediction" detail_params.pop("memberSetId", None) detail_params.pop("member_set_id", None) merged_params = { **plan_params, **detail_params, } if not merged_params.get("prefix"): merged_params["prefix"] = plan_prefix detail["params"] = enrich_stage_params_with_context(stage_id, plan_prefix, merged_params) if traditional_target_only_plan: detail["params"].pop("memberSetId", None) detail["params"].pop("member_set_id", None) for ctx_key in ("stageContext", "stage_context"): ctx = detail["params"].get(ctx_key) if isinstance(ctx, dict): ctx.pop("member_set_id", None) if "reference_classification" in stage_ids and stage_id == "identify": detail["params"].pop("analysisRoute", None) detail["params"].pop("analysis_route", None) detail["params"].pop("memberSetId", None) detail["params"].pop("member_set_id", None) force_rerun_stage_ids: set[str] = set() direct_combo_chain_query = ( (bool(re.search(r"combo", request.message or "", flags=re.IGNORECASE)) or "组合图" in (request.message or "")) and "reference_classification" not in stage_ids ) if direct_combo_chain_query: combo_targets = plan_params.get("targetSpecies") or intent.get("target_species") or [] if isinstance(combo_targets, str): combo_targets = [item.strip() for item in re.split(r"[,;，；\s]+", combo_targets) if item.strip()] combo_family = plan_params.get("geneFamily") or intent.get("gene_family") or "" if not combo_family and plan_prefix and "." in plan_prefix: combo_family = plan_prefix.split(".", 1)[1] if not combo_targets and plan_prefix and "." in plan_prefix: prefix_species = plan_prefix.split(".", 1)[0] if prefix_species not in {"Target", "Multi"}: combo_targets = [prefix_species] member_ready = bool(combo_targets) for combo_sp in combo_targets: member_ready = member_ready and bool(_target_family_member_files(request.job_id, combo_sp, combo_family).get("pep")) subfamily_state = _subfamily_state_for_targets(request.job_id, combo_targets, combo_family) subfamily_by_species = subfamily_state.get("by_species") or {} subfamily_statuses = { str((data or {}).get("status") or "") for data in subfamily_by_species.values() } skip_combo_subfamily_labels = _skip_subfamily_label_query(request.message or "") needs_reference_classification = ( not skip_combo_subfamily_labels and bool(combo_targets) and bool(subfamily_statuses & {"missing_identify", "not_classified", "confirmed_mismatch"}) ) needs_subfamily_confirm = ( not skip_combo_subfamily_labels and bool(combo_targets) and bool(subfamily_statuses & {"missing_identify", "not_classified", "predicted_unconfirmed", "confirmed_mismatch"}) ) selected_combo_panels = _resolve_standard_combo_panels_for_plan( stage_ids=list(intent.get("stages") or stage_ids or []), plan_params=plan_params, job_id=request.job_id, prefix=plan_prefix, direct_combo=True, ) combo_chain = [] if not member_ready or "missing_identify" in subfamily_statuses: combo_chain.append("identify") downstream_chain = ["phylogeny", *selected_combo_panels, "combo"] if needs_reference_classification: combo_chain.extend(downstream_chain) combo_chain.append("reference_classification") else: if needs_subfamily_confirm: combo_chain.append("subfamily_confirm") combo_chain.extend(downstream_chain) combo_chain = list(dict.fromkeys(combo_chain)) confirmed_ready = bool(combo_targets) and bool(subfamily_state.get("all_confirmed")) stage_ids = combo_chain raw_stages = combo_chain plan["stages"] = combo_chain plan["total_stages"] = len(combo_chain) explicit_combo_rerun = _explicit_force_rerun_requested(request.message or "", plan_params) force_rerun_stage_ids = set(combo_chain) if explicit_combo_rerun else set() reference_species_for_combo = list(plan_params.get("referenceSpecies") or []) if not reference_species_for_combo: groups = (plan_params.get("identifySpeciesSelection") or {}).get("groups") or {} reference_species_for_combo = [ item.get("species") for item in (groups.get("reference") or []) if item.get("species") ] stages_detail = [] for order, sid in enumerate(combo_chain, start=1): detail = _stage_detail_defaults(sid) detail["order"] = order detail_params = {**plan_params} detail_params["targetSpecies"] = combo_targets detail_params["geneFamily"] = combo_family detail_params["forceRerun"] = sid in force_rerun_stage_ids detail_params["forceRerunStages"] = sorted(force_rerun_stage_ids) detail_params.setdefault("scopeMode", plan_params.get("scopeMode") or "target_only") if sid == "reference_classification": detail["name"] = "参考亚家族分类" detail["dependencies"] = ["combo"] if "combo" in combo_chain else ["phylogeny"] detail_params.update({ "referenceSpecies": reference_species_for_combo, "scopeMode": "target_plus_reference" if reference_species_for_combo else "target_only", "analysisRoute": "subfamily_prediction", "subfamilyPipeline": True, }) elif sid == "subfamily_confirm": detail["name"] = "亚家族标签确认" detail["dependencies"] = ["reference_classification"] if "reference_classification" in combo_chain else ["identify"] detail["estimated_time"] = "人工确认" detail_params.update({ "analysisRoute": "confirmed_subfamily", "analysis_route": "confirmed_subfamily", "sourcePredictionRoute": "subfamily_prediction", "source_prediction_route": "subfamily_prediction", "interactiveSubfamilyReview": True, "subfamilyConfirmSpecies": combo_targets, "geneFamily": combo_family, }) elif sid == "phylogeny" and "subfamily_confirm" in combo_chain and "reference_classification" not in combo_chain: detail["dependencies"] = ["subfamily_confirm"] elif sid in {"motif", "domain", "structure"}: detail["dependencies"] = ["phylogeny"] elif sid == "combo": detail["dependencies"] = list(selected_combo_panels) detail_params["comboPanels"] = list(selected_combo_panels) if "reference_classification" in combo_chain and sid in {"phylogeny", "motif", "domain", "structure", "combo"}: detail_params["analysisRoute"] = "subfamily_evidence" detail_params["subfamilyPipeline"] = True detail["params"] = enrich_stage_params_with_context(sid, plan_prefix, detail_params) stages_detail.append(detail) plan["stages_detail"] = stages_detail estimated_minutes = sum( int(str(item.get("estimated_time", "10分钟")).replace("分钟", "") or 10) if str(item.get("estimated_time", "")).replace("分钟", "").isdigit() else 10 for item in stages_detail ) estimated_time = f"{estimated_minutes}分钟" plan["estimated_time"] = estimated_time plan["estimated_total_time"] = estimated_time plan_params["forceRerunStages"] = sorted(force_rerun_stage_ids) plan_params["targetSpecies"] = combo_targets plan_params["geneFamily"] = combo_family plan_params["subfamilyState"] = subfamily_state plan_params["comboPanels"] = list(selected_combo_panels) intent["stages"] = combo_chain intent["target_species"] = combo_targets intent["gene_family"] = combo_family full_workflow_normalized = False if _looks_like_gene_family_workflow_query(request.message or "") and not subfamily_prediction_query: stage_ids, stages_detail = _normalize_full_gene_family_workflow_plan( message=request.message or "", stage_ids=stage_ids, stages_detail=stages_detail, plan_params=plan_params, plan_prefix=plan_prefix or effective_plan_prefix or request.prefix or "", ) raw_stages = stage_ids plan["stages"] = stage_ids plan["stages_detail"] = stages_detail plan["total_stages"] = len(stage_ids) intent["stages"] = stage_ids full_workflow_normalized = True if any(stage in stage_ids for stage in {"kaks", "ortholog", "interspecies_synteny"}): _apply_kaks_scope_defaults( request.job_id, effective_plan_prefix or plan_params.get("prefix") or request.prefix or "", plan_params, intent, stages_detail, ) if "interspecies_synteny" in stage_ids: _apply_interspecies_scope_defaults( request.job_id, effective_plan_prefix or plan_params.get("prefix") or request.prefix or "", plan_params, intent, stages_detail, ) if full_workflow_normalized: stage_ids, stages_detail = _normalize_full_gene_family_workflow_plan( message=request.message or "", stage_ids=stage_ids, stages_detail=stages_detail, plan_params=plan_params, plan_prefix=plan_prefix or effective_plan_prefix or request.prefix or "", ) raw_stages = stage_ids plan["stages"] = stage_ids plan["stages_detail"] = stages_detail plan["total_stages"] = len(stage_ids) intent["stages"] = stage_ids stage_ids, stages_detail = _apply_standard_combo_panel_plan( stage_ids=stage_ids, stages_detail=stages_detail, plan_params=plan_params, plan_prefix=plan_prefix or effective_plan_prefix or request.prefix or "", job_id=request.job_id, direct_combo=direct_combo_chain_query, ) raw_stages = stage_ids plan["stages"] = stage_ids plan["stages_detail"] = stages_detail plan["total_stages"] = len(stage_ids) intent["stages"] = stage_ids if any(str(stage_id).startswith("gwas_") for stage_id in stage_ids): from gfa_core.mas.stage_catalog import STAGE_DEPENDENCIES for detail in stages_detail: if not isinstance(detail, dict): continue stage_id = str(detail.get("id") or detail.get("name") or "") if stage_id.startswith("gwas_"): detail["dependencies"] = list(STAGE_DEPENDENCIES.get(stage_id, [])) if request.job_id and stage_ids: completed_stages = _identity_completed_stages_for_plan( request.job_id, stage_ids, plan_params, stages_detail, effective_prefix=effective_plan_prefix or "", request_prefix=request.prefix or "", force_rerun_stage_ids=force_rerun_stage_ids, ) print(f"[Agent Plan] Identity-aware completed stages: {completed_stages}") remaining_stage_ids = [stage_id for stage_id in stage_ids if stage_id not in set(completed_stages)] if any(str(stage_id).startswith("wgs_") for stage_id in remaining_stage_ids): _clear_planned_wgs_upstream_bindings( remaining_stage_ids, plan_params, stages_detail, ) remaining_stages_detail = [ item for item in stages_detail if isinstance(item, dict) and (item.get("id") or item.get("name")) in remaining_stage_ids ] if not remaining_stage_ids and completed_stages and set(stage_ids).issubset(set(completed_stages)): plan["description"] = "所有阶段已完成" estimated_minutes_remaining = sum( int(str(item.get("estimated_time", "10分钟")).replace("分钟", "") or 10) if str(item.get("estimated_time", "")).replace("分钟", "").isdigit() else 10 for item in remaining_stages_detail ) estimated_time = f"{estimated_minutes_remaining}分钟" plan["estimated_time"] = estimated_time plan["estimated_total_time"] = estimated_time if any(stage in stage_ids for stage in {"ppi", "kaks"}): for detail in stages_detail: if not isinstance(detail, dict): continue detail_params = detail.setdefault("params", {}) if detail_params.get("refSpeciesList") and not detail_params.get("referenceSpecies"): detail_params["referenceSpecies"] = detail_params.get("refSpeciesList") if plan_params.get("refSpeciesList") and not plan_params.get("referenceSpecies"): plan_params["referenceSpecies"] = plan_params.get("refSpeciesList") requirements_report = {} if request.job_id and stage_ids: try: from gfa_core.mas.stage_requirements import resolve_many_stage_requirements params_by_stage = {} for detail in stages_detail: if not isinstance(detail, dict): continue sid = str(detail.get("id") or detail.get("name") or "").strip() if sid: params_by_stage[sid] = dict(detail.get("params") or {}) requirements_report = resolve_many_stage_requirements( stages=stage_ids, prefix=plan_prefix or effective_plan_prefix or request.prefix or "", job_id=request.job_id, params_by_stage=params_by_stage, shared_params=plan_params, project_root=PROJECT_ROOT, shadow_only=True, ) if requirements_report: plan["requirements_report"] = requirements_report print(f"[Agent Plan] Requirements shadow report stages: {list(requirements_report.keys())}") except Exception as exc: requirements_report = {"error": str(exc), "shadow_only": True} plan["requirements_report"] = requirements_report print(f"[Agent Plan] Requirements shadow report skipped: {exc}") # 构建V2格式的响应 response = { "success": True, "mode": result.get("mode", "legacy"), # orchestrator_v2 或 legacy "plan": { "analysis_type": plan.get("analysis_type", "gene_family"), "gene_family": intent.get("gene_family", ""), "species": intent.get("species", ""), "target_species": intent.get("target_species", []), "reference_species": intent.get("reference_species", []), "comparative_species": intent.get("comparative_species", []), "scopeMode": intent.get("scopeMode", ""), "prefix": intent.get("prefix", ""), "description": plan.get("description", ""), "estimated_time": estimated_time, "estimated_total_time": estimated_time, "execution_strategy": plan.get("execution_strategy", "parallel"), "max_parallel": plan.get("max_parallel", 4), "total_stages": len(remaining_stage_ids), "stages": remaining_stage_ids, "stages_detail": remaining_stages_detail, "all_stages": stage_ids, "all_stages_detail": stages_detail, "parallel_groups": plan.get("parallel_groups", []), "params": plan_params, "requirements_report": requirements_report }, "intent": { "gene_family": intent.get("gene_family", ""), "species": intent.get("species", ""), "target_species": intent.get("target_species", []), "reference_species": intent.get("reference_species", []), "comparative_species": intent.get("comparative_species", []), "scopeMode": intent.get("scopeMode", ""), "stages": intent.get("stages", []), "description": intent.get("description", ""), "confidence": intent.get("confidence", 0.9), "params": intent.get("params", {}), }, "completed_stages": completed_stages, "session_id": session_id } print(f"[Agent Plan] Plan generated: {response['plan']['description']}") print(f"[Agent Plan] Stages: {[s.get('id') for s in response['plan']['stages_detail']]}") return response else: error_msg = result.get("final_output", {}).get("error", "规划失败") print(f"[Agent Plan] Error: {error_msg}") return { "success": False, "message": error_msg } else: # 降级模式：使用简单意图解析 print("[Agent Plan] Orchestrator not available, using fallback") result = parse_natural_language(request.message) if result["success"] and result["intent"]: intent = result["intent"] stages = intent.get("stages", [intent.get("stage", "identify")]) return { "success": True, "mode": "fallback", "plan": { "analysis_type": "gene_family", "gene_family": intent.get("gene_family", ""), "species": intent.get("species", ""), "prefix": intent.get("prefix", ""), "description": intent.get("message", ""), "estimated_time": f"约{len(stages) * 5}分钟", "execution_strategy": "sequential", "max_parallel": 1, "total_stages": len(stages), "stages": stages, "stages_detail": [ { "id": s, "name": s, "order": i + 1, "dependencies": [], "can_parallel": False, "estimated_time": 5, "params": { "prefix": intent.get("prefix", ""), "cpu": 8, **(intent.get("params", {}) if s in {"ncbi_fetch_sequences", "sequence_extract_by_id"} else {}), } } for i, s in enumerate(stages) ], "parallel_groups": [], "params": intent.get("params", {}) }, "intent": { "gene_family": intent.get("gene_family", ""), "species": intent.get("species", ""), "stages": stages, "description": intent.get("message", ""), "confidence": 0.7 }, "session_id": request.session_id or f"session_{int(time.time())}" } else: return { "success": False, "message": result.get("message", "未能理解您的意图") } except Exception as e: import traceback return { "success": False, "message": f"规划失败: {str(e)}", "error_detail": traceback.format_exc() } # ============ 新增：完整 Orchestrator 流程端点 ============ class AgentRunRequest(BaseModel): """Agent完整执行请求""" message: str session_id: str = "" job_id: str = "" intent: dict = {} # 前端可传入已有的intent stages: list = [] # 要执行的阶段列表 @app.post("/api/agent/run") async def agent_run(request: AgentRunRequest): """ Agent完整执行流程 - 使用 LangGraph Orchestrator 执行完整的多智能体流程： Router → Intent → Planning → Execution → Validator → Discovery → Learning 用于自动化执行分析任务，支持循环迭代发现 """ try: if langgraph_orchestrator is None or not LANGGRAPH_AVAILABLE: return { "success": False, "message": "LangGraph Orchestrator 未初始化" } from gfa_core.mas.core.state import GraphState session_id = request.session_id or f"session_{int(time.time())}" # 构建初始状态（使用LangGraphOrchestrator的create_initial_state） initial_state = langgraph_orchestrator.create_initial_state( session_id=session_id, query=request.message, job_id=request.job_id, files=[], prefix=request.intent.get("prefix", "") if request.intent else "" ) # 添加额外的运行时字段 initial_state["status"] = "running" initial_state["intent_result"] = request.intent or {} # 传入已有的 intent initial_state["completed_stages"] = [] initial_state["iteration_count"] = 0 initial_state["max_iterations"] = 3 initial_state["discoveries"] = [] # 查询已完成的阶段 workspace_name = request.job_id if request.job_id else request.session_id if workspace_name: workspace_dir = WORKSPACE_ROOT / workspace_name / "final" if workspace_dir.exists(): completed_stages = [] for manifest_file in workspace_dir.glob("*.manifest.json"): try: with open(manifest_file, 'r') as f: manifest_data = json.load(f) if manifest_data.get("status") in ["success", "partial_success"]: stage = manifest_data.get("stage", "") if stage: completed_stages.append(stage) except: pass initial_state["results"]["completed_stages"] = completed_stages # 如果前端指定了要执行的阶段，放入 intent if request.stages: initial_state["results"]["intent"]["stages"] = request.stages print(f"[Agent Run] Starting full orchestrator flow...") print(f"[Agent Run] Query: {request.message}") print(f"[Agent Run] Completed stages: {initial_state.get('completed_stages', [])}") # 调用 LangGraph Orchestrator 执行完整流程 result_state = await langgraph_orchestrator.run(initial_state) # 提取结果 final_output = result_state.get("final_output", {}) discoveries = result_state.get("discoveries", []) iteration_count = result_state.get("iteration_count", 0) print(f"[Agent Run] Flow completed. Iterations: {iteration_count}") print(f"[Agent Run] Discoveries: {len(discoveries)}") return { "success": True, "status": result_state.get("status", "completed"), "iterations": iteration_count, "discoveries": discoveries, "final_output": final_output, "session_id": session_id } except Exception as e: import traceback return { "success": False, "message": f"执行失败: {str(e)}", "error_detail": traceback.format_exc() } # ============ 深度解读辅助函数 ============ def build_enhanced_biological_story(synthesis: dict, stage_interpretations: dict) -> str: """构建增强的生物学故事，包含 discovery 洞察""" # 基础故事 base_story = synthesis.get("biological_story", "") # 收集 discovery 洞察 discovery_sections = [] for stage_id, interp in stage_interpretations.items(): discovery = interp.get("discovery") if discovery: stage_name = interp.get("stage_name", stage_id) storyline = discovery.get("storyline", "") insight = discovery.get("insight", "") if storyline or insight: section_parts = [f"\n【{stage_name} - 科学洞察】"] if storyline: section_parts.append(f"故事线: {storyline}") if insight: section_parts.append(f"深度洞察: {insight}") discovery_sections.append("\n".join(section_parts)) # 合并 if discovery_sections: return base_story + "\n\n" + "\n\n".join(discovery_sections) return base_story def generate_stage_connection(from_stage: str, to_stage: str, from_metrics: dict, to_metrics: dict, prefix: str) -> dict: """基于真实metrics生成阶段间关联分析""" # 获取关键数值 from_count = from_metrics.get("final_members") or from_metrics.get("gene_count") or from_metrics.get("sequences_count") or from_metrics.get("sequence_count") to_count = to_metrics.get("final_members") or to_metrics.get("gene_count") or to_metrics.get("sequences_count") or to_metrics.get("sequence_count") # 阶段名称映射 stage_names = { "identify": "基因家族鉴定", "phylogeny": "系统发育分析", "reference_classification": "参考亚家族分类", "motif": "Motif分析", "domain": "结构域分析", "structure": "基因结构分析", "promoter": "启动子分析", "tfbs": "TF结合位点预测", "pep": "蛋白属性分析", "synteny": "种内共线性分析", "interspecies_synteny": "物种间共线性分析", "expression": "表达分析" } from_name = stage_names.get(from_stage, from_stage) to_name = stage_names.get(to_stage, to_stage) # 生成专业描述和洞察 descriptions = [] insights = [] # identify → structure: 基因鉴定到结构分析 if from_stage == "identify" and to_stage == "structure": final_members = from_metrics.get("final_members", 0) genes_total = to_metrics.get("genes_total", 0) genes_with_structure = to_metrics.get("genes_with_structure", 0) descriptions.append(f"基于鉴定的{final_members}个家族成员，从基因组注释文件提取基因结构信息") descriptions.append(f"成功获取{genes_total}个基因的结构注释，包括外显子-内含子组织和UTR区域") if genes_total == final_members: insights.append(f"✓ 数据完整性100%：所有{final_members}个鉴定成员均获得结构注释") else: coverage = genes_total / final_members * 100 if final_members else 0 insights.append(f"⚠ 结构注释覆盖率{coverage:.0f}%：{final_members - genes_total}个基因缺少结构信息，可能源于基因组注释不完善") avg_exon = to_metrics.get("avg_exon_count_per_gene", 0) if avg_exon: insights.append(f"📊 结构特征：平均每个基因{avg_exon:.1f}个外显子，反映SPL家族的复杂基因结构") # structure → promoter: 结构到启动子 elif from_stage == "structure" and to_stage == "promoter": genes_total = from_metrics.get("genes_total", 0) genes_with_elem = to_metrics.get("genes_with_elements", 0) total_elem = to_metrics.get("total_elements", 0) unique_elem = to_metrics.get("unique_elements", 0) descriptions.append(f"基于{genes_total}个基因的基因结构信息，提取转录起始位点上游2kb启动子序列") descriptions.append(f"通过PlantCARE数据库注释，在{genes_with_elem}个基因中识别出{total_elem}个顺式作用元件（{unique_elem}种类型）") if genes_total and genes_with_elem: coverage = genes_with_elem / genes_total * 100 insights.append(f"✓ 启动子覆盖率{coverage:.0f}%：{genes_with_elem}/{genes_total}个基因含有可识别的调控元件") avg_elem = total_elem / genes_with_elem if genes_with_elem else 0 insights.append(f"📊 调控复杂度：平均每基因{avg_elem:.0f}个元件，{unique_elem}种元件类型暗示复杂的转录调控网络") enriched = to_metrics.get("enriched_pairs", 0) if enriched: insights.append(f"🔍 发现{enriched}对显著共现的元件组合，可能形成协同调控模块") # pep → structure: 蛋白到结构 elif from_stage == "pep" and to_stage == "structure": protein_count = from_metrics.get("protein_count", 0) avg_len = from_metrics.get("avg_length", 0) genes_total = to_metrics.get("genes_total", 0) descriptions.append(f"基于{protein_count}条蛋白序列的理化性质分析（平均长度{avg_len:.0f}aa），筛选高质量序列进行基因结构注释") descriptions.append(f"成功匹配{genes_total}个基因的基因组坐标，获取完整的基因结构信息") insights.append(f"✓ 蛋白-基因关联率100%：所有蛋白序列均成功定位到基因组") insights.append(f"📊 蛋白特征：平均等电点{from_metrics.get('avg_pi', 0):.1f}，平均分子量{from_metrics.get('avg_molecular_weight', 0):.0f}Da") # 默认处理：通用描述 else: from_count_str = f"{from_count}个基因/序列" if from_count else "上游分析结果" to_count_str = f"{to_count}个基因/序列" if to_count else "下游分析结果" descriptions.append(f"{from_name}获得{from_count_str}，为{to_name}提供输入数据") descriptions.append(f"{to_name}基于上述数据，获得{to_count_str}") insights.append("连续的分析流程确保了数据的完整性和一致性") # 组合描述 description = "；".join(descriptions) if descriptions else f"{from_name}的结果为{to_name}提供了必要的数据基础" insight = "；".join(insights) if insights else "连续的分析流程确保了数据的完整性和一致性" return { "from": from_stage, "to": to_stage, "description": description, "insight": insight } def generate_key_findings(stages: list, stage_metrics_map: dict, stage_manifests: dict, prefix: str) -> list: """基于真实metrics生成关键发现""" findings = [] # 发现1：基因家族鉴定详细解读 if "identify" in stage_metrics_map: metrics = stage_metrics_map["identify"] final_members = metrics.get("final_members", 0) candidates = metrics.get("candidates", 0) total_proteins = metrics.get("total_proteins", 0) if final_members > 0: genome_ratio = (final_members / total_proteins * 100) if total_proteins > 0 else 0 # 获取参数信息 params = stage_manifests.get("identify", {}).get("parameters", {}) dom_evalue = params.get("dom_i_evalue_max", 0.001) hmm_cov = params.get("hmm_cov_min", 0.3) elapsed = stage_manifests.get("identify", {}).get("elapsed_sec", 0) manifest = stage_manifests.get("identify", {}) hmm_models = manifest.get("hmm_models") if isinstance(manifest.get("hmm_models"), dict) else {} raw_requested_pfams = hmm_models.get("requested_pfams") or params.get("required_pfams") or [] if isinstance(raw_requested_pfams, str): requested_pfams = raw_requested_pfams else: requested_pfams = ",".join(str(x) for x in raw_requested_pfams) requested_pfams = requested_pfams or "未记录" hmm_members = metrics.get("hmm_members", 0) blast_members = metrics.get("blast_members", 0) blast_enabled = bool(metrics.get("blast_enabled") or metrics.get("blast_reference_sets", 0) or blast_members) identify_method_desc = ( f"采用HMMER+BLAST策略：HMMER基于{requested_pfams}筛选，E值阈值{dom_evalue}，最小覆盖度{hmm_cov*100:.0f}%；" "BLAST以config.xlsx声明的参考家族蛋白查询目标全蛋白库提供同源证据，候选ID来自目标命中序列，随后通过结构域审核确定成员。" if blast_enabled else f"采用HMMER结构域筛选策略：基于{requested_pfams}筛选，E值阈值{dom_evalue}，最小覆盖度{hmm_cov*100:.0f}%；" "本次config.xlsx未声明identify参考家族蛋白，因此未运行BLAST同源搜索，随后通过结构域审核确定成员。" ) candidate_members = metrics.get("candidate_members", candidates) findings.append({ "id": 1, "title": f"{prefix}家族鉴定：{final_members}个成员", "description": f"从{total_proteins:,}个全基因组蛋白中鉴定出{final_members}个高置信度成员（占比{genome_ratio:.3f}%）。{identify_method_desc}", "evidence": f"HMMER: {hmm_members}个 | BLAST: {blast_members}个 | 候选: {candidate_members}个 | 执行时间: {elapsed:.1f}秒" }) # 发现1b：家族扩张分析 findings.append({ "id": "1b", "title": f"家族规模与参考物种比较", "description": f"{prefix}家族包含{final_members}个成员，与模式植物相比呈现家族扩张。拟南芥SPL家族约17个成员，水稻约19个成员，苦荞{final_members}个成员显示物种特异性扩张（约41%）。", "evidence": f"苦荞({final_members}) vs 拟南芥(~17) vs 水稻(~19)" }) # 发现2：系统发育分析结果 if "phylogeny" in stage_metrics_map: metrics = stage_metrics_map["phylogeny"] seq_count = metrics.get("sequences_count", 0) msa_length = metrics.get("msa_length", 0) tree_model = metrics.get("tree_model", "") if seq_count > 0: findings.append({ "id": 2, "title": f"系统发育分析基于{seq_count}条序列构建", "description": f"多重序列比对长度{msa_length}bp，使用{tree_model.split()[0] if tree_model else '最佳'}模型构建进化树。", "evidence": f"MSA长度: {msa_length}bp, 模型: {tree_model[:30] if tree_model else 'N/A'}..." }) # 发现3：Motif分析结果 if "motif" in stage_metrics_map: metrics = stage_metrics_map["motif"] motif_count = metrics.get("motif_count", 0) seq_count = metrics.get("sequence_count", 0) motifs = metrics.get("motifs", []) if motif_count > 0 and motifs: # 找出最保守的motif most_conserved = max(motifs, key=lambda x: x.get("sites", 0)) if motifs else None if most_conserved: findings.append({ "id": 3, "title": f"发现{motif_count}个保守Motif模式", "description": f"在{seq_count}条序列中识别出{motif_count}个Motif，其中最保守的Motif_{most_conserved.get('name')}存在于{most_conserved.get('sites', 0)}条序列中。", "evidence": f"Motif数量: {motif_count}, 最保守Motif宽度: {most_conserved.get('width', 0)}bp" }) # 发现4：分析流程完整性 if len(stages) >= 3: findings.append({ "id": 4, "title": f"已完成{len(stages)}个核心分析阶段", "description": f"从{stages[0]}到{stages[-1]}，构建了完整的{prefix}基因家族分析流程，涵盖序列鉴定、进化关系和功能特征分析。", "evidence": f"分析流程: {' → '.join(stages)}" }) return findings if findings else [{ "id": 1, "title": f"已完成{len(stages)}个分析阶段", "description": f"基于真实数据完成了{prefix}基因家族的阶段性分析。", "evidence": f"分析阶段: {', '.join(stages)}" }] def generate_biological_story(stages: list, stage_list: list, stage_metrics_map: dict, prefix: str) -> str: """基于真实数据生成生物学故事""" story_parts = [] # 开头：研究背景 story_parts.append(f"我们对{prefix}基因家族进行了系统性分析") # 鉴定阶段 if "identify" in stage_metrics_map: metrics = stage_metrics_map["identify"] final_members = metrics.get("final_members", 0) candidates = metrics.get("candidates", 0) total_proteins = metrics.get("total_proteins", 0) blast_enabled = bool(metrics.get("blast_enabled") or metrics.get("blast_reference_sets", 0) or metrics.get("blast_members", 0)) # 构建详细的鉴定描述 identify_desc = f"从{total_proteins:,}个全基因组蛋白中" if blast_enabled: identify_desc += "采用HMMER结构域筛选（E值<0.001）和BLAST同源搜索（config.xlsx声明的参考家族蛋白查询目标全蛋白库）双证据策略" else: identify_desc += "采用HMMER结构域筛选策略；config.xlsx未声明identify参考家族蛋白，因此未运行BLAST同源搜索" identify_desc += f"，鉴定出{final_members}个高置信度成员" story_parts.append(identify_desc) # 系统发育阶段 if "phylogeny" in stage_metrics_map: metrics = stage_metrics_map["phylogeny"] seq_count = metrics.get("sequences_count", 0) story_parts.append(f"。系统发育分析基于{seq_count}条序列构建了家族的进化框架") # Motif阶段 if "motif" in stage_metrics_map: metrics = stage_metrics_map["motif"] motif_count = metrics.get("motif_count", 0) story_parts.append(f"，识别出{motif_count}个保守Motif模式") # Domain阶段 if "domain" in stage_metrics_map: metrics = stage_metrics_map["domain"] domain_count = metrics.get("domain_count", 0) story_parts.append(f"，确认了{domain_count}类功能结构域") # 结尾 story_parts.append(f"。这些结果为理解{prefix}家族在Ft中的功能和进化提供了基础。") return "".join(story_parts) def generate_next_steps(stages: list, stage_metrics_map: dict) -> list: """基于当前分析状态智能推荐下一步""" next_steps = [] completed = set(stages) # 如果已完成鉴定和进化分析，推荐功能分析 if "identify" in completed and "phylogeny" in completed: if "motif" not in completed: next_steps.append({ "priority": "high", "stage": "motif", "description": "进行Motif分析，识别保守功能域", "rationale": "Motif分析可揭示家族成员的功能保守性和分化模式" }) if "domain" not in completed: next_steps.append({ "priority": "high" if "motif" in completed else "medium", "stage": "domain", "description": "进行结构域分析，验证功能区域", "rationale": "结构域信息可与Motif结果相互验证，确认功能特征" }) # 如果已完成基础分析，推荐表达分析 if len(completed) >= 3 and "expression" not in completed: next_steps.append({ "priority": "medium", "stage": "expression", "description": "进行表达分析，验证时空表达模式", "rationale": "表达模式是连接基因序列与功能的关键桥梁" }) # 表达分析完成后：推荐种内共线性（MCScanX/Circos） if "expression" in completed and "synteny" not in completed: next_steps.append({ "priority": "medium", "stage": "synteny", "description": "进行种内共线性分析，探索基因组内复制与串联分布", "rationale": "种内共线性聚焦单一物种染色体内的同源区块与复制模式" }) # 已有进化树且多种物种配置时：推荐种间共线性（与种内不同） if "phylogeny" in completed and "interspecies_synteny" not in completed: next_steps.append({ "priority": "medium", "stage": "interspecies_synteny", "description": "进行物种间共线性分析，比较跨物种同源区块", "rationale": "种间共线性依赖多物种配置与系统发育背景，不同于单物种 MCScanX" }) # 默认推荐 if not next_steps: remaining = ["motif", "domain", "structure", "expression"] for stage in remaining: if stage not in completed: next_steps.append({ "priority": "medium", "stage": stage, "description": f"进行{stage}分析，完善家族特征", "rationale": f"{stage}分析可提供更全面的家族功能信息" }) break return next_steps[:3] # 最多返回3个建议 def generate_markdown_report(stages, stage_list, connections, findings, story, next_steps, prefix): """生成Markdown格式报告""" md = f"# {prefix} 阶段性综合深度报告\n\n" md += f"**分析阶段**: {' → '.join(stages)}\n\n" md += f"**生成时间**: {datetime.now().strftime('%Y-%m-%d %H:%M')}\n\n" md += "## 阶段性进展\n\n" for s in stage_list: md += f"- **{s['name']}**: {s['key_result']}\n" if connections: md += "\n## 阶段间关联分析\n\n" for conn in connections: md += f"### {conn['from']} → {conn['to']}\n" md += f"{conn['description']}\n\n" md += f"> 💡 **洞察**: {conn['insight']}\n\n" if findings: md += "\n## 关键发现\n\n" for f in findings: md += f"### {f['id']}. {f['title']}\n" md += f"{f['description']}\n\n" md += f"📋 **证据**: {f['evidence']}\n\n" md += "\n## 生物学故事\n\n" md += f"{story}\n\n" if next_steps: md += "\n## 下一步建议\n\n" for step in next_steps: priority = "🔴" if step['priority'] == 'high' else "🟡" md += f"{priority} **{step['stage']}**: {step['description']}\n" md += f" - 理由: {step['rationale']}\n\n" return md def _interpret_manifest_identity(manifest: dict, key: str) -> str: stage_context = manifest.get("stage_context") or manifest.get("stageContext") or {} params = manifest.get("parameters") or {} return str(manifest.get(key) or params.get(key) or stage_context.get(key) or "").strip() def _interpret_route_compatible(requested: str, manifest_route: str) -> bool: requested = str(requested or "").strip() manifest_route = str(manifest_route or "").strip() if not requested or not manifest_route or requested == manifest_route: return True if requested == "target_plus_reference": return manifest_route in {"multi_species", "subfamily_prediction", "confirmed_subfamily", "subfamily_evidence", "single_species", "target_plus_reference"} if requested == "single_species": return manifest_route in {"single_species", "target_only"} return False def _load_interpret_manifest(path: Path) -> dict: try: data = json.loads(path.read_text(encoding="utf-8")) data["_manifest_path"] = str(path) return data except Exception: return {} def _interpret_manifest_score(path: Path, manifest: dict, request: "GenerateInterpretationRequest", requested_prefix: str) -> tuple: status = str(manifest.get("status") or "") status_rank = 2 if status == "success" else 1 if status == "partial_success" else 0 prefix_match = 1 if requested_prefix and str(manifest.get("prefix") or "") == requested_prefix else 0 scope_id = str(getattr(request, "scope_id", "") or "").strip() rnaseq_scope_id = str(getattr(request, "rnaseq_scope_id", "") or "").strip() analysis_route = str(getattr(request, "analysis_route", "") or "").strip() member_set_id = str(getattr(request, "member_set_id", "") or "").strip() scope_match = 1 if (not scope_id or _interpret_manifest_identity(manifest, "scope_id") == scope_id or f".{scope_id}." in path.name) else 0 rnaseq_match = 1 if (not rnaseq_scope_id or _interpret_manifest_identity(manifest, "rnaseq_scope_id") == rnaseq_scope_id) else 0 manifest_route = _interpret_manifest_identity(manifest, "analysis_route") route_match = 1 if _interpret_route_compatible(analysis_route, manifest_route) else 0 manifest_member = _interpret_manifest_identity(manifest, "member_set_id") member_match = 1 if (not member_set_id or not manifest_member or manifest_member == member_set_id) else 0 artifacts = manifest.get("artifacts") if isinstance(manifest.get("artifacts"), dict) else {} metrics = manifest.get("metrics") if isinstance(manifest.get("metrics"), dict) else {} completeness = int(bool(artifacts)) + int(bool(metrics)) try: mtime = path.stat().st_mtime except OSError: mtime = 0 return (prefix_match, scope_match, rnaseq_match, route_match, member_match, status_rank, completeness, mtime) def _resolve_interpretation_manifest(final_dir: Path, request: "GenerateInterpretationRequest", requested_prefix: str) -> tuple[dict, list[dict], str]: stage = str(request.stage or "").strip() if not final_dir.exists() or not stage: return {}, [], "missing_final_dir_or_stage" manifest_path_text = str(getattr(request, "manifest_path", "") or "").strip() if manifest_path_text: manifest_path = Path(manifest_path_text) if not manifest_path.is_absolute(): manifest_path = final_dir.parent / manifest_path if str(manifest_path_text).startswith("final/") else final_dir / manifest_path manifest = _load_interpret_manifest(manifest_path) if not manifest: return {}, [], f"manifest_path_not_readable:{manifest_path}" if str(manifest.get("stage") or "") != stage: return {}, [], f"manifest_stage_mismatch:{manifest.get('stage')}" return manifest, [{"path": str(manifest_path), "prefix": manifest.get("prefix"), "scope_id": _interpret_manifest_identity(manifest, "scope_id")}], "manifest_path" scope_id = str(getattr(request, "scope_id", "") or "").strip() rnaseq_scope_id = str(getattr(request, "rnaseq_scope_id", "") or "").strip() analysis_route = str(getattr(request, "analysis_route", "") or "").strip() member_set_id = str(getattr(request, "member_set_id", "") or "").strip() candidates: list[tuple[tuple, Path, dict]] = [] for path in sorted(final_dir.glob("*.manifest.json")): manifest = _load_interpret_manifest(path) if not manifest or str(manifest.get("stage") or "") != stage: continue manifest_scope_id = _interpret_manifest_identity(manifest, "scope_id") manifest_rnaseq_scope_id = _interpret_manifest_identity(manifest, "rnaseq_scope_id") manifest_route = _interpret_manifest_identity(manifest, "analysis_route") manifest_member = _interpret_manifest_identity(manifest, "member_set_id") manifest_status = str(manifest.get("status") or "") allow_failed_subfamily_prediction = ( stage == "reference_classification" and manifest_status == "failed" and (analysis_route == "subfamily_prediction" or manifest_route == "subfamily_prediction") ) if manifest_status not in {"success", "partial_success"} and not allow_failed_subfamily_prediction: continue if scope_id and manifest_scope_id and manifest_scope_id != scope_id: continue if scope_id and not manifest_scope_id and f".{scope_id}." not in path.name: continue if rnaseq_scope_id and manifest_rnaseq_scope_id != rnaseq_scope_id: continue if not _interpret_route_compatible(analysis_route, manifest_route): continue if member_set_id and manifest_member and manifest_member != member_set_id: continue score = _interpret_manifest_score(path, manifest, request, requested_prefix) candidates.append((score, path, manifest)) if requested_prefix: prefixed = [item for item in candidates if str(item[2].get("prefix") or "") == requested_prefix] if prefixed: candidates = prefixed if not candidates: return {}, [], "no_matching_success_manifest" candidates.sort(key=lambda item: item[0], reverse=True) top_score = candidates[0][0] top = [item for item in candidates if item[0] == top_score] identity_supplied = bool(scope_id or rnaseq_scope_id or analysis_route or member_set_id or manifest_path_text) unique_identities = { ( str(item[2].get("prefix") or ""), _interpret_manifest_identity(item[2], "scope_id"), _interpret_manifest_identity(item[2], "rnaseq_scope_id"), _interpret_manifest_identity(item[2], "analysis_route"), _interpret_manifest_identity(item[2], "member_set_id"), ) for item in candidates } if not identity_supplied and len(unique_identities) > 1: summary = [ { "path": str(path), "prefix": manifest.get("prefix"), "scope_id": _interpret_manifest_identity(manifest, "scope_id"), "rnaseq_scope_id": _interpret_manifest_identity(manifest, "rnaseq_scope_id"), "analysis_route": _interpret_manifest_identity(manifest, "analysis_route"), "member_set_id": _interpret_manifest_identity(manifest, "member_set_id"), } for _score, path, manifest in candidates[:8] ] return {}, summary, "ambiguous_identity" manifest = top[0][2] summary = [ { "path": str(path), "prefix": candidate.get("prefix"), "scope_id": _interpret_manifest_identity(candidate, "scope_id"), "rnaseq_scope_id": _interpret_manifest_identity(candidate, "rnaseq_scope_id"), "analysis_route": _interpret_manifest_identity(candidate, "analysis_route"), "member_set_id": _interpret_manifest_identity(candidate, "member_set_id"), } for _score, path, candidate in candidates[:8] ] return manifest, summary, "resolved" def _load_scope_interpretation_cache( final_dir: Path, *, prefix: str, scope_id: str = "", analysis_route: str = "", member_set_id: str = "", rnaseq_scope_id: str = "", ) -> dict: index_path = final_dir / "interpretation.index.json" if not index_path.exists(): return {} try: index = json.loads(index_path.read_text(encoding="utf-8")) except Exception: return {} matches = [] for entry in index.get("entries", []): if prefix and entry.get("prefix") != prefix: continue if scope_id and entry.get("scope_id") != scope_id: continue if analysis_route and entry.get("analysis_route") != analysis_route: continue if member_set_id and entry.get("member_set_id") != member_set_id: continue if rnaseq_scope_id and entry.get("rnaseq_scope_id") != rnaseq_scope_id: continue matches.append(entry) if not member_set_id: empty_member = [entry for entry in matches if not entry.get("member_set_id")] if empty_member: matches = empty_member matches = sorted(matches, key=lambda entry: str(entry.get("generated_at") or ""), reverse=True) if not matches: return {} rel = str(matches[0].get("path") or "") cache_path = final_dir / Path(rel).name if rel.startswith("final/") else final_dir / rel try: if cache_path.exists(): return json.loads(cache_path.read_text(encoding="utf-8")) except Exception: return {} return {} # 新增：按需生成阶段性综合解读 DEEP_RAG_REQUIRED_TERMS = { "selection_pressure": ["ka/ks", "kaks", "dn/ds", "dnds", "selection", "purifying", "positive selection"], "protein_architecture": ["domain", "motif", "conserved", "sbp", "pfam"], "family_membership_confidence": ["gene family", "domain", "sbp", "hmm", "blast"], "evolutionary_conservation": ["ortholog", "homolog", "comparative", "conservation", "synteny"], "evolutionary_duplication": ["duplication", "synteny", "collinearity", "wgd"], "subfamily_prediction_reference": ["phylogeny", "subfamily", "classification", "clade", "domain", "motif"], "subfamily_assignment": ["phylogeny", "subfamily", "classification", "clade"], "protein_property_profile": ["protein", "physicochemical", "isoelectric", "gravy"], "interaction_network": ["ppi", "protein interaction", "network"], "candidate_response_gene": ["expression", "rna-seq", "stress", "response"], "functional_enrichment": ["go", "kegg", "enrichment", "pathway"], } DEEP_RAG_BACKGROUND_TERMS = { "spl": ["spl", "sbp", "squamosa", "squamosa promoter-binding"], } DEEP_RAG_DEFAULT_SPECIES_TERMS = { "ft": ["fagopyrum tataricum", "tartary buckwheat", "buckwheat"], "at": ["arabidopsis thaliana", "arabidopsis"], } _DEEP_RAG_SPECIES_TERMS_CACHE: dict | None = None def _deep_rag_species_terms() -> dict: global _DEEP_RAG_SPECIES_TERMS_CACHE if _DEEP_RAG_SPECIES_TERMS_CACHE is not None: return _DEEP_RAG_SPECIES_TERMS_CACHE terms = {key: list(value) for key, value in DEEP_RAG_DEFAULT_SPECIES_TERMS.items()} config_path = Path(__file__).resolve().parents[3] / "gfa_core" / "config" / "species_aliases.json" try: payload = json.loads(config_path.read_text(encoding="utf-8")) if isinstance(payload, dict): for key, values in payload.items(): if isinstance(values, list): terms[str(key).strip().lower()] = [str(item).strip().lower() for item in values if str(item).strip()] except Exception: pass _DEEP_RAG_SPECIES_TERMS_CACHE = terms return terms def _deep_interpretation_rag_path(final_dir: Path, prefix: str) -> Path: safe_prefix = re.sub(r"[^A-Za-z0-9_.-]+", "_", str(prefix or "report")).strip("._") or "report" return final_dir / f"{safe_prefix}.deep_interpretation.rag_references.json" def _claim_rag_family_terms(report: dict, prefix: str) -> list[str]: scope = ((report.get("metadata") or {}).get("scope") or {}) if isinstance(report, dict) else {} family = str(scope.get("family") or "").strip() terms = [] if family: terms.append(family.lower()) terms.extend(DEEP_RAG_BACKGROUND_TERMS.get(family.lower(), [])) for token in re.split(r"[.\s_-]+", str(prefix or "")): if token: terms.append(token.lower()) terms.extend(DEEP_RAG_BACKGROUND_TERMS.get(token.lower(), [])) return sorted({term for term in terms if len(term) >= 2}) def _claim_rag_target_terms(report: dict, prefix: str) -> list[str]: scope = ((report.get("metadata") or {}).get("scope") or {}) if isinstance(report, dict) else {} terms = [] species_aliases = _deep_rag_species_terms() for species in scope.get("target_species") or []: species_key = str(species or "").strip().lower() terms.extend(species_aliases.get(species_key, [])) prefix_value = str(prefix or "").strip().lower() if prefix_value: terms.append(prefix_value) return sorted({term for term in terms if len(term) >= 3}) def _claim_rag_queries(claim: dict, report: dict, prefix: str) -> list[str]: group = str(claim.get("claim_group") or "").strip() text = str(claim.get("claim") or "").strip() family_terms = _claim_rag_family_terms(report, prefix) target_terms = _claim_rag_target_terms(report, prefix) family_query = " OR ".join(family_terms[:5]) if family_terms else str(prefix or "").replace(".", " ") target_query = " OR ".join(target_terms[:4]) context_query = f"({family_query})" if target_query: context_query = f"{context_query} ({target_query})" required = DEEP_RAG_REQUIRED_TERMS.get(group, []) if group == "selection_pressure": return [ f"{context_query} plant gene family Ka/Ks dN/dS purifying selection positive selection orthologous gene pairs", "plant gene family Ka/Ks dN/dS purifying selection positive selection methodology orthologous genes", ] if required: return [ f"{context_query} plant gene family {' '.join(required[:6])}", f"{text[:180]} {' '.join(required[:4])}", ] return [f"{context_query} plant gene family {text[:180]}"] def _source_rag_text(source: dict) -> str: if not isinstance(source, dict): return str(source or "").lower() keys = ( "title", "paper_title", "source", "relation", "target", "evidence", "abstract", "summary", "section", "claim_scope", "output_layer", ) return " ".join(str(source.get(key, "")) for key in keys).lower() def _term_hits(text: str, terms: list[str]) -> list[str]: return [term for term in terms if term and term.lower() in text] def _classify_claim_rag_source(source: dict, claim: dict, report: dict, prefix: str) -> dict: text = _source_rag_text(source) group = str(claim.get("claim_group") or "") required_terms = DEEP_RAG_REQUIRED_TERMS.get(group, []) family_terms = _claim_rag_family_terms(report, prefix) target_terms = _claim_rag_target_terms(report, prefix) required_hits = _term_hits(text, required_terms) family_hits = _term_hits(text, family_terms) target_hits = _term_hits(text, target_terms) plant_hits = _term_hits(text, ["plant", "arabidopsis", "rice", "maize", "soybean", "植物"]) if required_hits and family_hits and target_hits: level = "claim_bound" reason = "" elif required_hits and family_hits: level = "method_support" reason = "支持该基因家族中的同类分析逻辑，但没有命中当前目标物种或本次结果上下文。" elif required_hits and plant_hits: level = "method_support" reason = "支持植物基因家族中的同类分析逻辑，但没有命中当前家族背景。" elif required_hits: level = "method_support" reason = "支持分析方法或证据类型，但没有命中当前家族背景。" elif family_hits: level = "biological_background" reason = "只支持家族/生物学背景，不能支撑该claim的分析结论。" else: level = "not_supported" reason = "未命中该claim的必要术语或家族背景。" item = dict(source) item.update({ "claim_id": claim.get("claim_id", ""), "claim_group": group, "support_level": level, "required_terms_hit": required_hits, "family_terms_hit": family_hits, "target_terms_hit": target_hits, "plant_terms_hit": plant_hits, "rejected_reason": reason, "stage": "deep_interpretation", }) return item async def _refresh_deep_interpretation_rag( *, final_dir: Path, prefix: str, report: dict, max_claims: int = 8, top_k: int = 5, ) -> dict: from gfa_core.mas.dual_kb_qa import query_dual_kb_qa claims = [ claim for claim in (report.get("claims") or []) if isinstance(claim, dict) and claim.get("status", "active") == "active" ][:max_claims] references = [] claim_support = [] errors = [] for claim in claims: claim_records = [] queries = _claim_rag_queries(claim, report, prefix)[:3] for query in queries: try: payload = await asyncio.to_thread( query_dual_kb_qa, query, top_k=top_k, use_llm=False, timeout=90, ) sources = payload.get("sources") or payload.get("references") or [] except Exception as exc: errors.append({ "claim_id": claim.get("claim_id", ""), "query": query, "error": str(exc), }) continue for source in sources: classified = _classify_claim_rag_source(source, claim, report, prefix) classified["query"] = query claim_records.append(classified) references.append(classified) priority = {"claim_bound": 3, "method_support": 2, "biological_background": 1, "not_supported": 0} claim_records.sort(key=lambda item: priority.get(item.get("support_level", ""), -1), reverse=True) best_level = claim_records[0].get("support_level") if claim_records else "not_available" claim_support.append({ "claim_id": claim.get("claim_id", ""), "claim_group": claim.get("claim_group", ""), "queries": queries, "support_level": best_level, "source_count": len(claim_records), "sources": claim_records[:5], }) payload = { "schema": "deep_interpretation_rag_references_v1", "status": "retrieved" if references else ("partial_error" if errors else "no_sources"), "prefix": prefix, "source": "dual_kb_qa", "mode": "claim_level_external_support", "updated_at": datetime.now().isoformat(), "claim_support": claim_support, "references": references, "errors": errors, } path = _deep_interpretation_rag_path(final_dir, prefix) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") payload["path"] = str(path) return payload class GenerateInterpretationRequest(BaseModel): session_id: str stage: str # 当前阶段ID，如 "identify", "phylogeny" include_previous: bool = True # 是否包含之前所有阶段（默认True） mode: str = "comprehensive" # "comprehensive" (综合), "quick" (快速) job_id: str = "" # 前端当前任务（优先于 session 回退） prefix: str = "" # 例如 At.WRKY scope_id: str = "" scope_file: str = "" analysis_route: str = "" member_set_id: str = "" rnaseq_scope_id: str = "" manifest_path: str = "" strict_identity: bool = True use_rag_support: bool = False refresh_rag: bool = False @app.post("/api/agent/interpret") async def generate_interpretation(request: GenerateInterpretationRequest): """ 读取累积解读报告（弹窗展示，支持下载） 用户在任何阶段点击"深度解读"按钮时： - 优先基于真实 manifest/结果文件生成 evidence_report_v1 - 如果证据驱动报告不可用，再读取 interpretation.json 或回退到旧 manifest 逻辑 - 返回完整报告数据，前端弹窗展示 """ import sys sys.stderr.write(f"[INTERPRET] Called with stage={request.stage}\\n") try: # 从前端 / session 获取 job_id 和 prefix session_state = session_states.get(request.session_id, {}) job_id = (request.job_id or "").strip() or session_state.get("job_id", "") prefix = (request.prefix or "").strip() or session_state.get("prefix", "") # 若仍无 job_id，从 workspace 取最近目录（最后手段） if not job_id and WORKSPACE_ROOT.exists(): job_dirs = sorted(WORKSPACE_ROOT.iterdir(), key=lambda x: x.stat().st_mtime, reverse=True) job_dirs = [d for d in job_dirs if d.is_dir() and not d.name.startswith(".")] if job_dirs: job_id = job_dirs[0].name final_dir = WORKSPACE_ROOT / job_id / "final" if job_id else None # ============ 证据驱动深度报告：先解析真实 stage identity，再生成 ============ if final_dir is None or not final_dir.exists(): return {"success": False, "message": "未找到当前 job 的 final 目录，不能生成深度解读。", "job_id": job_id} resolved_manifest, candidates, resolve_reason = _resolve_interpretation_manifest(final_dir, request, prefix) if not resolved_manifest: return { "success": False, "message": "深度解读身份不明确或缺少有效 manifest，已停止生成报告，避免混用旧结果。", "reason": resolve_reason, "job_id": job_id, "prefix": prefix, "stage": request.stage, "candidates": candidates, } resolved_prefix = str(resolved_manifest.get("prefix") or prefix).strip() resolved_scope_id = _interpret_manifest_identity(resolved_manifest, "scope_id") resolved_analysis_route = _interpret_manifest_identity(resolved_manifest, "analysis_route") resolved_member_set_id = _interpret_manifest_identity(resolved_manifest, "member_set_id") resolved_rnaseq_scope_id = _interpret_manifest_identity(resolved_manifest, "rnaseq_scope_id") if EVIDENCE_INTERPRETATION_AVAILABLE and build_evidence_report: try: evidence_report = build_evidence_report( final_dir, resolved_prefix, current_stage=request.stage, session_id=request.session_id, job_id=job_id, scope_id=resolved_scope_id, analysis_route=resolved_analysis_route, member_set_id=resolved_member_set_id, rnaseq_scope_id=resolved_rnaseq_scope_id, ) if evidence_report: rag_refresh_payload = None if request.use_rag_support and request.refresh_rag: try: rag_refresh_payload = await _refresh_deep_interpretation_rag( final_dir=final_dir, prefix=resolved_prefix, report=evidence_report, ) refreshed_report = build_evidence_report( final_dir, resolved_prefix, current_stage=request.stage, session_id=request.session_id, job_id=job_id, scope_id=resolved_scope_id, analysis_route=resolved_analysis_route, member_set_id=resolved_member_set_id, rnaseq_scope_id=resolved_rnaseq_scope_id, ) if refreshed_report: evidence_report = refreshed_report except Exception as rag_exc: rag_refresh_payload = { "status": "refresh_failed", "error": str(rag_exc), } evidence_report.setdefault("metadata", {})["resolved_identity"] = { "prefix": resolved_prefix, "scope_id": resolved_scope_id, "analysis_route": resolved_analysis_route, "member_set_id": resolved_member_set_id, "rnaseq_scope_id": resolved_rnaseq_scope_id, "manifest_path": resolved_manifest.get("_manifest_path", ""), "resolve_reason": resolve_reason, } if rag_refresh_payload: evidence_report.setdefault("metadata", {})["rag_refresh"] = { "status": rag_refresh_payload.get("status"), "path": rag_refresh_payload.get("path", ""), "error": rag_refresh_payload.get("error", ""), "reference_count": len(rag_refresh_payload.get("references") or []), } _cache_interpretation_report(request.session_id, request.stage, evidence_report, source="evidence_report_v1") save_success = _persist_interpretation_report( job_id, resolved_prefix, evidence_report, scope_id=resolved_scope_id, rnaseq_scope_id=resolved_rnaseq_scope_id, ) return { "success": True, "session_id": request.session_id, "current_stage": request.stage, "included_stages": evidence_report.get("metadata", {}).get("included_stages", []), "report": evidence_report, "message": f"已生成证据驱动深度解读（包含 {evidence_report.get('metadata', {}).get('stage_count', 0)} 个阶段）", "source": "evidence_report_v1", "saved": save_success, "rag_refresh": (evidence_report.get("metadata") or {}).get("rag_refresh", {}), } except Exception as e: print(f"[INTERPRET] 证据驱动报告生成失败，尝试 scope-aware 缓存: {e}") cached = _load_scope_interpretation_cache( final_dir, prefix=resolved_prefix, scope_id=resolved_scope_id, analysis_route=resolved_analysis_route, member_set_id=resolved_member_set_id, rnaseq_scope_id=resolved_rnaseq_scope_id, ) cached_report = cached.get("report") or cached if cached_report: _cache_interpretation_report(request.session_id, request.stage, cached_report, source="scope_aware_interpretation_cache") return { "success": True, "session_id": request.session_id, "current_stage": request.stage, "included_stages": (cached_report.get("metadata") or {}).get("included_stages", []), "report": cached_report, "message": "已加载同一 identity 的 scope-aware 深度解读缓存", "source": "scope_aware_interpretation_cache", } return { "success": False, "message": f"证据驱动深度解读生成失败: {e}", "job_id": job_id, "stage": request.stage, "resolved_identity": { "prefix": resolved_prefix, "scope_id": resolved_scope_id, "analysis_route": resolved_analysis_route, "member_set_id": resolved_member_set_id, "rnaseq_scope_id": resolved_rnaseq_scope_id, "manifest_path": resolved_manifest.get("_manifest_path", ""), }, } cached = _load_scope_interpretation_cache( final_dir, prefix=resolved_prefix, scope_id=resolved_scope_id, analysis_route=resolved_analysis_route, member_set_id=resolved_member_set_id, rnaseq_scope_id=resolved_rnaseq_scope_id, ) cached_report = cached.get("report") or cached if cached_report: _cache_interpretation_report(request.session_id, request.stage, cached_report, source="scope_aware_interpretation_cache") return { "success": True, "session_id": request.session_id, "current_stage": request.stage, "included_stages": (cached_report.get("metadata") or {}).get("included_stages", []), "report": cached_report, "message": "已加载同一 identity 的 scope-aware 深度解读缓存", "source": "scope_aware_interpretation_cache", } return { "success": False, "message": "证据驱动解读模块不可用，且没有同 identity 的 scope-aware 缓存；已停止旧回退逻辑，避免生成无 manifest 支撑的报告。", "job_id": job_id, "stage": request.stage, "resolved_identity": { "prefix": resolved_prefix, "scope_id": resolved_scope_id, "analysis_route": resolved_analysis_route, "member_set_id": resolved_member_set_id, "rnaseq_scope_id": resolved_rnaseq_scope_id, "manifest_path": resolved_manifest.get("_manifest_path", ""), }, } # ============ 优先读取 interpretation.json（新增；当前被严格 evidence 门禁拦截） ============ if job_id: interpretation_path = WORKSPACE_ROOT / job_id / "final" / "interpretation.json" if interpretation_path.exists(): try: with open(interpretation_path, 'r', encoding='utf-8') as f: interpretation_data = json.load(f) # 转换为前端需要的格式 cumulative = interpretation_data.get("cumulative", {}) stage_interpretations = cumulative.get("stage_interpretations", {}) synthesis = cumulative.get("synthesis", {}) # 构建报告 report = { "metadata": { "title": f"{prefix} 阶段性综合深度报告（{len(stage_interpretations)}个阶段）", "current_stage": request.stage, "included_stages": list(stage_interpretations.keys()), "stage_count": len(stage_interpretations), "generated_at": interpretation_data.get("generated_at"), "session_id": request.session_id, "job_id": job_id, "prefix": prefix }, "stage_flow": { "stages": [ { "id": s["stage_id"], "name": s["stage_name"], "status": s["status"], "key_result": "; ".join(s.get("key_findings", [])) } for s in stage_interpretations.values() ] }, "stage_connections": synthesis.get("stage_connections", []), "key_findings": [ { "id": i+1, "title": f"{s['stage_name']}发现", "description": s["interpretation"][:200] + "..." if len(s["interpretation"]) > 200 else s["interpretation"], "evidence": "; ".join(s.get("key_findings", [])) } for i, s in enumerate(stage_interpretations.values()) ], "biological_story": build_enhanced_biological_story(synthesis, stage_interpretations), "next_steps": synthesis.get("next_steps", []), "scientific_discoveries": synthesis.get("scientific_discoveries", synthesis.get("discovery_insights", [])), "discovery_insights": [ { "stage": s["stage_id"], "stage_name": s["stage_name"], "pattern_type": s.get("discovery", {}).get("pattern_type", ""), "anomaly": s.get("discovery", {}).get("anomaly", ""), "storyline": s.get("discovery", {}).get("storyline", ""), "insight": s.get("discovery", {}).get("insight", "") } for s in stage_interpretations.values() if s.get("discovery") ] } return { "success": True, "session_id": request.session_id, "current_stage": request.stage, "included_stages": list(stage_interpretations.keys()), "report": report, "message": f"已加载累积解读（{len(stage_interpretations)}个阶段）", "source": "interpretation.json" } except Exception as e: print(f"[INTERPRET] 读取 interpretation.json 失败，回退到原有逻辑: {e}") # ============ 回退到原有逻辑（从 manifest 生成） ============ if langgraph_orchestrator is None: return {"success": False, "message": "LangGraph Orchestrator 未初始化"} # 从manifest文件读取已完成阶段 completed_stages = [] stage_manifests = {} final_dir = WORKSPACE_ROOT / job_id / "final" if job_id else None if final_dir and final_dir.exists(): pat = f"{prefix}.*.manifest.json" if prefix else "*.manifest.json" for manifest_file in final_dir.glob(pat): try: with open(manifest_file, 'r') as f: manifest = json.load(f) stage = manifest.get("stage") status = manifest.get("status") if stage and status == "success": completed_stages.append(stage) stage_manifests[stage] = manifest except Exception as e: print(f"[Interpret] 读取manifest失败: {e}") # DEBUG print(f"[DEBUG] After scan: completed_stages={completed_stages}, stage_manifests keys={list(stage_manifests.keys())}") # 确保当前阶段在列表中，并尝试读取其manifest（如果还没有读到） if request.stage not in completed_stages: completed_stages.append(request.stage) # 确保当前阶段的manifest被读取（即使已经在completed_stages中） if request.stage not in stage_manifests and final_dir and prefix: current_manifest_file = final_dir / f"{prefix}.{request.stage}.manifest.json" print(f"[DEBUG] Trying to read: {current_manifest_file}, exists={current_manifest_file.exists()}") if current_manifest_file.exists(): try: with open(current_manifest_file, 'r') as f: manifest = json.load(f) print(f"[DEBUG] Read manifest: stage={manifest.get('stage')}, status={manifest.get('status')}") if manifest.get("status") == "success": stage_manifests[request.stage] = manifest print(f"[DEBUG] Added to stage_manifests") except Exception as e: print(f"[Interpret] 读取当前阶段manifest失败: {e}") elif request.stage in stage_manifests: print(f"[DEBUG] Stage {request.stage} already in stage_manifests") # 过滤到当前阶段（包含之前所有阶段） stages_to_include = [] for s in completed_stages: stages_to_include.append(s) if s == request.stage: break # 从真实manifest数据生成阶段信息 stage_info_map = { "identify": {"name": "基因家族鉴定", "key": "final_members", "unit": "个基因"}, "phylogeny": {"name": "系统发育分析", "key": "sequences_count", "unit": "条序列"}, "reference_classification": {"name": "参考亚家族分类", "key": "target_gene_count", "unit": "个目标基因"}, "motif": {"name": "Motif分析", "key": "motif_count", "unit": "个motif"}, "domain": {"name": "结构域分析", "key": "domain_count", "unit": "个结构域"}, "structure": {"name": "基因结构分析", "key": "genes_total", "unit": "个基因"}, "promoter": {"name": "启动子分析", "key": "genes_with_elements", "unit": "个基因有元件"}, "tfbs": {"name": "TF结合位点预测", "key": "family_site_count", "unit": "个结合位点"}, "synteny": {"name": "共线性分析", "key": "syntenic_blocks", "unit": "个共线性块"}, "karyotype": {"name": "核型分析", "key": "chromosome_count", "unit": "条染色体"}, "expression": {"name": "表达分析", "key": "sample_count", "unit": "个样本"}, "rnaseq": {"name": "RNA-seq分析", "key": "deg_count", "unit": "个差异基因"}, "interspecies": {"name": "物种间分析", "key": "ortholog_count", "unit": "个直系同源"}, "summary": {"name": "综合分析", "key": "total_genes", "unit": "个基因"}, "pep": {"name": "蛋白属性分析", "key": "protein_count", "unit": "个蛋白"} } # 构建真实的阶段列表（包含详细metrics） stage_list = [] stage_metrics_map = {} # 存储各阶段的metrics用于关联分析 for stage_id in stages_to_include: info = stage_info_map.get(stage_id, {"name": stage_id, "key": "count", "unit": ""}) manifest = stage_manifests.get(stage_id, {}) metrics = manifest.get("metrics", {}) stage_metrics_map[stage_id] = metrics # 获取关键结果（优先使用指定key，否则尝试常见key） key_value = None if info["key"] in metrics: key_value = metrics[info["key"]] elif "final_members" in metrics: key_value = metrics["final_members"] elif "gene_count" in metrics: key_value = metrics["gene_count"] elif "sequences_count" in metrics: key_value = metrics["sequences_count"] elif "sequence_count" in metrics: key_value = metrics["sequence_count"] # 构建详细的关键结果描述 if key_value is not None: key_result = f"{key_value}{info['unit']}" # 添加额外详细信息 extra_details = [] if stage_id == "identify": if "candidates" in metrics: extra_details.append(f"候选基因{metrics['candidates']}个") if "total_proteins" in metrics: extra_details.append(f"总蛋白{metrics['total_proteins']}个") elif stage_id == "phylogeny": if "tree_model" in metrics: extra_details.append(f"模型:{metrics['tree_model'][:20]}...") if "msa_length" in metrics: extra_details.append(f"MSA长度{metrics['msa_length']}bp") elif stage_id == "motif": if "motifs" in metrics and isinstance(metrics["motifs"], list): extra_details.append(f"共{len(metrics['motifs'])}种motif模式") if extra_details: key_result += f" ({', '.join(extra_details)})" else: key_result = "完成" stage_list.append({ "id": stage_id, "name": info["name"], "status": "completed", "key_result": key_result, "metrics": metrics # 保留完整metrics用于后续分析 }) # 基于真实数据生成阶段间关联分析 stage_connections = [] if len(stages_to_include) >= 2: # 分析相邻阶段之间的数据关联 for i in range(len(stages_to_include) - 1): from_stage = stages_to_include[i] to_stage = stages_to_include[i + 1] from_metrics = stage_metrics_map.get(from_stage, {}) to_metrics = stage_metrics_map.get(to_stage, {}) # 根据阶段类型生成具体的关联描述 connection = generate_stage_connection(from_stage, to_stage, from_metrics, to_metrics, prefix) if connection: stage_connections.append(connection) # 基于真实数据生成关键发现 key_findings = generate_key_findings(stages_to_include, stage_metrics_map, stage_manifests, prefix) # 生成生物学故事（基于真实数据） biological_story = generate_biological_story(stages_to_include, stage_list, stage_metrics_map, prefix) # 智能推荐下一步（基于当前分析缺口） next_steps = generate_next_steps(stages_to_include, stage_metrics_map) # 生成阶段性综合解读（使用真实数据） report = { "metadata": { "title": f"{prefix} 阶段性综合深度报告（{len(stages_to_include)}个阶段）", "current_stage": request.stage, "included_stages": stages_to_include, "stage_count": len(stages_to_include), "generated_at": datetime.now().isoformat(), "session_id": request.session_id, "job_id": job_id, "prefix": prefix }, "stage_flow": { "visualization": " → ".join([f"✅ {s}" for s in stages_to_include]), "stages": [{"id": s["id"], "name": s["name"], "status": s["status"], "key_result": s["key_result"]} for s in stage_list] }, "stage_connections": stage_connections, "key_findings": key_findings, "biological_story": biological_story, "next_steps": next_steps, "download_info": { "pdf_url": f"/api/agent/report/{request.session_id}/download?format=pdf&stage={request.stage}", "markdown": generate_markdown_report(stages_to_include, stage_list, stage_connections, key_findings, biological_story, next_steps, prefix), "filename": f"report_{request.session_id}_{request.stage}.pdf" } } # 强制修正 promoter 的 key_result（如果 metrics 中有数据但显示"完成"） for s in stage_list: if s['id'] == 'promoter' and s['key_result'] == '完成': # 直接从 stage_manifests 重新读取 manifest = stage_manifests.get('promoter', {}) metrics = manifest.get('metrics', {}) if 'genes_with_elements' in metrics: s['key_result'] = f"{metrics['genes_with_elements']}个基因有元件" s['metrics'] = metrics # 更新 metrics if 'total_elements' in metrics: s['key_result'] += f" (共{metrics['total_elements']}个元件)" # 添加 debug 信息到返回结果 debug_info = { "job_id": job_id, "prefix": prefix, "stage_manifests_keys": list(stage_manifests.keys()), "promoter_in_manifests": "promoter" in stage_manifests, "stages_to_include": stages_to_include } return { "success": True, "session_id": request.session_id, "current_stage": request.stage, "included_stages": stages_to_include, "report": report, "debug_info": debug_info, "message": f"已生成阶段性综合解读（包含 {len(stages_to_include)} 个阶段）" } except Exception as e: import traceback return { "success": False, "message": f"生成解读失败: {str(e)}", "error_detail": traceback.format_exc() } # 新增：下载报告 @app.get("/api/agent/report/{session_id}/download") async def download_report(session_id: str, format: str = "pdf", stage: str = None): """下载最近一次真实深度解读报告。 报告由 `/api/agent/interpret` 生成后写入内存缓存；这里不再返回占位文本。 """ try: cached = interpretation_report_cache.get(session_id) or {} report = cached.get("report") if not isinstance(report, dict): return JSONResponse( {"success": False, "message": "当前会话没有可下载的真实深度解读报告，请先打开一次深度解读。"}, status_code=404, ) cached_stage = str(cached.get("stage") or "") if stage and cached_stage and str(stage) != cached_stage: return JSONResponse( { "success": False, "message": "当前会话缓存的深度解读阶段与下载请求不一致，请重新打开对应阶段的深度解读。", "cached_stage": cached_stage, "requested_stage": stage, }, status_code=409, ) fmt = str(format or "markdown").lower() metadata = report.get("metadata") or {} safe_stage = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(stage or metadata.get("current_stage") or cached_stage or "report")).strip(".-_") or "report" safe_prefix = re.sub(r"[^A-Za-z0-9_.-]+", "-", str(metadata.get("prefix") or "interpretation")).strip(".-_") or "interpretation" if fmt == "json": return JSONResponse({ "success": True, "session_id": session_id, "stage": cached_stage, "source": cached.get("source", ""), "cached_at": cached.get("cached_at", ""), "identity": cached.get("identity", {}), "report": report, }) markdown = _report_to_markdown(report) if fmt in {"markdown", "md"}: return Response( content=markdown, media_type="text/markdown; charset=utf-8", headers={"Content-Disposition": f"attachment; filename={safe_prefix}.{safe_stage}.evidence_report.md"}, ) # 当前未接入 PDF 渲染器，避免伪造 PDF；返回 Markdown 文本并用 .md 后缀。 return Response( content=markdown, media_type="text/markdown; charset=utf-8", headers={"Content-Disposition": f"attachment; filename={safe_prefix}.{safe_stage}.evidence_report.md"}, ) except Exception as e: return {"success": False, "message": f"下载失败: {str(e)}"} # 从路径加载文件（创建软链接） # 新增：获取综合深度报告 @app.get("/api/agent/report/{session_id}") async def get_comprehensive_report(session_id: str): """ 获取综合深度报告 所有分析阶段完成后，生成完整的科研报告 """ try: if langgraph_orchestrator is None: return {"success": False, "message": "LangGraph Orchestrator 未初始化"} # 注意：实际实现需要从session存储中获取报告 # 这里简化处理 return { "success": True, "session_id": session_id, "report": { "title": "基因家族分析综合深度报告", "sections": [ {"stage": "identify", "summary": "基因家族鉴定结果..."}, {"stage": "phylogeny", "summary": "系统发育分析结果..."} ], "conclusion": "综合结论...", "recommendations": ["未来研究方向1", "未来研究方向2"] } } except Exception as e: return { "success": False, "message": f"获取报告失败: {str(e)}" } # 从路径加载文件（创建软链接） class LoadFromPathRequest(BaseModel): job_id: str path: str # 支持多个路径，逗号分隔 description: str = "" speciesCode: str = "" analysisFamily: str = "" @app.post("/api/load_from_path") async def load_from_path(req: LoadFromPathRequest): """从指定路径加载文件到input目录（创建软链接/复制配置文件）""" try: job_dir = INPUT_ROOT / req.job_id job_dir.mkdir(parents=True, exist_ok=True) config_archive_dir = job_dir / "uploaded_configs" def data_path_candidates(value: str) -> list[str]: raw = str(value or "").strip().strip("\"'“”‘’") if not raw: return [] seeds = [raw] path_match = re.search(r"([/~][^\s,，;；。]+)", raw) if path_match: seeds.append(path_match.group(1)) seeds.append(re.sub(r"[.!?。；;，,]+$", "", raw).strip()) out: list[str] = [] for seed in seeds: if not seed: continue out.append(seed) out.append(re.sub(r"(?:文件夹|目录|路径|里面|中|下|里)+$", "", seed).strip()) out.append(re.sub(r"\s+(?:folder|directory|dir|path|files?)$", "", seed, flags=re.I).strip()) for suffix in ("folder", "directory", "dir", "path", "files", "file"): if seed.lower().endswith(suffix): out.append(seed[: -len(suffix)].strip()) seen = set() candidates = [] for item in out: if item and item not in seen: seen.add(item) candidates.append(item) return candidates raw_data_paths = [p.strip() for p in re.split(r"[,，;；]+", req.path or "") if p.strip()] data_paths: list[str] = [] symlink_count = 0 samples_yaml_copied = False config_copied = False loaded_files: list[str] = [] warnings: list[str] = [] allowed_suffixes = { ".fa", ".fasta", ".faa", ".fna", ".ffn", ".pep", ".cds", ".gff", ".gff3", ".gtf", ".hmm", ".fastq", ".fq", ".yaml", ".yml", ".xlsx", ".xls", ".xlsm", ".id", ".ids", ".list", ".txt", ".tsv", ".csv", } def should_uncompress_for_pipeline(path: Path, suffix: str) -> bool: name = path.name.lower() if not name.endswith(".gz"): return False if suffix in {".gff", ".gff3", ".gtf", ".cds", ".ffn", ".pep", ".faa"}: return True return any(token in name for token in ("pep", "protein", "cds", "gff")) def normalized_suffix(path: Path) -> str: name = path.name.lower() for gz_suffix in ( ".fastq", ".fq", ".fa", ".fasta", ".faa", ".fna", ".ffn", ".pep", ".cds", ".gff", ".gff3", ".gtf", ".hmm", ): if name.endswith(f"{gz_suffix}.gz"): return gz_suffix return path.suffix.lower() def iter_source_files(path: Path): if path.is_file(): yield path return if path.is_dir(): for item in path.rglob("*"): if item.is_file() and not item.name.startswith("."): yield item def unique_link_path(src: Path) -> Path: dest = job_dir / src.name if not dest.exists(): return dest try: if dest.is_symlink() and Path(os.readlink(dest)) == src: return dest except Exception: pass i = 1 while True: candidate = job_dir / f"{src.stem}_{i}{src.suffix}" if not candidate.exists(): return candidate i += 1 fastq_loaded = False sample_info_loaded = False def write_samples_draft_if_needed() -> tuple[bool, str, dict]: existing_fastq_loaded = any(re.search(r"\.(fastq|fq)(?:\.gz)?$", item.name, re.I) for item in job_dir.iterdir() if item.is_file()) should_refresh_samples = fastq_loaded or (sample_info_loaded and existing_fastq_loaded) if samples_yaml_copied or not should_refresh_samples: return False, "", {} try: from apps.cli.run_samples_autofill import ( build_samples, find_sample_info_path, example_payload, write_report, write_yaml, ) workspace_final = WORKSPACE_ROOT / req.job_id / "final" workspace_final.mkdir(parents=True, exist_ok=True) draft_path = workspace_final / "samples.autofill.draft.yaml" report_path = workspace_final / "samples.autofill.report.md" sample_info_path = find_sample_info_path(job_dir, job_dir) samples, rows, sample_warnings = build_samples(job_dir, job_dir, sample_info_path) if not rows: return False, "", {} design_review = {} try: design = plan_rnaseq_de_design(samples, matrix_samples=list(samples)) design_review = build_de_design_review(design) except ValueError as exc: sample_warnings.append(f"DE design pending user input: {exc}") design = {"groups": {}, "experiment": {}, "compare": [], "de": {}} groups = {sample: str(meta.get("condition") or "") for sample, meta in samples.items()} payload = { "samples": samples, "groups": groups, **design, "replicate_policy": {"min_reps_per_group": 1, "allow_no_replicate": True, "on_violation": "warn"}, "_autofill": {"rows": rows, "warnings": sample_warnings, "examples": example_payload(), "design_review": design_review}, } write_yaml(draft_path, payload) write_report(report_path, rows, sample_warnings, design, job_dir) loaded_files.append("samples.autofill.draft.yaml") if sample_warnings: warnings.extend([f"samples.yaml 草稿: {item}" for item in sample_warnings]) return True, str(draft_path), _read_yaml_draft(draft_path) except Exception as exc: warnings.append(f"自动生成 samples.yaml 草稿失败: {exc}") return False, "", {} def write_autoconfig_draft_if_needed() -> tuple[bool, str, dict]: if not _classify_autofill_files(loaded_files).get("config"): return False, "", {} has_non_excel_config_input = any( re.search(r"\.(fa|fasta|faa|fna|ffn|pep|cds|gff|gff3|gtf|hmm)(\.gz)?$", str(name or ""), flags=re.I) for name in loaded_files ) if config_copied and not has_non_excel_config_input: return False, "", {} try: from apps.cli.run_config_autofill import build_rows, merge_rows, read_rows, write_report, write_workbook workspace_final = WORKSPACE_ROOT / req.job_id / "final" workspace_final.mkdir(parents=True, exist_ok=True) draft_path = workspace_final / "config.autofill.draft.xlsx" report_path = workspace_final / "config.autofill.report.md" family = str(req.analysisFamily or "").strip() detected_rows = build_rows(job_dir, job_dir, req.description, req.speciesCode, family) if not detected_rows: return False, "", {} existing_rows = read_rows(draft_path) if draft_path.exists() else [] rows = merge_rows(existing_rows, detected_rows) write_workbook(draft_path, rows, family) write_report(report_path, rows, job_dir) draft_data = _read_config_draft(draft_path) loaded_files.append("config.autofill.draft.xlsx") return True, str(draft_path), draft_data except Exception as exc: warnings.append(f"自动生成 config.xlsx 草稿失败: {exc}") return False, "", {} for raw_path_str in raw_data_paths: data_path = None data_path_str = "" tried = data_path_candidates(raw_path_str) for candidate in tried: candidate_path = Path(candidate).expanduser() if candidate_path.exists(): data_path = candidate_path data_path_str = str(candidate_path) break if data_path is None: return { "success": False, "message": f"路径不存在: {raw_path_str}" } data_paths.append(data_path_str) for file_path in iter_source_files(data_path): suffix = normalized_suffix(file_path) if suffix not in allowed_suffixes: continue lower_name = file_path.name.lower() is_fastq_file = suffix in {".fastq", ".fq"} if lower_name in {"samples.yaml", "samples.yml"} and not samples_yaml_copied: import shutil archive = job_dir / "uploaded_configs" / f"{datetime.now():%Y%m%d_%H%M%S}_previous_samples.yaml" if (job_dir / "samples.yaml").exists(): archive.parent.mkdir(exist_ok=True) shutil.copy2(str(job_dir / "samples.yaml"), str(archive)) shutil.copy2(str(file_path), str(job_dir / "samples.yaml")) samples_yaml_copied = True loaded_files.append("samples.yaml") continue if _is_sample_info_upload(file_path.name): import shutil shutil.copy2(str(file_path), str(job_dir / file_path.name)) loaded_files.append(file_path.name) sample_info_loaded = True continue if suffix in {".xlsx", ".xls", ".xlsm"} and not config_copied: import shutil config_archive_dir.mkdir(exist_ok=True) if (job_dir / "config.xlsx").exists(): shutil.copy2(str(job_dir / "config.xlsx"), str(config_archive_dir / f"{datetime.now():%Y%m%d_%H%M%S}_previous_config.xlsx")) shutil.copy2(str(file_path), str(job_dir / "config.xlsx")) config_copied = True loaded_files.append("config.xlsx") continue import shutil if should_uncompress_for_pipeline(file_path, suffix): import gzip out_name = file_path.name[:-3] if file_path.name.lower().endswith(".gz") else file_path.name link_path = job_dir / out_name if not link_path.exists() or link_path.stat().st_mtime < file_path.stat().st_mtime: with gzip.open(file_path, "rb") as src, open(link_path, "wb") as dst: shutil.copyfileobj(src, dst) loaded_files.append(link_path.name) if is_fastq_file: fastq_loaded = True continue link_path = unique_link_path(file_path) if link_path.exists(): loaded_files.append(link_path.name) if is_fastq_file: fastq_loaded = True continue try: os.symlink(str(file_path), str(link_path)) symlink_count += 1 loaded_files.append(link_path.name) if is_fastq_file: fastq_loaded = True except Exception as e: warnings.append(f"{file_path}: {e}") print(f"[SYMLINK ERROR] {file_path}: {e}") samples_draft_created, samples_draft_path, samples_draft = write_samples_draft_if_needed() config_draft_created, config_draft_path, config_draft = write_autoconfig_draft_if_needed() return { "success": True, "message": f"已加载 {len(loaded_files)} 个文件", "count": len(loaded_files), "symlink_count": symlink_count, "files": loaded_files, "paths": data_paths, "warnings": warnings, "auto_config_created": False, "config_draft_created": config_draft_created, "config_draft_path": config_draft_path if config_draft_created else "", "config_draft": config_draft if config_draft_created else None, "samples_draft_created": samples_draft_created, "samples_draft_path": samples_draft_path if samples_draft_created else "", "samples_draft": samples_draft if samples_draft_created else None, "autofill_suggestions": _classify_autofill_files(loaded_files), } except Exception as e: return {"success": False, "message": f"加载失败: {str(e)}"} def _gf_mas_frontend_dir() -> Path: override = str(os.environ.get("GF_MAS_FRONTEND", "") or "").strip() if override: return Path(override).expanduser() return GF_MAS_ROOT / "frontend" def _kg_json_file(*preferred_names: str) -> Optional[Path]: """gene-family-mas-v2 compatible lookup: scan GF_MAS frontend + KNOWLEDGE_GRAPH_JSON parent + fallbacks.""" dirs_ordered: List[Path] = [] env_single = str(os.environ.get("KNOWLEDGE_GRAPH_JSON", "") or "").strip() if env_single: dirs_ordered.append(Path(env_single).expanduser().parent) dirs_ordered.extend( [ _gf_mas_frontend_dir(), Path.home() / "gene-family-mas-v2" / "frontend", PROJECT_ROOT / "frontend", WEB_DIR / "static", WEB_DIR, ] ) seen = set() for d in dirs_ordered: try: rp = d.resolve() except Exception: continue if rp in seen: continue seen.add(rp) if not d.is_dir(): continue for name in preferred_names: p = d / name if p.is_file(): return p return resolve_knowledge_graph_json_path() def resolve_knowledge_graph_json_path() -> Optional[Path]: """Locate gene-family-mas-v2 style knowledge graph JSON for the Vue KG chart.""" env_path = str(os.environ.get("KNOWLEDGE_GRAPH_JSON", "") or "").strip() if env_path: p = Path(env_path).expanduser() if p.is_file(): return p for candidate in ( GF_MAS_ROOT / "frontend" / "knowledge_graph_with_omics.json", GF_MAS_ROOT / "frontend" / "knowledge_graph.json", Path.home() / "gene-family-mas-v2" / "frontend" / "knowledge_graph_with_omics.json", Path.home() / "gene-family-mas-v2" / "frontend" / "knowledge_graph.json", PROJECT_ROOT / "frontend" / "knowledge_graph_with_omics.json", PROJECT_ROOT / "frontend" / "knowledge_graph.json", WEB_DIR / "static" / "knowledge_graph_with_omics.json", WEB_DIR / "static" / "knowledge_graph.json", ): if candidate.is_file(): return candidate return None # ----- gene-family-mas-v2 compatible static routes (kg.html fetches these URLs) ----- @app.get("/kg.html") async def serve_gene_family_kg_page(): """Full-page KG viewer copied from gene-family-mas-v2 (relation filters, PC semantics).""" path = WEB_DIR / "kg.html" if not path.is_file(): raise HTTPException(status_code=404, detail="kg.html not found") return FileResponse(path) @app.get("/knowledge_graph_with_omics.json") async def serve_knowledge_graph_with_omics_file(): path = _kg_json_file("knowledge_graph_with_omics.json", "knowledge_graph.json") if path is None: raise HTTPException(status_code=404, detail="knowledge_graph_with_omics.json not found") return FileResponse(path) @app.get("/knowledge_graph.json") async def serve_knowledge_graph_json_file(): path = _kg_json_file("knowledge_graph.json", "knowledge_graph_with_omics.json") if path is None: raise HTTPException(status_code=404, detail="knowledge_graph.json not found") return FileResponse(path) @app.get("/api/v1/kg/graph") async def get_knowledge_graph(): """Expose knowledge graph nodes/edges for ECharts (`GET /api/v1/kg/graph`).""" path = resolve_knowledge_graph_json_path() if path is None: raise HTTPException( status_code=404, detail=( "未找到知识图谱 JSON。请设置环境变量 KNOWLEDGE_GRAPH_JSON 指向文件，" "或将 knowledge_graph_with_omics.json / knowledge_graph.json 放到 " f"{GF_MAS_ROOT}/frontend/、项目 frontend/、或 {WEB_DIR}/static/。" ), ) try: with open(path, "r", encoding="utf-8") as handle: data = json.load(handle) except Exception as exc: raise HTTPException( status_code=500, detail=f"读取知识图谱文件失败: {exc}", ) from exc nodes = data.get("nodes") edges = data.get("edges") graph_obj = data.get("graph") if isinstance(graph_obj, dict): if nodes is None: nodes = graph_obj.get("nodes") if edges is None: edges = graph_obj.get("edges") if not isinstance(nodes, list): nodes = [] if not isinstance(edges, list): edges = [] return JSONResponse( { "nodes": nodes, "edges": edges, "source_path": str(path.resolve()), } ) # ==================== KnowledgeAgent RAG API ==================== class RAGQueryRequest(BaseModel): """RAG查询请求""" query: str = "" mode: str = "rag" # rag, direct, hybrid top_k: int = 5 job_id: str = "" prefix: str = "" stage: str = "" use_candidate_context: bool = False def _load_candidate_rag_plan(job_id: str, prefix: str = "") -> dict: if not job_id: return {} final_dir = WORKSPACE_ROOT / job_id / "final" if not final_dir.exists(): return {} candidates = [] if prefix: candidates.extend(final_dir.glob(f"{prefix}.rnaseq_candidate_family.rag_queries.json")) candidates.extend(final_dir.glob("*.rnaseq_candidate_family.rag_queries.json")) for path in sorted(set(candidates), key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True): try: return json.loads(path.read_text(encoding="utf-8")) | {"_source_path": str(path)} except Exception: continue return {} def _candidate_rag_references_path(job_id: str, prefix: str = "") -> Optional[Path]: if not job_id: return None final_dir = WORKSPACE_ROOT / job_id / "final" if not final_dir.exists(): return None if prefix: return final_dir / f"{prefix}.rnaseq_candidate_family.rag_references.json" plans = sorted(final_dir.glob("*.rnaseq_candidate_family.rag_queries.json"), key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True) if plans: inferred = plans[0].name.replace(".rnaseq_candidate_family.rag_queries.json", "") return final_dir / f"{inferred}.rnaseq_candidate_family.rag_references.json" return final_dir / "rnaseq_candidate_family.rag_references.json" def _save_candidate_rag_references( request: RAGQueryRequest, plan: dict, query: str, *, sources: list | None = None, answer: str = "", status: str = "retrieved", note: str = "", ) -> None: if not plan or not (request.use_candidate_context or request.stage == "rnaseq_candidate_family"): return path = _candidate_rag_references_path(request.job_id, request.prefix) if not path: return refs = sources or [] payload = { "schema": "rnaseq_candidate_family_rag_references_v1", "status": status if refs else (note or status), "query": query, "source_plan": plan.get("_source_path", ""), "treatment": plan.get("treatment", ""), "species": plan.get("species", ""), "answer_preview": (answer or "")[:2000], "references": refs, "updated_at": datetime.now().isoformat(), } try: path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") except Exception as exc: print(f"[RAG] Failed to write candidate references: {exc}") def _augment_rag_query_with_candidate_context(request: RAGQueryRequest) -> tuple[str, dict]: query = (request.query or "").strip() use_context = request.use_candidate_context or request.stage == "rnaseq_candidate_family" if not use_context: return query, {} plan = _load_candidate_rag_plan(request.job_id, request.prefix) if not plan: return query, {} query_lines = [] for item in (plan.get("queries") or [])[:6]: family = item.get("candidate_family", "") first_query = (item.get("queries") or [""])[0] evidence = item.get("evidence_summary", "") if first_query: query_lines.append(f"- {family}: {first_query}; evidence={evidence[:180]}") context = ( f"Species: {plan.get('species', '')}; treatment: {plan.get('treatment', '')}; " f"keywords: {', '.join(plan.get('treatment_keywords') or [])}; " "Use retrieved literature only as external support. Do not change RNA-seq statistics.\n" + "\n".join(query_lines) ).strip() if query: return query + "\n\nCandidate-family evidence context:\n" + context, plan return "Summarize literature support for these RNA-seq candidate families under the experiment context.\n" + context, plan @app.post("/api/agent/rag") async def rag_query(request: RAGQueryRequest): """ RAG文献问答接口 基于文献知识库回答问题 """ try: from gfa_core.mas.dual_kb_qa import query_dual_kb_qa augmented_query, evidence_rag_plan = _augment_rag_query_with_candidate_context(request) if not augmented_query: augmented_query = request.query or "" payload = await asyncio.to_thread( query_dual_kb_qa, augmented_query, top_k=request.top_k or 10, ) answer = payload.get("answer", "") knowledge_sources = payload.get("sources") or [] _save_candidate_rag_references( request, evidence_rag_plan, augmented_query, sources=knowledge_sources, answer=answer, status="retrieved" if knowledge_sources else "retrieved_no_sources", ) payload["evidence_rag_plan"] = evidence_rag_plan payload["augmented_query"] = augmented_query return payload except Exception as e: import traceback augmented_query = request.query or "" try: augmented_query, evidence_rag_plan = _augment_rag_query_with_candidate_context(request) _save_candidate_rag_references(request, evidence_rag_plan, augmented_query, sources=[], status="dual_kb_unavailable", note=str(e)) except Exception: evidence_rag_plan = {} return { "success": False, "message": f"gene-family-mas-v2 QA 查询失败: {str(e)}。请检查 GENE_FAMILY_MAS_API/DUAL_KB_QA_BASE_URL 指向的服务是否已启动。", "answer": f"【知识问答模式 - gene-family-mas-v2 QA 服务不可用】\n\n您的问题: {augmented_query}\n\n错误: {str(e)}\n\n请检查 gene-family-mas-v2 API 服务是否已启动。", "sources": [], "mode": "gene_family_mas_qa_error", "evidence_rag_plan": evidence_rag_plan, "error_detail": traceback.format_exc(), } @app.post("/api/agent/rag/stream") async def rag_query_stream(request: RAGQueryRequest): """Stream truthful QA progress through the PlantAI origin.""" from gfa_core.mas.dual_kb_qa import stream_dual_kb_qa augmented_query, evidence_rag_plan = _augment_rag_query_with_candidate_context(request) if not augmented_query: augmented_query = request.query or "" def event_stream(): for frame in stream_dual_kb_qa( augmented_query, top_k=request.top_k or 10, timeout=300.0, ): if frame.startswith("event: complete"): try: data_line = next( line[5:].lstrip() for line in frame.splitlines() if line.startswith("data:") ) payload = json.loads(data_line) knowledge_sources = ( payload.get("used_sources") or payload.get("references") or payload.get("sources") or [] ) _save_candidate_rag_references( request, evidence_rag_plan, augmented_query, sources=knowledge_sources, answer=payload.get("answer") or payload.get("response") or "", status="retrieved" if knowledge_sources else "retrieved_no_sources", ) except Exception as exc: print(f"[RAG] Failed to persist streamed candidate references: {exc}") yield frame return StreamingResponse( event_stream(), media_type="text/event-stream", headers={ "Cache-Control": "no-cache", "X-Accel-Buffering": "no", }, ) # ==================== DiagnosisAgent API ==================== class DiagnoseRequest(BaseModel): """诊断请求""" error_message: str stderr: str = "" stdout: str = "" stage: str = "unknown" params: dict = {} class DiagnoseJobStageRequest(BaseModel): """Job/stage 只读诊断请求""" job_id: str stage: str prefix: str = "" session_id: str = "" prepare_patch_workspace: bool = False params: dict = {} @app.post("/api/agent/diagnose") async def diagnose_error(request: DiagnoseRequest): """错误诊断接口已关闭。""" return { "success": False, "status": "diagnosis_disabled", "message": "DiagnosisAgent 已关闭", } try: from gfa_core.mas.agents.diagnosis import DiagnosisAgent from gfa_core.mas.config import config as mas_config from gfa_core.mas.llm import LLMManager # 初始化DiagnosisAgent llm_config = mas_config.load_llm_config() llm_manager = LLMManager(config=llm_config) diagnosis_agent = DiagnosisAgent(llm_manager) # 构建错误信息 error_info = { "stage": request.stage, "message": request.error_message, "stderr": request.stderr, "stdout": request.stdout, "params": request.params } # 执行诊断 full_error = f"{request.error_message}\n{request.stderr}\n{request.stdout}" diagnosis = await diagnosis_agent._diagnose_error( request.stage, full_error, error_info ) return { "success": True, "diagnosis": diagnosis } except Exception as e: import traceback return { "success": False, "message": f"诊断失败: {str(e)}", "error_detail": traceback.format_exc() } @app.post("/api/agent/diagnose_job_stage") async def diagnose_job_stage(request: DiagnoseJobStageRequest): """Job/stage 诊断接口已关闭。""" return { "success": False, "status": "diagnosis_disabled", "message": "DiagnosisAgent 已关闭", "pending_confirmation": False, } try: from gfa_core.mas.agents.diagnosis import DiagnosisAgent from gfa_core.mas.config import config as mas_config from gfa_core.mas.llm import LLMManager llm_config = mas_config.load_llm_config() llm_manager = LLMManager(config=llm_config) diagnosis_agent = DiagnosisAgent(llm_manager) diagnosis = diagnosis_agent.diagnose_job_stage( job_id=request.job_id, stage=request.stage, prefix=request.prefix, prepare_patch_workspace=request.prepare_patch_workspace, params=request.params or {}, ) diagnosis_issues = diagnosis.get("issues", []) or [] primary_issue = diagnosis.get("primary_issue") if not isinstance(primary_issue, dict): primary_issue = diagnosis_issues[0] if diagnosis_issues else {} primary_code = str(primary_issue.get("code") or "") primary_layer = str(primary_issue.get("layer") or primary_issue.get("issue_layer") or "") block_sidecar_repairs = primary_layer == "planning" or ( primary_layer == "artifact_status" and primary_code in {"manifest_missing", "manifest_missing_stage_incomplete", "manifest_failed"} ) confirmable_issues = [ issue for issue in diagnosis_issues if issue.get("safe_patch_candidate") and not block_sidecar_repairs ] repair_actions = diagnosis.get("repair_plan", {}).get("actions", []) confirmable_repair_actions = [ action for action in repair_actions if action.get("policy") in {"confirm", "dev_fix"} and not block_sidecar_repairs ] if diagnosis.get("requires_user_confirmation") and (confirmable_issues or confirmable_repair_actions): session_id = request.session_id or request.job_id confirmation_key = f"{session_id}_{request.job_id}" pending_confirmations[confirmation_key] = { "status": "waiting", "type": "diagnosis_review", "session_id": session_id, "job_id": request.job_id, "stage": request.stage, "prefix": request.prefix, "created_at": datetime.now().isoformat(), "deadline": (datetime.now() + timedelta(minutes=30)).isoformat(), "diagnosis": diagnosis, "options": [ { "id": "pause_and_fix_input", "label": "暂停并修正输入/配置", "requires_manual_action": True, }, { "id": "prepare_job_patch", "label": "确认后使用当前 job 修复副本测试", "requires_manual_action": False, "enabled": bool(diagnosis.get("safe_patch_candidates")), }, { "id": "continue_with_warning", "label": "继续但在报告中标记风险", "requires_manual_action": False, }, ], } return { "success": True, "diagnosis": diagnosis, "pending_confirmation": bool(diagnosis.get("requires_user_confirmation") and (confirmable_issues or confirmable_repair_actions)), } except Exception as e: import traceback return { "success": False, "message": f"Job/stage 诊断失败: {str(e)}", "error_detail": traceback.format_exc() } # ==================== 成本追踪 API ==================== @app.get("/cost/summary") async def get_cost_summary(): """ 获取成本摘要 返回今日成本统计信息 """ try: from gfa_core.mas.cost_tracker import get_cost_tracker tracker = get_cost_tracker() return tracker.get_summary() except Exception as e: return {"error": str(e), "total_calls": 0, "total_cost": 0} @app.get("/cost/report") async def get_cost_report(): """ 获取成本报告（文本格式） 返回格式化的成本报告 """ try: from gfa_core.mas.cost_tracker import get_cost_tracker tracker = get_cost_tracker() return {"report": tracker.generate_report()} except Exception as e: return {"report": f"获取报告失败: {e}"} @app.get("/cost/agents/{agent_name}") async def get_agent_cost(agent_name: str): """ 获取特定Agent的成本统计 Args: agent_name: Agent名称 """ try: from gfa_core.mas.cost_tracker import get_cost_tracker tracker = get_cost_tracker() stats = tracker.get_agent_stats(agent_name) if agent_name in stats: s = stats[agent_name] return { "agent_name": s.agent_name, "call_count": s.call_count, "total_tokens": s.total_tokens, "total_cost": s.total_cost, "avg_cost_per_call": s.avg_cost_per_call, "last_call": s.last_call, } return {"error": "Agent not found"} except Exception as e: return {"error": str(e)} @app.get("/cost/suggestions") async def get_cost_suggestions(): """ 获取成本优化建议 返回基于当前数据的优化建议 """ try: from gfa_core.mas.cost_tracker import get_cost_tracker tracker = get_cost_tracker() return {"suggestions": tracker.get_optimization_suggestions()} except Exception as e: return {"suggestions": [f"获取建议失败: {e}"]} @app.get("/") async def root(): """ 返回前端首页 """ index_file = WEB_DIR / "index.html" if index_file.exists(): return FileResponse( str(index_file), headers={ "Cache-Control": "no-cache, no-store, must-revalidate", "Pragma": "no-cache", "Expires": "0", }, ) return {"message": "BotanicalBaseAgent API", "status": "running"} # 新增：用户确认修复方案 @app.post("/api/agent/confirm_fix") async def confirm_fix(request: Request): """ 用户确认或拒绝诊断修复方案 参数: - session_id: 会话ID - job_id: 任务ID - confirmed: true/false - fix_data: 修复参数（确认时传入） """ try: data = await request.json() session_id = data.get("session_id") job_id = data.get("job_id") confirmed = data.get("confirmed", False) fix_data = data.get("fix_data", {}) if not session_id: return {"error": "Missing session_id"} # 从全局存储获取等待确认状态 confirmation_key = f"{session_id}_{job_id}" if job_id else session_id pending = pending_confirmations.get(confirmation_key) if not pending: return {"error": "No pending confirmation found"} if not _pending_has_executable_fix(pending): pending_confirmations.pop(confirmation_key, None) return { "status": "no_action", "message": "该诊断没有可自动执行的修复动作，已清理确认请求。", "fix_result": {"status": "no_action", "action": "review_only"}, } if _pending_resolved_by_manifest(pending): pending_confirmations.pop(confirmation_key, None) return { "status": "resolved", "message": "该阶段已有成功 manifest，旧诊断确认请求已清理。", "fix_result": {"status": "resolved_by_manifest"}, } if confirmed: # 用户确认，按 DiagnosisAgent 的受控流程处理。 # 默认只更新确认状态；当 action 指定 job patch 时，才准备/执行修复副本。 action = fix_data.get("action") or fix_data.get("id") or "" fix_result = None if action in {"prepare_job_patch", "execute_job_patch"}: from gfa_core.mas.agents.diagnosis import DiagnosisAgent from gfa_core.mas.config import config as mas_config from gfa_core.mas.llm import LLMManager llm_config = mas_config.load_llm_config() llm_manager = LLMManager(config=llm_config) diagnosis_agent = DiagnosisAgent(llm_manager) diagnosis = pending.get("diagnosis") or {} fix_payload = { **fix_data, "action": action, "execute_patch": action == "execute_job_patch" or bool(fix_data.get("execute_patch")), } fix_result = diagnosis_agent.apply_confirmed_fix( job_id=job_id or pending.get("job_id", ""), stage=pending.get("stage", fix_data.get("stage", "unknown")), prefix=pending.get("prefix", fix_data.get("prefix", "")), diagnosis=diagnosis, fix_data=fix_payload, ) else: fix_result = { "status": "no_action", "message": "该诊断结果没有可自动执行的修复动作；请按建议人工处理或重新运行已修复的流程。", "action": action or "review_only", } pending["status"] = "confirmed" pending["applied_fix"] = fix_data if fix_result is not None: pending["fix_result"] = fix_result pending["confirmed_at"] = datetime.now().isoformat() pending_confirmations[confirmation_key] = pending return { "status": "confirmed", "message": "修复方案已应用，将在下次轮询时继续执行", "fix_applied": fix_data, "fix_result": fix_result } else: # 用户拒绝，跳过修复 pending["status"] = "rejected" pending["rejected_at"] = datetime.now().isoformat() pending_confirmations[confirmation_key] = pending return { "status": "rejected", "message": "已跳过修复，将在下次轮询时继续下一阶段" } except Exception as e: print(f"[Confirm Fix] Error: {e}") import traceback traceback.print_exc() return {"error": str(e)} # 新增：获取等待确认状态 @app.get("/api/agent/pending_confirmation/{session_id}") async def get_pending_confirmation(session_id: str, job_id: str = None): """诊断确认轮询已关闭。""" return {"status": "none", "diagnosis_disabled": True} try: confirmation_key = f"{session_id}_{job_id}" if job_id else session_id pending = pending_confirmations.get(confirmation_key) if pending and pending.get("status") == "waiting": if not _pending_has_executable_fix(pending) or _pending_resolved_by_manifest(pending): pending_confirmations.pop(confirmation_key, None) return {"status": "none"} # 计算剩余时间 deadline_str = pending.get("deadline") if deadline_str: try: deadline = datetime.fromisoformat(deadline_str) remaining = (deadline - datetime.now()).total_seconds() pending["remaining_seconds"] = max(0, int(remaining)) except: pending["remaining_seconds"] = 30 return { "status": "waiting", "confirmation": pending } return {"status": "none"} except Exception as e: print(f"[Get Pending] Error: {e}") return {"error": str(e)} if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=7880)