admin/PlantAI-dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: complete PPI and KaKs reference handoff

Browse Source
This commit is contained in:
potaser
2026-08-10 23:13:05 +08:00
parent 39a2c969de
commit 7b7e334213
37 changed files with 4770 additions and 561 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/web/pipeline_backend.py
+234 -117
View File
@@ -1,6 +1,7 @@
# web/pipeline_backend.py
import csv
import json
import csv
import http.client
import json
import os
import re
import shlex
@@ -48,6 +49,7 @@ from gfa_core.ppi_provenance import (
record_ppi_preflight_decision,
)
from gfa_core.string_species_registry import (
ARABIDOPSIS_TAXON_ID,
StringSnapshotError,
load_string_snapshot,
resolve_ppi_string_species,
@@ -1721,7 +1723,7 @@ def infer_string_species_from_ortholog_mapping(mapping_file: Path, fallback: str
return fallback
def infer_reference_species_from_ortholog_mapping(mapping_file: Path, fallback: str = "") -> str:
def infer_reference_species_from_ortholog_mapping(mapping_file: Path, fallback: str = "") -> str:
"""Infer the reference species label from the first data row of an ortholog mapping."""
try:
with mapping_file.open("r", encoding="utf-8", errors="ignore") as src:
@@ -1735,10 +1737,34 @@ def infer_reference_species_from_ortholog_mapping(mapping_file: Path, fallback:
break
except Exception:
pass
return fallback
def _reference_species_from_manifest_data(data: dict[str, Any]) -> str:
return fallback
def ortholog_mapping_candidates(
final_dir: Path,
prefix: str,
*,
dependency_owner: str = "",
scope_id: str = "",
) -> list[Path]:
from gfa_core.ortholog_materialize import ortholog_output_stem
output_stem = ortholog_output_stem(prefix, dependency_owner)
candidates: list[Path] = []
if scope_id:
scoped_stem = f"{output_stem}.{sanitize_token(scope_id)}"
candidates.extend([
final_dir / f"{scoped_stem}.orthologs.txt",
final_dir / f"{scoped_stem}.ortholog_pairs.txt",
])
candidates.extend([
final_dir / f"{output_stem}.orthologs.txt",
final_dir / f"{output_stem}.ortholog_pairs.txt",
])
return candidates
def _reference_species_from_manifest_data(data: dict[str, Any]) -> str:
for key in ("reference_species", "referenceSpecies"):
value = data.get(key)
if isinstance(value, list) and value:
@@ -1905,11 +1931,10 @@ NCBI_REFERENCE_ANNOTATION_TYPES = {
}
def _reference_bundle_file_plan(requested_stages: Optional[list[str]], include_recommended: bool = False) -> list[str]:
stages = _normalized_dependency_stages(requested_stages)
files = ["pep"]
if "kaks" in stages:
files.append("cds")
def _reference_bundle_file_plan(requested_stages: Optional[list[str]], include_recommended: bool = False) -> list[str]:
# PPI owns reference preparation. Download PEP and CDS together so the
# later Ka/Ks choice can reuse the exact same bundle without a second job.
files = ["pep", "cds"]
if include_recommended:
for item in ("gff3", "genome"):
if item not in files:
@@ -2005,7 +2030,7 @@ def _download_url_to_file(
raise RuntimeError("downloaded file is empty")
temp_path.replace(output_path)
return total
except (urllib.error.URLError, TimeoutError, OSError, RuntimeError) as exc:
except (http.client.IncompleteRead, urllib.error.URLError, TimeoutError, OSError, RuntimeError) as exc:
last_error = exc
if temp_path.exists():
try:
@@ -2119,6 +2144,8 @@ def _fasta_header_index(fasta_path: Path | str | None) -> dict[str, dict[str, st
if not seq_id or seq_id in records:
continue
gene_match = re.search(r"\[gene=([^\]]+)\]", header)
if not gene_match:
gene_match = re.search(r"(?:^|\s)gene:([^\s]+)", header)
ncbi_gene_match = re.search(r"\[db_xref=GeneID:(\d+)\]", header)
protein_match = re.search(r"\[protein_id=([^\]]+)\]", header)
records[seq_id] = {
@@ -2220,7 +2247,7 @@ def download_reference_annotation_bundle(
include_recommended=include_recommended,
)
assembly = {"accession": accession}
emit({"phase": "select_assembly", "message": "正在解析推荐参考物种装配", "percentage": 5, "indeterminate": True})
emit({"phase": "select_assembly", "message": "Resolving the recommended reference assembly", "percentage": 5, "indeterminate": True})
if not accession:
assembly = _select_ncbi_reference_assembly(taxon_id=taxon_id, species_name=species_name, timeout_sec=min(timeout_sec, 60))
accession = assembly.get("accession", "")
@@ -2231,24 +2258,24 @@ def download_reference_annotation_bundle(
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
work_dir = ws / "reference_bundle_download" / f"{code}_{stamp}"
zip_path = work_dir / f"{code}.ncbi_dataset.zip"
emit({"phase": "download", "message": "正在下载 NCBI 参考注释包", "percentage": 10, "indeterminate": True})
emit({"phase": "download", "message": "Downloading reference PEP and CDS from NCBI", "percentage": 10, "indeterminate": True})
download_kwargs = {"timeout_sec": timeout_sec}
if progress_callback:
download_kwargs["progress_callback"] = lambda event: emit({"message": "正在下载 NCBI 参考注释包", **event})
download_kwargs["progress_callback"] = lambda event: emit({"message": "Downloading reference PEP and CDS from NCBI", **event})
bytes_downloaded = _download_url_to_file(download_url, zip_path, **download_kwargs)
if not zipfile.is_zipfile(zip_path):
raise RuntimeError(
f"NCBI reference bundle download returned an invalid zip archive: {zip_path}"
)
species_dir = in_dir / code
emit({"phase": "extract", "message": "正在解压参考 PEP/CDS", "percentage": 80, "indeterminate": True})
emit({"phase": "extract", "message": "Extracting reference PEP and CDS", "percentage": 80, "indeterminate": True})
extracted = _extract_ncbi_reference_bundle_zip(
zip_path=zip_path,
destination_dir=species_dir,
species_code=code,
requested_file_types=file_types,
)
emit({"phase": "validate_ids", "message": "正在校验 PEP/CDS ID 对齐", "percentage": 90, "indeterminate": True})
emit({"phase": "validate_ids", "message": "Validating PEP/CDS identifier correspondence", "percentage": 90, "indeterminate": True})
id_validation = _build_reference_bundle_id_map(
species_code=code,
pep_path=extracted.get("pep"),
@@ -2288,7 +2315,7 @@ def download_reference_annotation_bundle(
manifest["manifest_path"] = str(manifest_path)
emit({
"phase": "completed",
"message": "参考物种注释包下载完成",
"message": "PPI and Ka/Ks reference species data are ready",
"percentage": 100,
"indeterminate": False,
"bytes_downloaded": bytes_downloaded,
@@ -2311,6 +2338,7 @@ def _ppi_only_preflight_response(
selected_string_species: str = "",
reference_storage_key: str = "",
reference_pep: str = "",
reference_cds: str = "",
mapping_file: Optional[Path] = None,
mapping_reference_species: str = "",
required_next_stage: str = "",
@@ -2319,22 +2347,28 @@ def _ppi_only_preflight_response(
) -> dict[str, Any]:
candidates = list(species_resolution.get("candidates") or [])
selection_mode = str(species_resolution.get("selection_mode") or "unresolved")
required_files = ["pep", "cds"] if reference_storage_key else []
bundle_files = {}
if reference_pep:
bundle_files["pep"] = reference_pep
if reference_cds:
bundle_files["cds"] = reference_cds
reference_bundle_status = {
"required_files": ["pep"],
"required_files": required_files,
"recommended_files": [],
"files": {"pep": reference_pep} if reference_pep else {},
"files": bundle_files,
"missing_required": list(missing_required or []),
"missing_recommended": [],
"mode": (
"complete"
if reference_pep
if not required_files or (reference_pep and reference_cds)
else "not_selected"
if status in {"confirmation_required", "unresolved", "conflict"}
else "unavailable"
if status == "resource_error"
else "incomplete"
),
"kaks_ready": True,
"kaks_ready": (not required_files) or bool(reference_pep and reference_cds),
}
diagnostics = {
"issue_code": issue_code,
@@ -2343,7 +2377,7 @@ def _ppi_only_preflight_response(
"required_next_stage": required_next_stage,
"recommendation": message,
"dependency_owner_stages": ["ppi"],
"required_bundle_files": ["pep"],
"required_bundle_files": required_files,
"reference_bundle_status": reference_bundle_status,
"ortholog_dependency_role": "producer" if required_next_stage == "ortholog" else "not_requested",
"reference_recommendations": candidates,
@@ -2399,24 +2433,12 @@ def _effective_ppi_owner_scope_id(
scope_data: dict[str, Any],
reference_storage_key: str,
) -> str:
scope_mode = str(scope_data.get("scope_mode") or "target_plus_reference")
comparative_species = [
str(x) for x in scope_data.get("comparative_species", []) if str(x)
]
if scope_mode == "target_only":
return str(scope_data.get("scope_id") or "") or build_scope_id(
family,
selected_targets,
[str(x) for x in scope_data.get("reference_species", []) if str(x)],
comparative_species,
scope_mode,
)
return build_scope_id(
family,
selected_targets,
[reference_storage_key],
comparative_species,
scope_mode,
[],
[],
"target_only",
)
@@ -2616,6 +2638,16 @@ def _resolve_ppi_only_snapshot_preflight(
),
None,
)
explicit_model_fallback = bool(
str(ppi_reference_selection_mode or "").strip().lower() == "model_fallback"
and chosen_taxon_id == ARABIDOPSIS_TAXON_ID
and chosen_taxon_id in snapshot.registry
)
if explicit_model_fallback:
chosen = {
"selected_string_taxon_id": chosen_taxon_id,
"selected_string_name": snapshot.nodes[chosen_taxon_id].taxon_name,
}
if not (ppi_reference_user_confirmed and chosen):
return _ppi_only_preflight_response(
status="confirmation_required",
@@ -2628,6 +2660,13 @@ def _resolve_ppi_only_snapshot_preflight(
requested_ppi_species=requested_ppi_species,
dependency_owner=dependency_owner,
)
if explicit_model_fallback:
species_resolution = {
**species_resolution,
"selection_mode": "model_fallback",
"candidate_source": "explicit_model_fallback",
"issue_code": "",
}
storage_key = str(ppi_reference_storage_key or "").strip() or f"string_taxon_{chosen_taxon_id}"
selected_name = str(chosen.get("selected_string_name") or "")
@@ -2638,16 +2677,16 @@ def _resolve_ppi_only_snapshot_preflight(
ws, final_dir, in_dir, scope_data, storage_key, family, "pep", family_only=False
)
reference_pep = str(reference_pep_path) if reference_pep_path else ""
mapping_candidates: list[Path] = []
if scope_id:
mapping_candidates.extend([
final_dir / f"{prefix}.{sanitize_token(scope_id)}.orthologs.txt",
final_dir / f"{prefix}.{sanitize_token(scope_id)}.ortholog_pairs.txt",
])
mapping_candidates.extend([
final_dir / f"{prefix}.orthologs.txt",
final_dir / f"{prefix}.ortholog_pairs.txt",
])
reference_cds_path = resolve_species_sequence_file(
ws, final_dir, in_dir, scope_data, storage_key, family, "cds", family_only=False
)
reference_cds = str(reference_cds_path) if reference_cds_path else ""
mapping_candidates = ortholog_mapping_candidates(
final_dir,
prefix,
dependency_owner="ppi",
scope_id=scope_id,
)
mapping_file = None if ignore_existing_ortholog else next(
(path for path in mapping_candidates if path.exists() and path.stat().st_size > 0),
None,
@@ -2696,18 +2735,24 @@ def _resolve_ppi_only_snapshot_preflight(
"selected_string_species": species_to_string_code(selected_name, ""),
"reference_storage_key": storage_key,
"reference_pep": reference_pep,
"reference_cds": reference_cds,
"mapping_file": mapping_file,
"mapping_reference_species": mapping_reference_species,
"dependency_owner": dependency_owner,
}
if not reference_pep:
missing_reference_files = [
file_type
for file_type, file_path in (("pep", reference_pep), ("cds", reference_cds))
if not file_path
]
if missing_reference_files:
return _ppi_only_preflight_response(
status="needs_reference_data",
issue_code="ppi_reference_data_missing",
message="The confirmed STRING reference requires a protein FASTA (PEP) before Ortholog.",
message="The confirmed STRING reference requires matching PEP and CDS before Ortholog.",
decision="needs_reference_bundle",
required_next_stage="",
missing_required=["pep"],
missing_required=missing_reference_files,
**common,
)
if mapping_file and not mapping_matches:
@@ -2775,9 +2820,7 @@ def resolve_ppi_species_resolution(
dependency_stages = _normalized_dependency_stages(requested_stages)
needs_ppi = "ppi" in dependency_stages
needs_kaks = "kaks" in dependency_stages
required_bundle_files = ["pep"]
if needs_kaks and "cds" not in required_bundle_files:
required_bundle_files.append("cds")
required_bundle_files = ["pep", "cds"]
recommended_bundle_files = ["gff3", "genome"] if needs_kaks else []
selected_targets = canonicalize_species_codes_for_job(in_dir, [str(x).strip() for x in (target_species or []) if str(x).strip()])
references = canonicalize_species_codes_for_job(in_dir, [str(x).strip() for x in (reference_species or []) if str(x).strip()])
@@ -2818,6 +2861,22 @@ def resolve_ppi_species_resolution(
)
if needs_ppi and not needs_kaks:
scope_id = _effective_ppi_owner_scope_id(
family=family,
selected_targets=selected_targets,
scope_data=scope_data,
reference_storage_key=str(ppi_reference_storage_key or ""),
)
scope_data = {
**scope_data,
"family": family,
"scope_id": scope_id,
"scope_mode": "target_only",
"target_species": list(selected_targets),
"reference_species": [],
"comparative_species": [],
"selected_species": list(selected_targets),
}
result = _resolve_ppi_only_snapshot_preflight(
job=job,
prefix=prefix,
@@ -2886,17 +2945,15 @@ def resolve_ppi_species_resolution(
and target_string_code
and selected_string_species == target_string_code
)
mapping_candidates: list[Path] = []
if not use_target_string_directly or needs_kaks:
if scope_id:
mapping_candidates.extend([
final_dir / f"{prefix}.{sanitize_token(scope_id)}.orthologs.txt",
final_dir / f"{prefix}.{sanitize_token(scope_id)}.ortholog_pairs.txt",
])
mapping_candidates.extend([
final_dir / f"{prefix}.orthologs.txt",
final_dir / f"{prefix}.ortholog_pairs.txt",
])
mapping_candidates: list[Path] = []
if not use_target_string_directly or needs_kaks:
mapping_owner = str(dependency_owner or "").strip().lower()
mapping_candidates = ortholog_mapping_candidates(
final_dir,
prefix,
dependency_owner=mapping_owner if mapping_owner in {"ppi", "kaks"} else "",
scope_id=scope_id,
)
mapping_file = None if ignore_existing_ortholog else next((p for p in mapping_candidates if p.exists() and p.stat().st_size > 0), None)
reference_data_source = ""
@@ -2917,7 +2974,7 @@ def resolve_ppi_species_resolution(
reference_data_source = str(ref_pep) if ref_pep else ""
if ref_pep:
reference_bundle_files["pep"] = str(ref_pep)
if needs_kaks:
if "cds" in required_bundle_files:
ref_cds = resolve_species_sequence_file(
ws,
final_dir,
@@ -6025,6 +6082,7 @@ def run_stage(
ppi_reference_selection_mode: str = "",
ppi_reference_user_confirmed: bool = False,
dependency_owner: str = "",
ortholog_mapping_owner: str = "",
ppi_score: int = 200,
# Genome Prepare 参数
genome_prepare_type: str = "auto",
@@ -12394,20 +12452,42 @@ def run_stage(
if not references:
return "❌ Ortholog scope 中没有 reference_species，无法确定 subject/reference 物种。", manifest
reference_species = references[0]
subject_fasta = first_existing_scope_file(ortholog_scope_data, reference_species, ["family_pep", "pep"])
if not subject_fasta:
subject_fasta = resolve_species_family_pep(ws, final_dir, in_dir, reference_species, family)
if not subject_fasta:
subject_fasta = resolve_species_sequence_file(
ws,
final_dir,
if ppi_owned_producer:
subject_fasta = resolve_species_sequence_file(
ws,
final_dir,
in_dir,
ortholog_scope_data,
reference_species,
family,
"pep",
family_only=False,
)
"pep",
family_only=False,
)
else:
subject_fasta = first_existing_scope_file(
ortholog_scope_data,
reference_species,
["family_pep", "pep"],
)
if not subject_fasta:
subject_fasta = resolve_species_family_pep(
ws,
final_dir,
in_dir,
reference_species,
family,
)
if not subject_fasta:
subject_fasta = resolve_species_sequence_file(
ws,
final_dir,
in_dir,
ortholog_scope_data,
reference_species,
family,
"pep",
family_only=False,
)
if not subject_fasta:
return (
f"❌ Ortholog stage requires reference protein FASTA for {reference_species}. "
@@ -12454,11 +12534,14 @@ def run_stage(
ortholog_pairs_has_data,
write_ortholog_run_sidecar,
)
ortholog_owner = str(dependency_owner or "").strip().lower()
if ortholog_owner not in {"", "ppi", "kaks"}:
return f"❌ Unsupported Ortholog dependency owner: {dependency_owner}", manifest
scope_fields: dict[str, Any] = {}
if ortholog_scope_data:
source_scope_mode = str(ortholog_scope_data.get("scope_mode") or "")
preserve_target_only_scope = ppi_owned_producer and source_scope_mode == "target_only"
preserve_target_only_scope = ppi_owned_producer
effective_scope_id = str(ortholog_scope_data.get("scope_id") or "")
if ppi_owned_producer:
effective_scope_id = _effective_ppi_owner_scope_id(
@@ -12468,19 +12551,19 @@ def run_stage(
reference_storage_key=reference_species,
)
scope_fields = {
"scope_file": str(ortholog_scope_path),
"scope_file": str(ortholog_scope_path) if source_scope_mode == "target_only" else "",
"scope_id": effective_scope_id,
"scope_mode": ortholog_scope_data.get("scope_mode", ""),
"scope_mode": "target_only" if ppi_owned_producer else ortholog_scope_data.get("scope_mode", ""),
"target_species": [record[0] for record in run_specs],
"reference_species": (
ortholog_scope_data.get("reference_species", [])
if preserve_target_only_scope
[]
if ppi_owned_producer
else [reference_species] if reference_species else []
),
"comparative_species": ortholog_scope_data.get("comparative_species", []),
"comparative_species": [] if ppi_owned_producer else ortholog_scope_data.get("comparative_species", []),
"selected_species": (
ortholog_scope_data.get("selected_species", [record[0] for record in run_specs])
if preserve_target_only_scope
[record[0] for record in run_specs]
if ppi_owned_producer
else [record[0] for record in run_specs] + ([reference_species] if reference_species else [])
),
"family": family,
@@ -12559,9 +12642,10 @@ def run_stage(
run_identities.append(completed_identity)
try:
pair_count, ortholog_manifest = materialize_ortholog_final_outputs(
final_dir=final_dir,
prefix=prefix,
pair_count, ortholog_manifest = materialize_ortholog_final_outputs(
final_dir=final_dir,
prefix=prefix,
dependency_owner=ortholog_owner,
run_records=run_records,
job_id=job.job_id,
workspace=ws,
@@ -12735,18 +12819,15 @@ def run_stage(
and target_string_taxon_id
and selected_execution_taxon_id == target_string_taxon_id
)
mapping_candidates = []
if not use_target_string_directly:
if scope_id:
mapping_candidates.extend([
final_dir / f"{prefix}.{sanitize_token(scope_id)}.orthologs.txt",
final_dir / f"{prefix}.{sanitize_token(scope_id)}.ortholog_pairs.txt",
])
mapping_candidates.extend([
final_dir / f"{prefix}.orthologs.txt",
final_dir / f"{prefix}.ortholog_pairs.txt",
])
mapping_file = next((p for p in mapping_candidates if p.exists() and p.stat().st_size > 0), None)
mapping_candidates = []
if not use_target_string_directly:
mapping_candidates = ortholog_mapping_candidates(
final_dir,
prefix,
dependency_owner="ppi" if ppi_owner_intent else "",
scope_id=scope_id,
)
mapping_file = next((p for p in mapping_candidates if p.exists() and p.stat().st_size > 0), None)
if mapping_file:
gene_id_file = ws / "ppi" / "analysis" / f"{prefix}.{sanitize_token(scope_id) if scope_id else 'scope'}.ppi_source_ids.txt"
mapped_source_count = build_ppi_gene_ids_from_ortholog_mapping(mapping_file, gene_id_file, ppi_selected_targets)
@@ -12931,7 +13012,7 @@ def run_stage(
if mapping_file:
effective_normalization_label = (
ppi_reference_storage_key
if ppi_owner_auto_fallback and ppi_reference_storage_key
if ppi_reference_storage_key
else ppi_reference_species or ppi_species
)
elif len(ppi_selected_targets) == 1:
@@ -12998,6 +13079,41 @@ def run_stage(
if not ppi_storage_key:
ppi_storage_key = str(ppi_species)
id_map_file = Path(job.input_dir) / ppi_storage_key / f"{ppi_storage_key}.pep_cds_idmap.tsv"
if mapping_file and not id_map_file.is_file():
reference_family = str(
(ppi_scope_data or {}).get("family")
or (prefix.split(".", 1)[1] if "." in prefix else prefix)
)
reference_pep = resolve_species_sequence_file(
ws,
final_dir,
in_dir,
ppi_scope_data or {},
ppi_storage_key,
reference_family,
"pep",
family_only=False,
)
reference_cds = resolve_species_sequence_file(
ws,
final_dir,
in_dir,
ppi_scope_data or {},
ppi_storage_key,
reference_family,
"cds",
family_only=False,
)
if reference_pep and reference_cds:
try:
_build_reference_bundle_id_map(
species_code=ppi_storage_key,
pep_path=reference_pep,
cds_path=reference_cds,
output_path=id_map_file,
)
except (OSError, UnicodeError, csv.Error) as exc:
print(f"[PPI] Could not build species-matched ID map: {exc}")
if mapping_file and not id_map_file.is_file():
return fail_ppi_execution(
f"❌ PPI projected analysis requires species-matched ID map: {id_map_file}"
@@ -13279,22 +13395,23 @@ def run_stage(
f"Target.{target_species_name}.{family}",
])
candidate_prefixes = list(dict.fromkeys(str(x) for x in candidate_prefixes if str(x)))
mapping_candidates = []
if scope_id:
for candidate_prefix in candidate_prefixes:
mapping_candidates.extend([
final_dir / f"{candidate_prefix}.{sanitize_token(scope_id)}.orthologs.txt",
final_dir / f"{candidate_prefix}.{sanitize_token(scope_id)}.ortholog_pairs.txt",
])
for candidate_prefix in candidate_prefixes:
mapping_candidates.extend([
final_dir / f"{candidate_prefix}.orthologs.txt",
final_dir / f"{candidate_prefix}.ortholog_pairs.txt",
])
mapping_candidates.append(in_dir / "ortholog_pairs.txt")
scoped_pairs_source = next((p for p in mapping_candidates if p.exists() and p.stat().st_size > 0), None)
if not scoped_pairs_source:
return "❌ KaKs scope 模式需要 Ortholog 映射文件。请先运行 Ortholog 阶段。", manifest
selected_mapping_owner = str(ortholog_mapping_owner or "").strip().lower()
if selected_mapping_owner not in {"", "ppi", "kaks"}:
return f"❌ KaKs Ortholog 映射所有者无效: {ortholog_mapping_owner}", manifest
mapping_candidates = []
for candidate_prefix in candidate_prefixes:
mapping_candidates.extend(ortholog_mapping_candidates(
final_dir,
candidate_prefix,
dependency_owner=selected_mapping_owner,
scope_id=scope_id,
))
if not selected_mapping_owner:
mapping_candidates.append(in_dir / "ortholog_pairs.txt")
scoped_pairs_source = next((p for p in mapping_candidates if p.exists() and p.stat().st_size > 0), None)
if not scoped_pairs_source:
owner_hint = f"（{selected_mapping_owner} 专属）" if selected_mapping_owner else ""
return f"❌ KaKs scope 模式需要{owner_hint} Ortholog 映射文件。", manifest
ref_cds = resolve_species_sequence_file(ws, final_dir, in_dir, kaks_scope_data, ref_species, family, "cds")
ref_pep = resolve_species_sequence_file(ws, final_dir, in_dir, kaks_scope_data, ref_species, family, "pep", family_only=False)
apps/web/vue3/index.html
+849 -308
View File
File diff suppressed because it is too large Load Diff
apps/web/vue3/main.py
+125 -27
View File
@@ -1479,18 +1479,62 @@ def _canonicalize_species_codes_for_job_strict(job_id: str, species_codes: list[
def _reference_bundle_species_available(job_id: str, species_code: str, require_cds: bool = False) -> tuple[bool, str]:
"""Return whether a downloaded job-local reference bundle can satisfy PPI/KaKs."""
"""Return whether job-local or config-resolved reference data can satisfy PPI/KaKs."""
code = str(species_code or "").strip()
if not job_id or not code:
return False, "missing job_id/species_code"
species_dir = INPUT_ROOT / job_id / code
input_dir = INPUT_ROOT / job_id
species_dir = input_dir / code
pep_path = species_dir / f"{code}.pep.fasta"
cds_path = species_dir / f"{code}.cds.fasta"
if not pep_path.exists() or pep_path.stat().st_size <= 0:
local_pep_ready = pep_path.exists() and pep_path.stat().st_size > 0
local_cds_ready = cds_path.exists() and cds_path.stat().st_size > 0
if local_pep_ready and (not require_cds or local_cds_ready):
return True, str(species_dir)
try:
scope = build_gene_family_scope(
input_dir=input_dir,
workspace=WORKSPACE_ROOT / job_id,
prefix="",
target_species=[],
reference_species=[code],
comparative_species=[],
scope_mode="target_plus_reference",
)
species_files = scope.files.get(code, {})
def has_sequence(file_type: str) -> bool:
return any(
record.get("path")
and Path(str(record["path"])).exists()
and Path(str(record["path"])).stat().st_size > 0
for record in species_files.get(file_type, [])
)
if has_sequence("pep") and (not require_cds or has_sequence("cds")):
return True, str(input_dir)
except Exception:
pass
if not local_pep_ready:
return False, f"missing PEP: {pep_path}"
if require_cds and (not cds_path.exists() or cds_path.stat().st_size <= 0):
return False, f"missing CDS: {cds_path}"
return True, str(species_dir)
return False, f"missing CDS: {cds_path}"
def _technical_reference_bundle_lookup_code(req: Any, species_code: str) -> str:
"""Resolve the local bundle identity without changing the biological species code."""
code = str(species_code or "").strip()
storage_key = str(getattr(req, "ppiReferenceStorageKey", "") or "").strip()
if not storage_key or not bool(getattr(req, "ppiReferenceUserConfirmed", False)):
return code
stage = str(getattr(req, "stage", "") or "").strip().lower()
if stage == "ppi":
return storage_key
mapping_owner = str(getattr(req, "orthologMappingOwner", "") or "").strip().lower()
if stage == "kaks" and mapping_owner == "ppi":
return storage_key
return code
def _normalize_species_alias_for_job(value: str) -> str:
@@ -1682,6 +1726,23 @@ def _looks_like_gene_family_workflow_query(message: str) -> bool:
return any(token in text for token in ["基因家族分析", "家族分析", "完整分析", "全流程", "全部阶段", "完整流程"])
def _full_gene_family_workflow_query_text(message: str, history: list[Any]) -> str:
"""Recover the user's workflow request across the MainAgent confirmation turn."""
current = str(message or "").strip()
if _looks_like_gene_family_workflow_query(current):
return current
for item in reversed(history or []):
if not isinstance(item, dict):
continue
role = str(item.get("role") or item.get("sender") or "").strip().lower()
if role not in {"user", "human"}:
continue
text = str(item.get("content") or item.get("message") or item.get("text") or "").strip()
if _looks_like_gene_family_workflow_query(text):
return text
return ""
def _looks_like_intraspecies_synteny_query(message: str) -> bool:
text = str(message or "").lower()
return (
@@ -1985,7 +2046,7 @@ def _downstream_gene_family_stages_for_context(message: str) -> list[str]:
stages = [
"phylogeny", "motif", "domain", "structure", "combo",
"promoter", "tfbs", "synteny", "interspecies_synteny", "karyotype", "pep",
"ortholog", "kaks", "ppi",
"ppi", "kaks",
]
if any(token in text for token in ["表达", "expression", "rna"]):
stages.append("expression")
@@ -1995,7 +2056,6 @@ def _downstream_gene_family_stages_for_context(message: str) -> list[str]:
if any(token in text for token in ["种间", "比较物种", "interspecies"]):
stages.append("interspecies_synteny")
if any(token in text for token in ["kaks", "ka/ks", "选择压力"]):
stages.append("ortholog")
stages.append("kaks")
if any(token in text for token in ["ppi", "互作"]):
stages.append("ppi")
@@ -2066,6 +2126,9 @@ def _resolve_contextual_gene_family_request(
confirmed = _confirmed_subfamily_species(job_id, family)
confirmed_species = [item["species"] for item in confirmed if item.get("source") == "confirmed"]
mentioned = _extract_species_codes_from_text_for_job(job_id, text)
has_explicit_scope = bool(
mentioned or structured_targets or structured_refs or structured_comps
)
prefix_species = ""
if request_prefix and "." in request_prefix:
prefix_species = request_prefix.split(".", 1)[0].strip()
@@ -2458,7 +2521,7 @@ def _stage_detail_defaults(stage_id: str) -> Dict[str, Any]:
"interspecies_synteny": ["identify"],
"karyotype": ["identify", "synteny"],
"ortholog": ["identify"],
"kaks": ["identify", "ortholog"],
"kaks": ["identify"],
"ppi": ["identify"],
"expression": ["identify"],
}
@@ -2485,18 +2548,16 @@ FULL_GENE_FAMILY_WORKFLOW_STAGES = [
"interspecies_synteny",
"karyotype",
"pep",
"ortholog",
"kaks",
"ppi",
"kaks",
]
FORMAL_GENE_FAMILY_CORE_STAGES = {"phylogeny", "motif", "domain", "structure", "combo"}
FULL_GENE_FAMILY_TARGET_ONLY_STAGES = {"promoter", "tfbs", "synteny", "karyotype", "pep"}
FULL_GENE_FAMILY_TARGET_ONLY_STAGES = {"promoter", "tfbs", "synteny", "karyotype", "pep", "ppi"}
FULL_GENE_FAMILY_REFERENCE_STAGES = FORMAL_GENE_FAMILY_CORE_STAGES | {
"interspecies_synteny",
"ortholog",
"kaks",
"ppi",
}
SCOPE_IDENTITY_PARAM_KEYS = (
"scopeFile",
@@ -2624,6 +2685,8 @@ def _normalize_full_gene_family_workflow_plan(
if sid not in ordered:
ordered.append(sid)
for sid in stage_ids:
if sid == "ortholog":
continue
if sid and sid not in ordered:
ordered.append(sid)
@@ -2652,6 +2715,8 @@ def _normalize_full_gene_family_workflow_plan(
detail["dependencies"] = ["phylogeny", "motif", "domain", "structure"]
detail_params["comboPanels"] = ["motif", "domain", "structure"]
detail_params["comboPanelSource"] = "regular_gene_family"
if sid in {"ppi", "kaks"}:
detail["dependencies"] = ["identify"]
if sid in GENE_FAMILY_SCOPE_STAGES:
detail["params"] = enrich_stage_params_with_context(sid, plan_prefix, detail_params)
else:
@@ -5178,6 +5243,7 @@ def _prepare_execute_technical_reference_scope(
stage == "ortholog"
and str(getattr(req, "dependencyOwner", "") or "").strip().lower() == "ppi"
)
ppi_consumer_intent = stage == "ppi"
if ppi_owner_intent:
missing_fields = []
if not bool(getattr(req, "ppiReferenceUserConfirmed", False)):
@@ -5198,6 +5264,8 @@ def _prepare_execute_technical_reference_scope(
)
ppi_owned_ortholog = ppi_owner_intent
if ppi_owned_ortholog:
req.analysisRoute = "single_species"
req.memberSetId = ""
technical_reference_species = [
str(getattr(req, "ppiReferenceStorageKey", "") or "").strip()
]
@@ -5256,12 +5324,22 @@ def _prepare_execute_technical_reference_scope(
scope_params["scope_id"] = original_scope_id
scope_params["refSpeciesList"] = list(identity.reference_species)
scope_params["_validatedPpiOwnerScopeId"] = original_scope_id
req.memberSetId = f"{original_scope_id}:single_species:all_members"
else:
scope_params["scopeMode"] = "target_only"
scope_params["referenceSpecies"] = []
scope_params["refSpeciesList"] = []
scope_params["comparativeSpecies"] = []
scope_params["scopeFile"] = ""
elif ppi_consumer_intent:
req.analysisRoute = "single_species"
req.memberSetId = ""
technical_reference_species = []
scope_params["scopeMode"] = "target_only"
scope_params["referenceSpecies"] = []
scope_params["refSpeciesList"] = []
scope_params["comparativeSpecies"] = []
scope_params["scopeFile"] = ""
else:
technical_reference_species = []
if stage in {"ortholog", "ppi", "kaks"}:
@@ -6898,6 +6976,7 @@ class ExecuteRequest(BaseModel):
ppiReferenceSelectionMode: str = ""
ppiReferenceUserConfirmed: bool = False
dependencyOwner: str = ""
orthologMappingOwner: str = ""
identifyBlastReferenceSpecies: List[str] = []
primaryReferenceSpecies: List[str] = []
refSpeciesList: List[str] = []
@@ -7802,6 +7881,7 @@ def run_stage_sync(task_id: str, stage: str, job_id: str, params: dict):
ppi_reference_selection_mode=params.get("ppiReferenceSelectionMode") or params.get("ppi_reference_selection_mode") or "",
ppi_reference_user_confirmed=_bool_param("ppiReferenceUserConfirmed", "ppi_reference_user_confirmed", default=False),
dependency_owner=params.get("dependencyOwner") or params.get("dependency_owner") or "",
ortholog_mapping_owner=params.get("orthologMappingOwner") or params.get("ortholog_mapping_owner") or "",
ppi_score=params.get("ppiScore", 200),
# KaKs参数
kaks_neutral_rate=params.get("kaksNeutralRate", 7.08e-9),
@@ -11434,7 +11514,12 @@ async def execute_stage(req: ExecuteRequest):
for raw_value in raw_values:
raw_code = str(raw_value or "").strip()
if raw_code and raw_code.lower() in technical_ref_lookup:
available, reason = _reference_bundle_species_available(job_id, raw_code, require_cds=require_cds)
bundle_lookup_code = _technical_reference_bundle_lookup_code(req, raw_code)
available, reason = _reference_bundle_species_available(
job_id,
bundle_lookup_code,
require_cds=require_cds,
)
if not available:
raise ValueError(
f"PPI/KaKs 技术参考物种 {raw_code} 已选择但参考包不完整：{reason}；"
@@ -12123,6 +12208,7 @@ async def execute_stage(req: ExecuteRequest):
"ppiScore": getattr(req, 'ppiScore', 200),
# KaKs参数
"kaksNeutralRate": getattr(req, 'kaksNeutralRate', 7.08e-9),
"orthologMappingOwner": getattr(req, 'orthologMappingOwner', ''),
# Genome Prepare参数
"genomePrepareType": getattr(req, 'genomePrepareType', 'auto'),
# Interspecies species selection
@@ -15246,7 +15332,7 @@ def _apply_reference_bundle_stale_watchdog(task: Dict[str, Any]) -> None:
progress = task.get("progress") if isinstance(task.get("progress"), dict) else {}
progress = {
"phase": progress.get("phase") or status or "running",
"message": progress.get("message") or task.get("message") or "参考物种注释包仍在后台下载",
"message": progress.get("message") or task.get("message") or "PPI and Ka/Ks reference species data are still downloading",
"percentage": progress.get("percentage"),
"indeterminate": progress.get("indeterminate", True),
"pending_seconds": task["pending_seconds"],
@@ -15264,13 +15350,13 @@ def _apply_reference_bundle_stale_watchdog(task: Dict[str, Any]) -> None:
def run_reference_bundle_download_task(task_id: str, payload: Dict[str, Any]) -> None:
tasks[task_id].update({
"status": "running",
"message": "正在后台下载参考物种注释包",
"message": "Downloading PPI and Ka/Ks reference species PEP and CDS",
"start_time": datetime.now().isoformat(),
"timeout_sec": int(payload.get("timeout_sec") or 300),
"payload": dict(payload),
"progress": {
"phase": "queued",
"message": "参考物种注释包下载任务已启动",
"message": "PPI and Ka/Ks reference species data download started",
"percentage": 0,
"indeterminate": True,
},
@@ -15301,7 +15387,7 @@ def run_reference_bundle_download_task(task_id: str, payload: Dict[str, Any]) ->
progress.setdefault("indeterminate", percentage is None)
tasks[task_id].update({
"status": "running",
"message": progress.get("message") or "正在后台下载参考物种注释包",
"message": progress.get("message") or "Downloading PPI and Ka/Ks reference species PEP and CDS",
"progress": progress,
})
_persist_reference_bundle_task(tasks[task_id])
@@ -15320,11 +15406,11 @@ def run_reference_bundle_download_task(task_id: str, payload: Dict[str, Any]) ->
success = result.get("status") == "success"
tasks[task_id].update({
"status": "success" if success else "failed",
"message": "参考物种注释包下载完成" if success else "参考物种注释包下载未满足必需文件",
"message": "PPI and Ka/Ks reference species data are ready" if success else "PPI and Ka/Ks reference species data are incomplete",
"end_time": datetime.now().isoformat(),
"progress": {
"phase": "completed" if success else "failed",
"message": "参考物种注释包下载完成" if success else "参考物种注释包下载未满足必需文件",
"message": "PPI and Ka/Ks reference species data are ready" if success else "PPI and Ka/Ks reference species data are incomplete",
"percentage": 100 if success else None,
"indeterminate": False if success else True,
},
@@ -15367,7 +15453,7 @@ async def download_reference_bundle_api(req: ReferenceBundleDownloadRequest):
"stage": "reference_bundle_download",
"job_id": req.job_id,
"status": "pending",
"message": "参考物种注释包下载任务已提交",
"message": "PPI and Ka/Ks reference species data download submitted",
"created_at": datetime.now().isoformat(),
"species_code": req.speciesCode,
"species_name": req.speciesName,
@@ -17146,7 +17232,11 @@ async def _run_agent_chat(
"session_id": session_id,
"input_paths": list(outcome.get("input_paths") or []),
"handoff_request": (
str((outcome.get("proposal") or {}).get("request") or "").strip()
str(
(outcome.get("proposal") or {}).get("original_request")
or (outcome.get("proposal") or {}).get("request")
or ""
).strip()
if isinstance(outcome.get("proposal"), dict)
else request.message
) or request.message,
@@ -18269,9 +18359,13 @@ def _build_contextual_gene_family_plan_response(request: Any, session_id: str, c
detail["params"] = enrich_stage_params_with_context(stage_id, plan_prefix, detail_params)
stages_detail.append(detail)
if _looks_like_gene_family_workflow_query(getattr(request, "message", "") or "") and not subfamily_prerequisite_mode:
full_workflow_query = _full_gene_family_workflow_query_text(
getattr(request, "message", "") or "",
getattr(request, "history", []) or [],
)
if full_workflow_query and not subfamily_prerequisite_mode:
stage_ids, stages_detail = _normalize_full_gene_family_workflow_plan(
message=getattr(request, "message", "") or "",
message=full_workflow_query,
stage_ids=stage_ids,
stages_detail=stages_detail,
plan_params=plan_params,
@@ -21919,9 +22013,13 @@ async def agent_plan(request: AgentPlanRequest):
intent["gene_family"] = combo_family
full_workflow_normalized = False
if _looks_like_gene_family_workflow_query(request.message or "") and not subfamily_prediction_query:
full_workflow_query = _full_gene_family_workflow_query_text(
request.message or "",
request.history or [],
)
if full_workflow_query and not subfamily_prediction_query:
stage_ids, stages_detail = _normalize_full_gene_family_workflow_plan(
message=request.message or "",
message=full_workflow_query,
stage_ids=stage_ids,
stages_detail=stages_detail,
plan_params=plan_params,
@@ -21953,7 +22051,7 @@ async def agent_plan(request: AgentPlanRequest):
if full_workflow_normalized:
stage_ids, stages_detail = _normalize_full_gene_family_workflow_plan(
message=request.message or "",
message=full_workflow_query,
stage_ids=stage_ids,
stages_detail=stages_detail,
plan_params=plan_params,