From cb122fd088a2b074d14ae35343ab723ce9bcc599 Mon Sep 17 00:00:00 2001 From: wtx Date: Sun, 9 Aug 2026 04:22:38 +0800 Subject: [PATCH] feat: add gene family result reports --- apps/cli/run_gene_family_report.py | 70 ++ apps/web/pipeline_backend.py | 62 +- apps/web/vue3/index.html | 76 +- apps/web/vue3/main.py | 310 ++++++- gfa_core/gene_family_report.py | 333 +++++++ gfa_core/gene_family_report_data.py | 773 ++++++++++++++++ .../mas/agents/interpretation_evidence.py | 24 +- gfa_core/mas/core/fast_intent.py | 8 +- gfa_core/mas/stage_capabilities.py | 17 + gfa_core/mas/stage_catalog.py | 4 + gfa_core/mas/tools/stage_tools.py | 22 + gfa_core/mas/utils/intent_parser.py | 33 + .../stages/gene_family_report_contract_v1.md | 82 ++ gfa_core/templates/gene_family_report_v1.md | 170 ++++ tasks.py | 2 +- .../e2e/verify_gene_family_report_journey.py | 226 +++++ tests/unit/test_gene_family_report.py | 844 ++++++++++++++++++ tests/unit/test_gene_family_report_intent.py | 112 +++ .../unit/test_gene_family_report_triggers.py | 181 ++++ tests/unit/test_gene_family_report_wiring.py | 347 +++++++ 20 files changed, 3676 insertions(+), 20 deletions(-) create mode 100644 apps/cli/run_gene_family_report.py create mode 100644 gfa_core/gene_family_report.py create mode 100644 gfa_core/gene_family_report_data.py create mode 100644 gfa_core/schemas/stages/gene_family_report_contract_v1.md create mode 100644 gfa_core/templates/gene_family_report_v1.md create mode 100644 tests/e2e/verify_gene_family_report_journey.py create mode 100644 tests/unit/test_gene_family_report.py create mode 100644 tests/unit/test_gene_family_report_intent.py create mode 100644 tests/unit/test_gene_family_report_triggers.py create mode 100644 tests/unit/test_gene_family_report_wiring.py diff --git a/apps/cli/run_gene_family_report.py b/apps/cli/run_gene_family_report.py new file mode 100644 index 000000000..a2edf8ab2 --- /dev/null +++ b/apps/cli/run_gene_family_report.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""CLI for the plan-scoped formal gene-family result report.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys +from typing import Sequence + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from gfa_core.gene_family_report import GeneFamilyReportResult, generate_gene_family_report + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Generate or open a gene-family result report") + parser.add_argument("--workspace", type=Path, required=True) + parser.add_argument("--prefix", required=True) + parser.add_argument("--job-id", default="") + parser.add_argument("--expected-stages", default="") + parser.add_argument("--stage-statuses-json", default="{}") + parser.add_argument("--scope-id", default="") + parser.add_argument("--analysis-route", default="") + parser.add_argument("--member-set-id", default="") + parser.add_argument("--rnaseq-scope-id", default="") + parser.add_argument("--action", choices=("generate", "open"), default="generate") + parser.add_argument("--no-model", action="store_true") + return parser + + +def main(argv: Sequence[str] | None = None) -> GeneFamilyReportResult: + args = _parser().parse_args(argv) + statuses = json.loads(args.stage_statuses_json) + if not isinstance(statuses, dict): + raise ValueError("--stage-statuses-json must contain a JSON object") + result = generate_gene_family_report( + args.workspace, + args.prefix, + expected_stages=[value.strip() for value in args.expected_stages.split(",") if value.strip()], + stage_statuses={str(key): str(value) for key, value in statuses.items()}, + scope_id=args.scope_id, + analysis_route=args.analysis_route, + member_set_id=args.member_set_id, + rnaseq_scope_id=args.rnaseq_scope_id, + use_model=not args.no_model, + action=args.action, + job_id=args.job_id, + ) + print( + json.dumps( + { + "status": "success", + "report_path": str(result.report_path), + "manifest_path": str(result.manifest_path), + "report_status": result.report_status, + "generation_mode": result.generation_mode, + "reused": result.reused, + }, + ensure_ascii=False, + ) + ) + return result + + +if __name__ == "__main__": + main() diff --git a/apps/web/pipeline_backend.py b/apps/web/pipeline_backend.py index 7834670b4..105056efd 100644 --- a/apps/web/pipeline_backend.py +++ b/apps/web/pipeline_backend.py @@ -6054,8 +6054,9 @@ def run_stage( target_species = kwargs.get("target_species") interspecies_ref_species = kwargs.get("interspecies_ref_species") - scope_file = str(kwargs.get("scope_file") or kwargs.get("scopeFile") or "").strip() - analysis_route = str(kwargs.get("analysis_route") or kwargs.get("analysisRoute") or "").strip() + scope_file = str(kwargs.get("scope_file") or kwargs.get("scopeFile") or "").strip() + scope_id = str(kwargs.get("scope_id") or kwargs.get("scopeId") or "").strip() + analysis_route = str(kwargs.get("analysis_route") or kwargs.get("analysisRoute") or "").strip() analysis_mode = str(kwargs.get("analysis_mode") or kwargs.get("analysisMode") or "").strip() member_set_id = str(kwargs.get("member_set_id") or kwargs.get("memberSetId") or "").strip() combo_output_label = str(combo_output_label or kwargs.get("comboOutputLabel") or kwargs.get("combo_output_label") or "").strip() @@ -8374,7 +8375,62 @@ def run_stage( else: return f"❌ PEP failed (exit code: {code}). Log: {log_file}", manifest - if stage == "summary": + if stage == "gene_family_report": + expected_stages = kwargs.get("expected_stages") or kwargs.get("expectedStages") or [] + if isinstance(expected_stages, str): + expected_stages = [value.strip() for value in expected_stages.split(",") if value.strip()] + else: + expected_stages = [str(value).strip() for value in expected_stages if str(value).strip()] + report_action = str(kwargs.get("report_action") or kwargs.get("reportAction") or "generate").strip() + if report_action == "generate" and not expected_stages: + return "❌ 基因家族结果报告生成需要非空 expected_stages", manifest + stage_statuses = kwargs.get("stage_statuses") or kwargs.get("stageStatuses") or {} + if isinstance(stage_statuses, str): + try: + stage_statuses = json.loads(stage_statuses) + except json.JSONDecodeError: + return "❌ 基因家族结果报告的 stage_statuses 不是合法 JSON", manifest + if not isinstance(stage_statuses, dict): + return "❌ 基因家族结果报告的 stage_statuses 必须是对象", manifest + report_use_model_raw = kwargs.get("report_use_model", kwargs.get("reportUseModel", True)) + rnaseq_scope_id = str(kwargs.get("rnaseq_scope_id") or kwargs.get("rnaseqScopeId") or "").strip() + cmd = [ + sys.executable, + "apps/cli/run_gene_family_report.py", + "--workspace", + str(ws), + "--prefix", + prefix, + "--job-id", + job.job_id, + "--expected-stages", + ",".join(expected_stages), + "--stage-statuses-json", + json.dumps(stage_statuses, ensure_ascii=False, separators=(",", ":")), + "--action", + report_action, + ] + for flag, value in ( + ("--scope-id", scope_id), + ("--analysis-route", analysis_route), + ("--member-set-id", member_set_id), + ("--rnaseq-scope-id", rnaseq_scope_id), + ): + if value: + cmd += [flag, value] + if not _truthy(report_use_model_raw): + cmd.append("--no-model") + code = run_cmd_logged(cmd, log_file, cwd=Path(".")) + report_manifests = sorted( + final_dir.glob(f"{prefix}*.gene_family_report.manifest.json"), + key=lambda path: path.stat().st_mtime, + ) + report_manifest = report_manifests[-1] if report_manifests else manifest + if code != 0: + return f"❌ 基因家族结果报告生成失败，日志：{log_file}", report_manifest + return f"✅ 基因家族结果报告已生成。日志：{log_file}", report_manifest + + if stage == "summary": summary_manifest_seed = ( ws / "final" / f"{prefix}.identify.manifest.json" if (ws / "final" / f"{prefix}.identify.manifest.json").exists() diff --git a/apps/web/vue3/index.html b/apps/web/vue3/index.html index d87445fe9..8f97daaae 100644 --- a/apps/web/vue3/index.html +++ b/apps/web/vue3/index.html @@ -7950,6 +7950,7 @@ RNA-seq KEGG/fgsea 会优先使用物种级 KEGG gene set；没有物种级缓 pop_snp_density: 'SNP密度分析', rnaseq_de: 'RNA-seq DE', combo: '组合图', + gene_family_report: '基因家族结果报告', reference_classification: '参考亚家族分类', subfamily_confirm: '亚家族标签确认', ppi_kaks_reference_data: 'PPI/KaKs 参考数据', @@ -10111,6 +10112,7 @@ compare: [[flesh, peel]]`; { name: 'pep', label: '1️⃣2️⃣ PEP', status: 'pending', images: [], files: [], elapsed_time: 0 }, { name: 'expression', label: '1️⃣3️⃣ Expression (FPKM)', status: 'pending', images: [], files: [], elapsed_time: 0 }, { name: 'combo', label: '6️⃣ Combo', status: 'pending', images: [], files: [], elapsed_time: 0 }, + { name: 'gene_family_report', label: '📄 基因家族结果报告', status: 'pending', images: [], files: [], elapsed_time: 0 }, // RNA-seq stages { name: 'rnaseq_qc', label: '1️⃣5️⃣ RNA-seq QC', status: 'pending', images: [], files: [], elapsed_time: 0 }, { name: 'rnaseq_align', label: '1️⃣6️⃣ RNA-seq Align', status: 'pending', images: [], files: [], elapsed_time: 0 }, @@ -15080,6 +15082,54 @@ compare: [[flesh, peel]]`; return `⏸️ 批量执行已暂停：成功 ${completedCount}/${totalCount}，暂停 ${pausedCount}/${totalCount}。真实原因：${reasonText}。`; }; + const openGeneratedGeneFamilyReport = async () => { + await refreshResults(jobInfo.job_id, 'gene_family_report'); + const reportStage = stages.find(stage => stage.name === 'gene_family_report'); + const reportFile = (reportStage?.files || []).find(file => + file.role === 'gene_family_report' || String(file.name || '').endsWith('.gene_family_report.md') + ); + if (!reportFile) return false; + rightPanelTab.value = 'results'; + selectFile('gene_family_report', reportFile); + return true; + }; + + const submitGeneFamilyReportForBatch = async (stages, pausedCount) => { + if (pausedCount > 0) return false; + const plannedIds = (stages || []).map(stage => getStageId(stage)).filter( + stageId => stageId && !['summary', 'gene_family_report', 'subfamily_confirm'].includes(stageId) + ); + const geneFamilyStages = new Set([ + 'identify', 'phylogeny', 'motif', 'domain', 'structure', 'promoter', + 'synteny', 'interspecies_synteny', 'karyotype', 'combo', 'ppi', 'expression', 'pep' + ]); + if (!plannedIds.some(stageId => geneFamilyStages.has(stageId))) return false; + const stageStatuses = Object.fromEntries(plannedIds.map(stageId => { + const status = stageExecutionStatus.value[stageId]; + return [stageId, status === 'completed' ? 'success' : status === 'failed' ? 'failed' : 'missing']; + })); + const batchParams = batchExecuteParams.value || {}; + const planParams = batchParams.plan?.params || {}; + const stageParams = (stages || []).find(stage => stage && typeof stage === 'object' && stage.params)?.params || {}; + try { + await executeSingleStageWithParams('gene_family_report', { + expectedStages: plannedIds, + stageStatuses, + reportAction: 'generate', + reportUseModel: true, + scopeId: batchParams.scopeId || planParams.scopeId || stageParams.scopeId || '', + analysisRoute: batchParams.analysisRoute || planParams.analysisRoute || stageParams.analysisRoute || '', + memberSetId: batchParams.memberSetId || planParams.memberSetId || stageParams.memberSetId || '', + rnaseqScopeId: batchParams.rnaseqScopeId || planParams.rnaseqScopeId || stageParams.rnaseqScopeId || '' + }); + addAgentMessage('assistant', '📄 基因家族结果报告已生成。'); + return true; + } catch (err) { + addAgentMessage('assistant', `⚠️ 基因家族结果报告生成失败：${err.message || err}`); + return false; + } + }; + const confirmAndStartBatchExecute = async (event) => { if (!event?.isTrusted) return; batchExecutionUserConfirmed.value = true; @@ -15195,6 +15245,7 @@ compare: [[flesh, peel]]`; }).length; const totalCount = stages.length; + let continuationOpened = false; if (failedCount > 0) { addAgentMessage('assistant', `⚠️ 批量执行完成：成功 ${completedCount}/${totalCount}，失败 ${failedCount}/${totalCount}`); } else if (pausedCount > 0) { @@ -15202,13 +15253,16 @@ compare: [[flesh, peel]]`; } else { addAgentMessage('assistant', `🎉 批量执行完成！成功: ${completedCount}/${totalCount}`); if (isMethodologyDataPreparationContinuationCandidate(stages, completedCount, totalCount, failedCount, pausedCount)) { - await openMethodologyDataPreparationContinuation(); + continuationOpened = await openMethodologyDataPreparationContinuation(); } else if (isMethodologyPostConfirmContinuationCandidate(stages, completedCount, totalCount, failedCount, pausedCount)) { - await openMethodologyPostConfirmContinuation(); + continuationOpened = await openMethodologyPostConfirmContinuation(); } else if (isContextualPostConfirmContinuationCandidate(stages, completedCount, totalCount, failedCount, pausedCount)) { - await openContextualPostConfirmContinuation(); + continuationOpened = await openContextualPostConfirmContinuation(); } } + if (pausedCount === 0 && !continuationOpened) { + await submitGeneFamilyReportForBatch(stages, pausedCount); + } } }; @@ -17496,6 +17550,16 @@ compare: [[flesh, peel]]`; requestBody.geoId = confirmParams.geoId || ''; requestBody.idTransform = confirmParams.idTransform || 'remove_T01'; } + if (stageName === 'gene_family_report') { + requestBody.expectedStages = activeStageParams.expectedStages || []; + requestBody.stageStatuses = activeStageParams.stageStatuses || {}; + requestBody.reportAction = activeStageParams.reportAction || 'generate'; + requestBody.reportUseModel = activeStageParams.reportUseModel !== false; + requestBody.scopeId = activeStageParams.scopeId || ''; + requestBody.analysisRoute = activeStageParams.analysisRoute || ''; + requestBody.memberSetId = activeStageParams.memberSetId || ''; + requestBody.rnaseqScopeId = activeStageParams.rnaseqScopeId || ''; + } if (stageName === 'promoter_plot_revise') { const elements = Array.isArray(confirmParams.promoterElements) ? confirmParams.promoterElements : []; const categories = Array.isArray(confirmParams.promoterCategories) ? confirmParams.promoterCategories : []; @@ -17586,6 +17650,9 @@ compare: [[flesh, peel]]`; } else if (data.async === false) { // 同步执行完成，直接返回结果 console.log(`[DEBUG-EXEC] ${stageName}: sync completed`); + if (stageName === 'gene_family_report') { + await openGeneratedGeneFamilyReport(); + } return data; } else { throw createStructuredExecuteError(data); @@ -18851,6 +18918,9 @@ compare: [[flesh, peel]]`; result: data }); await handleStageCompleted(stageName, data, {}); + if (stageName === 'gene_family_report') { + await openGeneratedGeneFamilyReport(); + } // 更新阶段结果 - 如果有分stage的文件，则更新所有stage的文件 if (stageIdx >= 0 && data.result) { diff --git a/apps/web/vue3/main.py b/apps/web/vue3/main.py index 79490f6d9..0da4410c9 100644 --- a/apps/web/vue3/main.py +++ b/apps/web/vue3/main.py @@ -6876,9 +6876,14 @@ class ExecuteRequest(BaseModel): domCoverage: float = 30.0 idTransform: str = "none" scopeFile: str = "" + scopeId: str = "" scopeMode: str = "" analysisRoute: str = "" memberSetId: str = "" + expectedStages: List[str] = [] + stageStatuses: Dict[str, str] = {} + reportAction: str = "generate" + reportUseModel: bool = True contextRoute: str = "" subfamilyLabelSource: str = "" requiresConfirmedSubfamily: bool = False @@ -7816,8 +7821,13 @@ def run_stage_sync(task_id: str, stage: str, job_id: str, params: dict): # Genome Prepare参数 genome_prepare_type=params.get("genomePrepareType", "auto"), scope_file=params.get("scopeFile", ""), + scope_id=params.get("scopeId", ""), analysis_route=params.get("analysisRoute", ""), member_set_id=params.get("memberSetId", ""), + expected_stages=params.get("expectedStages", []), + stage_statuses=params.get("stageStatuses", {}), + report_action=params.get("reportAction", "generate"), + report_use_model=params.get("reportUseModel", True), analysis_mode=params.get("analysisMode", ""), species_queue=params.get("speciesQueue", []), subfamily_label_source=params.get("subfamilyLabelSource", ""), @@ -11605,7 +11615,7 @@ async def execute_stage(req: ExecuteRequest): }) if req.stage == "gwas_fmt": stage_context["genotype_scope_id"] = trusted_genotype_scope - if req.stage in {"genome_prepare", "genome_prepare_v2"} or req.stage in RNA_SEQ_STAGES: + if req.stage in {"genome_prepare", "genome_prepare_v2", "gene_family_report"} or req.stage in RNA_SEQ_STAGES: stage_context.pop("scope_id", None) stage_context.pop("scope_mode", None) stage_context.pop("analysis_route", None) @@ -11738,6 +11748,7 @@ async def execute_stage(req: ExecuteRequest): skip_completed_check = ( is_utility_stage(req.stage) or is_repair_stage(req.stage) + or req.stage == "gene_family_report" or force_rerun_requested or (req.stage == "gwas_fmt" and not trusted_genotype_scope) or (req.stage == "gwas_mlm" and not gwas_mlm_full_identity) @@ -11966,9 +11977,15 @@ async def execute_stage(req: ExecuteRequest): "domCoverage": req.domCoverage, "idTransform": req.idTransform, "scopeFile": scope_params.get("scopeFile", ""), + "scopeId": getattr(req, "scopeId", ""), + "rnaseqScopeId": getattr(req, "rnaseqScopeId", ""), "scopeMode": scope_params.get("scopeMode", ""), "analysisRoute": stage_context.get("analysis_route") or getattr(req, "analysisRoute", ""), "memberSetId": stage_context.get("member_set_id") or getattr(req, "memberSetId", ""), + "expectedStages": list(getattr(req, "expectedStages", []) or []), + "stageStatuses": dict(getattr(req, "stageStatuses", {}) or {}), + "reportAction": getattr(req, "reportAction", "generate"), + "reportUseModel": bool(getattr(req, "reportUseModel", True)), "contextRoute": getattr(req, "contextRoute", ""), "subfamilyLabelSource": getattr(req, "subfamilyLabelSource", ""), "requiresConfirmedSubfamily": bool(getattr(req, "requiresConfirmedSubfamily", False)), @@ -12728,6 +12745,69 @@ def _confirmed_execute_request( return ExecuteRequest(**params) +def _confirmed_plan_identity(ordered: List[ConfirmedPlanStage]) -> Dict[str, str]: + identity: Dict[str, str] = {} + for stage in reversed(ordered): + params = stage.params or {} + for target, keys in { + "scopeId": ("scopeId", "scope_id"), + "analysisRoute": ("analysisRoute", "analysis_route"), + "memberSetId": ("memberSetId", "member_set_id"), + "rnaseqScopeId": ("rnaseqScopeId", "rnaseq_scope_id"), + }.items(): + if identity.get(target): + continue + identity[target] = next((str(params.get(key) or "").strip() for key in keys if params.get(key)), "") + return identity + + +def _confirmed_plan_report_request( + request: ConfirmedPlanExecutionRequest, + ordered: List[ConfirmedPlanStage], + status: Dict[str, Any], +) -> Optional[ExecuteRequest]: + """Build the one formal-report submission for a terminal confirmed plan.""" + terminal = str(status.get("status") or "") + if terminal not in {"success", "failed"}: + return None + planned = [stage.id for stage in ordered if stage.id not in {"summary", "gene_family_report", "subfamily_confirm"}] + if not planned: + return None + completed = set(status.get("completed_stages") or []) + failed_stage = str((status.get("failure") or {}).get("stage") or "") + stage_statuses = { + stage: "success" if stage in completed else "failed" if stage == failed_stage else "not_run" + for stage in planned + } + if terminal == "success": + stage_statuses = {stage: "success" for stage in planned} + return ExecuteRequest( + stage="gene_family_report", + session_id=request.session_id, + job_id=request.job_id, + prefix=request.prefix, + expectedStages=planned, + stageStatuses=stage_statuses, + reportAction="generate", + reportUseModel=True, + **_confirmed_plan_identity(ordered), + ) + + +async def _submit_confirmed_plan_report( + request: ConfirmedPlanExecutionRequest, + ordered: List[ConfirmedPlanStage], + status: Dict[str, Any], +) -> None: + report_request = _confirmed_plan_report_request(request, ordered, status) + if report_request is None: + return + try: + status["report_submission"] = await execute_stage(report_request) + except Exception as exc: + status["report_submission"] = {"success": False, "error_type": type(exc).__name__} + + async def _wait_for_submitted_stage( task_id: str, stage_id: str, @@ -12777,6 +12857,8 @@ async def _run_confirmed_plan_execution( "run_id": run_id, "plan_id": request.plan_id, "job_id": request.job_id, + "prefix": request.prefix, + **_confirmed_plan_identity(ordered), "status": "running", "stages": [stage.id for stage in ordered], "completed_stages": [], @@ -12820,9 +12902,11 @@ async def _run_confirmed_plan_execution( }, } ) + await _submit_confirmed_plan_report(request, ordered, status) _write_confirmed_plan_status(status_path, status) return status.update({"status": "success", "current_stage": ""}) + await _submit_confirmed_plan_report(request, ordered, status) _write_confirmed_plan_status(status_path, status) @@ -12850,6 +12934,8 @@ async def agent_execute_confirmed(request: ConfirmedPlanExecutionRequest): "run_id": run_id, "plan_id": request.plan_id, "job_id": request.job_id, + "prefix": request.prefix, + **_confirmed_plan_identity(_confirmed_plan_stages(request)), "status": "queued", "stages": [stage.id for stage in _confirmed_plan_stages(request)], } @@ -16759,6 +16845,93 @@ def _answer_methodology_sandbox_qa(message: str) -> dict[str, Any]: return payload +def _gene_family_report_plan_context(job_id: str, prefix: str = "") -> Optional[Dict[str, Any]]: + """Recover only a user-confirmed or previously reported stage plan.""" + safe_job_id = str(job_id or "").strip() + requested_prefix = str(prefix or "").strip() + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", safe_job_id): + return None + final_dir = WORKSPACE_ROOT / safe_job_id / "final" + excluded = {"summary", "gene_family_report", "subfamily_confirm"} + + confirmed: list[tuple[float, Dict[str, Any]]] = [] + confirmed_dir = final_dir / ".confirmed_plans" + for path in confirmed_dir.glob("*.json") if confirmed_dir.is_dir() else (): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + terminal = str(payload.get("status") or "") + plan_prefix = str(payload.get("prefix") or "").strip() + if ( + terminal not in {"success", "failed"} + or str(payload.get("job_id") or "") != safe_job_id + or not plan_prefix + or (requested_prefix and plan_prefix != requested_prefix) + ): + continue + planned = [ + str(stage).strip() + for stage in (payload.get("stages") or []) + if str(stage).strip() and str(stage).strip() not in excluded + ] + if not planned: + continue + completed = {str(stage) for stage in (payload.get("completed_stages") or [])} + failed_stage = str((payload.get("failure") or {}).get("stage") or "") + statuses = { + stage: "success" if terminal == "success" or stage in completed else "failed" if stage == failed_stage else "not_run" + for stage in planned + } + context = { + "prefix": plan_prefix, + "expectedStages": planned, + "stageStatuses": statuses, + "scopeId": str(payload.get("scopeId") or payload.get("scope_id") or ""), + "analysisRoute": str(payload.get("analysisRoute") or payload.get("analysis_route") or ""), + "memberSetId": str(payload.get("memberSetId") or payload.get("member_set_id") or ""), + "rnaseqScopeId": str(payload.get("rnaseqScopeId") or payload.get("rnaseq_scope_id") or ""), + } + confirmed.append((path.stat().st_mtime, context)) + if confirmed: + return max(confirmed, key=lambda item: item[0])[1] + + pattern = f"{requested_prefix}.gene_family_report.manifest.json" if requested_prefix else "*.gene_family_report.manifest.json" + reports: list[tuple[float, Dict[str, Any]]] = [] + for path in final_dir.glob(pattern) if final_dir.is_dir() else (): + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + report_prefix = str(manifest.get("prefix") or "").strip() + parameters = manifest.get("parameters") or {} + planned = [ + str(stage).strip() + for stage in (parameters.get("expected_stages") or []) + if str(stage).strip() and str(stage).strip() not in excluded + ] + if ( + manifest.get("stage") != "gene_family_report" + or manifest.get("status") != "success" + or not report_prefix + or (requested_prefix and report_prefix != requested_prefix) + or not planned + ): + continue + recorded_statuses = parameters.get("stage_statuses") or {} + context = { + "prefix": report_prefix, + "expectedStages": planned, + "stageStatuses": {stage: str(recorded_statuses.get(stage) or "missing") for stage in planned}, + "scopeId": "", + "analysisRoute": "", + "memberSetId": "", + "rnaseqScopeId": "", + } + reports.append((path.stat().st_mtime, context)) + return max(reports, key=lambda item: item[0])[1] if reports else None + + async def _run_agent_chat( request: AgentChatRequest, progress_callback=None, @@ -16857,6 +17030,7 @@ async def _run_agent_chat( from gfa_core.mas.utils.intent_parser import ( deterministic_agent_chat_intent, parse_sequence_extract_request, + resolve_gene_family_report_intent, ) utility_params = parse_sequence_extract_request(request.message) @@ -16903,11 +17077,10 @@ async def _run_agent_chat( "session_id": session_id, } - deterministic_intent = deterministic_agent_chat_intent( + compact_message = re.sub(r"\s+", "", str(request.message or "").lower()) + deterministic_intent = None if "基因家族" in compact_message else deterministic_agent_chat_intent( request.message, - rnaseq_context=any( - str(stage).startswith("rnaseq_") for stage in completed_stages - ), + rnaseq_context=any(str(stage).startswith("rnaseq_") for stage in completed_stages), ) if deterministic_intent is not None: report_action = str( @@ -16955,6 +17128,76 @@ async def _run_agent_chat( "handoff_request": request.message, } + gene_report_intent = resolve_gene_family_report_intent(request.message) + if gene_report_intent is not None: + plan_context = _gene_family_report_plan_context( + request.job_id, + str(router_context_terms.get("prefix") or ""), + ) + report_action = str((gene_report_intent.get("params") or {}).get("reportAction") or "generate") + if plan_context is None: + report_message = "已识别为基因家族结果报告请求，但当前 job 缺少可验证的分析计划，无法生成或打开正式报告。" + if conversation_turn is not None: + active_store.finish_turn( + conversation_turn, + assistant_message=report_message, + interaction_kind="independent", + action="reply", + intent=None, + ) + return { + "success": False, + "status": "report_plan_missing", + "action": None, + "intent": None, + "message": report_message, + "sources": [], + "tool_trace": [], + "mode": "analysis", + "completed_stages": completed_stages, + "model_used": "rule", + "session_id": session_id, + } + report_message = ( + "已识别为打开基因家族结果报告。" + if report_action == "open" + else "已识别为生成基因家族结果报告。" + ) + report_params = {**(gene_report_intent.get("params") or {}), **plan_context} + resolved_intent = { + **gene_report_intent, + "stage": "gene_family_report", + "primary_stage": "gene_family_report", + "taskType": "analysis", + "task_kind": "report", + "prefix": plan_context["prefix"], + "params": report_params, + "job_id": request.job_id, + "workspace": str(WORKSPACE_ROOT / request.job_id) if request.job_id else "", + "message": report_message, + } + if conversation_turn is not None: + active_store.finish_turn( + conversation_turn, + assistant_message=report_message, + interaction_kind="independent", + action="propose_analysis", + intent=resolved_intent, + ) + return { + "success": True, + "status": "ok", + "action": "propose_analysis", + "intent": resolved_intent, + "sources": [], + "tool_trace": [], + "mode": "analysis", + "completed_stages": completed_stages, + "model_used": "rule", + "session_id": session_id, + "handoff_request": request.message, + } + global llm_manager active_llm_manager = llm_manager if active_llm_manager is None: @@ -20060,6 +20303,63 @@ async def agent_plan(request: AgentPlanRequest): extension_intent = _match_extension_stage_intent(request.message, request.job_id or "") if extension_intent: return _extension_stage_plan_response(extension_intent, session_id, request.job_id or "") + report_intent = dict(request.intent or {}) + if report_intent.get("stages") == ["gene_family_report"]: + report_params = dict(report_intent.get("params") or {}) + report_prefix = str( + report_intent.get("prefix") + or report_params.get("prefix") + or request.prefix + or "" + ).strip() + if not report_params.get("expectedStages"): + recovered = _gene_family_report_plan_context(request.job_id, report_prefix) + if recovered: + report_params.update(recovered) + report_prefix = recovered["prefix"] + if not report_params.get("expectedStages"): + return { + "success": False, + "mode": "gene_family_report_deterministic", + "status": "report_plan_missing", + "message": "当前 job 缺少可验证的基因家族分析计划，无法生成或打开正式报告。", + "session_id": session_id, + } + report_params["prefix"] = report_prefix + report_intent.update( + { + "stage": "gene_family_report", + "primary_stage": "gene_family_report", + "stages": ["gene_family_report"], + "prefix": report_prefix, + "params": report_params, + } + ) + detail = _stage_detail_defaults("gene_family_report") + detail.update({"order": 1, "dependencies": [], "params": dict(report_params)}) + return { + "success": True, + "mode": "gene_family_report_deterministic", + "plan": { + "analysis_type": "gene_family", + "prefix": report_prefix, + "description": "生成或打开基因家族结果报告", + "estimated_time": "2分钟", + "estimated_total_time": "2分钟", + "execution_strategy": "sequential", + "max_parallel": 1, + "total_stages": 1, + "stages": ["gene_family_report"], + "stages_detail": [detail], + "all_stages": ["gene_family_report"], + "all_stages_detail": [detail], + "parallel_groups": [], + "params": report_params, + }, + "intent": report_intent, + "completed_stages": [], + "session_id": session_id, + } plan_role_resolution = ( _reconcile_confirmed_gene_family_species( request.job_id, diff --git a/gfa_core/gene_family_report.py b/gfa_core/gene_family_report.py new file mode 100644 index 000000000..5fd145909 --- /dev/null +++ b/gfa_core/gene_family_report.py @@ -0,0 +1,333 @@ +"""Generate the formal report for one gene-family analysis plan.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +import json +from pathlib import Path +import re +from typing import Any, Iterable, Mapping + +from gfa_core.gene_family_report_data import ( + ReportData, + ReportSection, + build_report_data, +) +from gfa_core.mas.agents.interpretation_evidence import load_stage_manifests + + +REPORT_STAGE = "gene_family_report" +REPORT_TEMPLATE_VERSION = "gene_family_report_template_v1" +EXCLUDED_PLAN_STAGES = {"summary", REPORT_STAGE, "subfamily_confirm"} +TEMPLATE_PATH = Path(__file__).with_name("templates") / "gene_family_report_v1.md" +CONNECTORS = { + "also": "此外，", + "however": "不过，", + "therefore": "因此，", +} + + +class NoSuccessfulGeneFamilyEvidence(RuntimeError): + """Raised when the requested plan has no successful result to report.""" + + +@dataclass(frozen=True) +class GeneFamilyReportResult: + report_path: Path + manifest_path: Path + report_status: str + generation_mode: str + included_stages: tuple[str, ...] + reused: bool = False + + +def _normalized_stages(stages: Iterable[str] | None) -> list[str]: + result: list[str] = [] + for raw in stages or (): + stage = str(raw).strip() + if stage and stage not in EXCLUDED_PLAN_STAGES and stage not in result: + result.append(stage) + return result + + +def _parse_json_object(text: str) -> dict[str, Any]: + value = text.strip() + if value.startswith("```"): + value = re.sub(r"^```(?:json)?\s*", "", value, flags=re.IGNORECASE) + value = re.sub(r"\s*```$", "", value) + parsed = json.loads(value) + if not isinstance(parsed, dict): + raise ValueError("model output must be a JSON object") + return parsed + + +def _api_payload(report_data: ReportData) -> dict[str, Any]: + return { + "sections": [ + { + "section_id": section.section_id, + "sentences": [ + {"sentence_id": sentence.id, "text": sentence.text} + for sentence in section.sentences + ], + } + for section in report_data.sections + ], + "allowed_connectors": list(CONNECTORS), + } + + +def _generate_api_narrative(payload: dict, stages: list[str]) -> tuple[dict[str, Any], dict[str, Any]]: + from gfa_core.mas.config import config as mas_config + from gfa_core.mas.llm.base import LLMMessage + from gfa_core.mas.llm.manager import LLMManager + + prompt = ( + "仅编排以下不可改写句子。输出 JSON 字段只能是 sections；每节只能含 section_id、paragraphs，" + "每段只能含 sentence_ids、connector_ids。必须保留所有章节和句子且各出现一次，" + "句子不得跨章节；connector_ids 只能来自 allowed_connectors。计划阶段为：" + f"{','.join(stages)}。\n" + + json.dumps(payload, ensure_ascii=False) + ) + response = LLMManager(config=mas_config.load_llm_config()).call( + [ + LLMMessage(role="system", content="你是证据约束的生物信息学报告编辑器，只返回 JSON。"), + LLMMessage(role="user", content=prompt), + ], + track_agent_name="GeneFamilyReport", + ) + return _parse_json_object(response.content), { + "status": "generated", + "provider": response.provider, + "model": response.model, + } + + +def _default_arrangement(report_data: ReportData) -> dict[str, Any]: + return { + "sections": [ + { + "section_id": section.section_id, + "paragraphs": [ + {"sentence_ids": [sentence.id], "connector_ids": []} + for sentence in section.sentences + ], + } + for section in report_data.sections + ] + } + + +def _validate_arrangement(arrangement: Any, report_data: ReportData) -> None: + if not isinstance(arrangement, dict) or set(arrangement) != {"sections"}: + raise ValueError("API response has unknown or missing root fields") + sections = arrangement["sections"] + if not isinstance(sections, list): + raise ValueError("API sections must be a list") + expected = {section.section_id: {sentence.id for sentence in section.sentences} for section in report_data.sections} + seen_sections: set[str] = set() + for section in sections: + if not isinstance(section, dict) or set(section) != {"section_id", "paragraphs"}: + raise ValueError("API section has unknown or missing fields") + section_id = section["section_id"] + if not isinstance(section_id, str) or section_id not in expected or section_id in seen_sections: + raise ValueError("API changed or repeated a report section") + seen_sections.add(section_id) + paragraphs = section["paragraphs"] + if not isinstance(paragraphs, list): + raise ValueError("API paragraphs must be a list") + seen_sentences: list[str] = [] + for paragraph in paragraphs: + if not isinstance(paragraph, dict) or set(paragraph) != {"sentence_ids", "connector_ids"}: + raise ValueError("API paragraph has unknown or missing fields") + sentence_ids = paragraph["sentence_ids"] + connector_ids = paragraph["connector_ids"] + if not isinstance(sentence_ids, list) or not sentence_ids or any(not isinstance(value, str) for value in sentence_ids): + raise ValueError("API sentence_ids must be a non-empty string list") + if not isinstance(connector_ids, list) or len(connector_ids) != len(sentence_ids) - 1: + raise ValueError("API connector count does not match its paragraph") + if any(not isinstance(value, str) or value not in CONNECTORS for value in connector_ids): + raise ValueError("API used an unknown connector") + if any(value not in expected[section_id] for value in sentence_ids): + raise ValueError("API used an unknown or cross-section sentence") + seen_sentences.extend(sentence_ids) + if len(seen_sentences) != len(set(seen_sentences)) or set(seen_sentences) != expected[section_id]: + raise ValueError("API omitted or repeated a required sentence") + if seen_sections != set(expected): + raise ValueError("API omitted a report section") + + +def _template_blocks() -> list[tuple[str, str]]: + template = TEMPLATE_PATH.read_text(encoding="utf-8") + marker = re.compile(r"(?m)^\s*$") + matches = list(marker.finditer(template)) + if not matches: + raise ValueError(f"报告模板没有章节边界：{TEMPLATE_PATH}") + return [ + ( + match.group(1), + template[match.end(): matches[index + 1].start() if index + 1 < len(matches) else len(template)].strip(), + ) + for index, match in enumerate(matches) + ] + + +def _arranged_content(section: ReportSection, arranged: Mapping[str, Any]) -> str: + sentences = {sentence.id: sentence.text for sentence in section.sentences} + paragraphs: list[str] = [] + for paragraph in arranged["paragraphs"]: + sentence_ids = paragraph["sentence_ids"] + text = sentences[sentence_ids[0]] + for connector_id, sentence_id in zip(paragraph["connector_ids"], sentence_ids[1:]): + text += f" {CONNECTORS[connector_id]}{sentences[sentence_id]}" + paragraphs.append(text) + return "\n\n".join(paragraphs) + + +def _render_markdown( + report_data: ReportData, + *, + job_id: str, + generation_mode: str, + arrangement: Mapping[str, Any], +) -> str: + section_data = {section.section_id: section for section in report_data.sections} + arranged = {section["section_id"]: section for section in arrangement["sections"]} + rendered: list[str] = [] + for section_id, block in _template_blocks(): + if section_id not in section_data: + continue + rendered.append(block.replace("{{SECTION_CONTENT}}", _arranged_content(section_data[section_id], arranged[section_id]))) + markdown = "\n\n".join(rendered) + fallback_notice = ( + "> 生成模式：候补脚本。模型 API 不可用或输出越过语言编排边界；报告事实仍来自确定性结果数据。" + if generation_mode == "fallback_script" + else "" + ) + replacements = { + "{{PREFIX}}": report_data.prefix, + "{{JOB_ID}}": job_id or "未指定", + "{{GENERATION_MODE}}": "API 编排" if generation_mode == "api" else "候补脚本", + "{{GENERATED_AT}}": datetime.now().isoformat(timespec="seconds"), + "{{FALLBACK_NOTICE}}": fallback_notice, + } + for placeholder, value in replacements.items(): + markdown = markdown.replace(placeholder, value) + if "{{" in markdown or "【填写】" in markdown: + raise ValueError("报告模板仍含未替换占位符") + return markdown.rstrip() + "\n" + + +def generate_gene_family_report( + workspace: Path | str, + prefix: str, + *, + expected_stages: Iterable[str] | None = None, + stage_statuses: Mapping[str, str] | None = None, + scope_id: str = "", + analysis_route: str = "", + member_set_id: str = "", + rnaseq_scope_id: str = "", + use_model: bool = True, + action: str = "generate", + job_id: str = "", +) -> GeneFamilyReportResult: + if action not in {"generate", "open"}: + raise ValueError("action must be 'generate' or 'open'") + final_dir = Path(workspace).expanduser().resolve() / "final" + report_path = final_dir / f"{prefix}.gene_family_report.md" + manifest_path = final_dir / f"{prefix}.gene_family_report.manifest.json" + if action == "open" and report_path.is_file() and manifest_path.is_file(): + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + metrics = manifest.get("metrics") or {} + return GeneFamilyReportResult( + report_path, + manifest_path, + str(metrics.get("report_status") or "partial"), + str(metrics.get("generation_mode") or "fallback_script"), + tuple(metrics.get("included_stages") or []), + reused=True, + ) + + planned = _normalized_stages(expected_stages) + if not planned: + raise ValueError("expected_stages is required when generating a gene-family report") + manifests = load_stage_manifests( + final_dir, + prefix, + scope_id=scope_id, + analysis_route=analysis_route, + member_set_id=member_set_id, + rnaseq_scope_id=rnaseq_scope_id, + ) + statuses = {str(key): str(value) for key, value in (stage_statuses or {}).items()} + for stage in planned: + statuses.setdefault(stage, str((manifests.get(stage) or {}).get("status") or "missing")) + report_data = build_report_data(final_dir, prefix, planned, statuses, manifests) + if not report_data.included: + raise NoSuccessfulGeneFamilyEvidence("暂无可生成的基因家族分析结果") + + arrangement = _default_arrangement(report_data) + generation_mode = "fallback_script" + model_metadata: dict[str, Any] = {"status": "disabled"} + if use_model: + try: + candidate, model_metadata = _generate_api_narrative(_api_payload(report_data), list(report_data.included)) + _validate_arrangement(candidate, report_data) + arrangement = candidate + generation_mode = "api" + except Exception: + model_metadata = { + "status": "fallback", + "reason": "api_unavailable_or_boundary_violation", + } + + markdown = _render_markdown( + report_data, + job_id=job_id, + generation_mode=generation_mode, + arrangement=arrangement, + ) + final_dir.mkdir(parents=True, exist_ok=True) + report_path.write_text(markdown, encoding="utf-8") + manifest = { + "manifest_version": "1.0", + "stage": REPORT_STAGE, + "status": "success", + "prefix": prefix, + "job_id": job_id, + "scope_id": scope_id, + "analysis_route": analysis_route, + "member_set_id": member_set_id, + "rnaseq_scope_id": rnaseq_scope_id, + "parameters": {"expected_stages": planned, "stage_statuses": {stage: statuses[stage] for stage in planned}}, + "metrics": { + "report_status": report_data.report_status, + "generation_mode": generation_mode, + "included_stages": list(report_data.included), + "unsupported_stages": list(report_data.unsupported_stages), + "failed_or_missing_stages": [stage for stage in planned if stage not in report_data.included], + }, + "model": model_metadata, + "artifacts": { + "gene_family_report": { + "path": f"final/{report_path.name}", + "full_path": str(report_path), + "type": "markdown", + }, + "gene_family_report_manifest": { + "path": f"final/{manifest_path.name}", + "full_path": str(manifest_path), + "type": "manifest", + }, + }, + } + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return GeneFamilyReportResult( + report_path, + manifest_path, + report_data.report_status, + generation_mode, + report_data.included, + ) diff --git a/gfa_core/gene_family_report_data.py b/gfa_core/gene_family_report_data.py new file mode 100644 index 000000000..968765ac8 --- /dev/null +++ b/gfa_core/gene_family_report_data.py @@ -0,0 +1,773 @@ +"""Deterministic data model for the formal gene-family report.""" + +from __future__ import annotations + +from dataclasses import dataclass +import csv +import json +from pathlib import Path +import re +from typing import Any, Mapping, Sequence + + +@dataclass(frozen=True) +class MissingMetric: + """A metric that is genuinely absent from every declared source.""" + + label: str + checked_sources: tuple[str, ...] + + def render(self) -> str: + checked = "、".join(self.checked_sources) + return f"{self.label}：源结果未提供此指标（已检查：{checked}）。" + + +@dataclass(frozen=True) +class Sentence: + """One immutable report sentence decided by deterministic code.""" + + id: str + text: str + kind: str + evidence_ids: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ReportSection: + section_id: str + stage: str | None + facts: tuple[Sentence, ...] = () + observations: tuple[Sentence, ...] = () + interpretations: tuple[Sentence, ...] = () + limitations: tuple[Sentence, ...] = () + recommendations: tuple[Sentence, ...] = () + + @property + def sentences(self) -> tuple[Sentence, ...]: + return ( + self.facts + + self.observations + + self.interpretations + + self.limitations + + self.recommendations + ) + + +@dataclass(frozen=True) +class ReportData: + prefix: str + planned: tuple[str, ...] + statuses: Mapping[str, str] + included: tuple[str, ...] + unsupported_stages: tuple[str, ...] + report_status: str + sections: tuple[ReportSection, ...] + + +STAGE_SECTION_IDS = { + "identify": "stage.identify", + "phylogeny": "stage.phylogeny", + "reference_classification": "stage.reference_classification", + "motif": "stage.motif", + "domain": "stage.domain", + "structure": "stage.structure", + "combo": "stage.combo", + "promoter": "stage.promoter", + "tfbs": "stage.tfbs", + "synteny": "stage.synteny", + "karyotype": "stage.karyotype", + "interspecies_synteny": "stage.interspecies_synteny", + "pep": "stage.pep", +} + + +_REQUIRED_METRICS = { + "identify": (("member_count", "最终成员数"),), + "phylogeny": ( + ("sequences_count", "序列数"), + ("msa_length", "多序列比对长度"), + ), + "motif": ( + ("motif_count", "Motif 类别数"), + ("total_sites", "Motif 总位点数"), + ("motif_width_range", "Motif 宽度范围"), + ), + "pep": ( + ("protein_count", "蛋白数"), + ("tasks_success", "成功子任务数"), + ("tasks_total", "子任务总数"), + ), +} + + +_ARTIFACT_METRIC_ROLES = { + ("identify", "member_count"): ("identified_members", "identified_members_id", "member_ids"), + ("pep", "protein_count"): ("pep_properties", "properties_table"), +} + + +def _manifest_source(manifest: Mapping[str, Any], field: str) -> str: + path = str(manifest.get("_manifest_path") or "stage manifest") + return f"{Path(path).name}#metrics.{field}" + + +def _artifact_path(final_dir: Path, artifact: Any) -> Path | None: + if not isinstance(artifact, Mapping): + return None + raw = artifact.get("full_path") or artifact.get("path") + if not raw: + return None + path = Path(str(raw)) + if path.is_file(): + return path + candidates = [final_dir / path.name] + if path.is_absolute(): + job_index = next((index for index, part in enumerate(path.parts) if part.startswith("job_")), None) + if job_index is not None: + candidates.append(final_dir.parent.joinpath(*path.parts[job_index + 1 :])) + else: + candidates.append(final_dir.parent / path) + return next((candidate for candidate in candidates if candidate.is_file()), None) + + +def _artifact_by_suffix(final_dir: Path, manifest: Mapping[str, Any], suffix: str) -> Path | None: + artifacts = manifest.get("artifacts") if isinstance(manifest.get("artifacts"), Mapping) else {} + for role in sorted(artifacts): + if str(role).endswith(suffix): + path = _artifact_path(final_dir, artifacts[role]) + if path is not None: + return path + return None + + +def _count_named_records(path: Path) -> int: + suffix = path.suffix.lower() + try: + if suffix in {".fa", ".faa", ".fasta", ".pep"}: + return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.startswith(">")) + if suffix in {".tsv", ".csv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8", newline="") as handle: + return sum(1 for _ in csv.DictReader(handle, delimiter=delimiter)) + if suffix == ".json": + value = json.loads(path.read_text(encoding="utf-8")) + if isinstance(value, list): + return len(value) + if isinstance(value, Mapping): + for key in ("members", "proteins", "records"): + if isinstance(value.get(key), list): + return len(value[key]) + raise ValueError("JSON does not contain a declared record list") + return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.strip()) + except (OSError, UnicodeError, csv.Error, json.JSONDecodeError, ValueError) as exc: + raise ValueError(f"无法解析声明的报告数据源 {path}: {exc}") from exc + + +def _metric( + final_dir: Path, + stage: str, + manifest: Mapping[str, Any], + field: str, + label: str, +) -> tuple[Any | MissingMetric, tuple[str, ...]]: + metrics = manifest.get("metrics") if isinstance(manifest.get("metrics"), Mapping) else {} + manifest_source = _manifest_source(manifest, field) + if stage == "identify" and field == "member_count": + for alias in ("member_count", "identified_members_count", "final_members"): + if metrics.get(alias) is not None: + source = _manifest_source(manifest, alias) + return metrics[alias], (source,) + elif field == "motif_width_range": + motifs = metrics.get("motifs") + if isinstance(motifs, Sequence) and not isinstance(motifs, (str, bytes)): + widths = [item.get("width") for item in motifs if isinstance(item, Mapping) and item.get("width") is not None] + if widths: + try: + numeric = [int(value) for value in widths] + except (TypeError, ValueError) as exc: + raise ValueError(f"无法解析 {manifest_source.replace(field, 'motifs[].width')}") from exc + source = _manifest_source(manifest, "motifs[].width") + return (min(numeric), max(numeric)), (source,) + elif field in metrics and metrics[field] is not None: + return metrics[field], (manifest_source,) + + checked = [manifest_source] + artifacts = manifest.get("artifacts") if isinstance(manifest.get("artifacts"), Mapping) else {} + for role in _ARTIFACT_METRIC_ROLES.get((stage, field), ()): + checked.append(f"artifact:{role}") + path = _artifact_path(final_dir, artifacts.get(role)) + if path is not None: + return _count_named_records(path), (str(path),) + return MissingMetric(label, tuple(checked)), tuple(checked) + + +def _gene_aliases(gene_id: str) -> tuple[str, ...]: + value = str(gene_id).strip() + aliases = {value, re.sub(r"\.T\d+$", "", value), re.sub(r"\.\d+$", "", value)} + return tuple(alias for alias in aliases if alias) + + +def _parse_interspecies_artifacts( + final_dir: Path, + manifest: Mapping[str, Any], +) -> tuple[int, int, tuple[str, ...]] | None: + artifacts = manifest.get("artifacts") if isinstance(manifest.get("artifacts"), Mapping) else {} + mapping_path = _artifact_path(final_dir, artifacts.get("gene_to_species")) + collinearity_path = _artifact_by_suffix(final_dir, manifest, "_collinearity") + if mapping_path is None or collinearity_path is None: + return None + groups: dict[str, str] = {} + try: + with mapping_path.open("r", encoding="utf-8", newline="") as handle: + for row in csv.DictReader(handle, delimiter="\t"): + gene_id = str(row.get("gene_id") or "").strip() + group = str(row.get("group") or "").strip().lower() + if gene_id and group in {"target", "ref"}: + for alias in _gene_aliases(gene_id): + groups[alias] = group + alignment_blocks = 0 + target_ref_pairs = 0 + with collinearity_path.open("r", encoding="utf-8") as handle: + for line in handle: + if line.startswith("## Alignment"): + alignment_blocks += 1 + continue + match = re.search(r":\s*(\S+)\s+(\S+)", line) if line.startswith(" ") else None + if not match: + continue + pair_groups = { + groups[alias] + for gene_id in match.groups() + for alias in _gene_aliases(gene_id) + if alias in groups + } + if pair_groups == {"target", "ref"}: + target_ref_pairs += 1 + except (OSError, UnicodeError, csv.Error) as exc: + raise ValueError(f"无法解析种间共线性命名产物：{mapping_path}；{collinearity_path}: {exc}") from exc + return alignment_blocks, target_ref_pairs, (str(collinearity_path), str(mapping_path)) + + +def _sentence(sentence_id: str, text: str, kind: str, *evidence_ids: str) -> Sentence: + return Sentence(sentence_id, text, kind, tuple(evidence_ids)) + + +def _metrics(manifest: Mapping[str, Any]) -> Mapping[str, Any]: + value = manifest.get("metrics") + return value if isinstance(value, Mapping) else {} + + +def _artifacts(manifest: Mapping[str, Any]) -> Mapping[str, Any]: + value = manifest.get("artifacts") + return value if isinstance(value, Mapping) else {} + + +def _artifact_for_roles( + final_dir: Path, + manifest: Mapping[str, Any], + *roles: str, +) -> tuple[Path | None, str | None]: + artifacts = _artifacts(manifest) + for role in roles: + path = _artifact_path(final_dir, artifacts.get(role)) + if path is not None: + return path, role + for role, artifact in artifacts.items(): + if any(token in str(role) for token in roles): + path = _artifact_path(final_dir, artifact) + if path is not None: + return path, str(role) + return None, None + + +def _relative_artifact(final_dir: Path, manifest: Mapping[str, Any], *roles: str) -> tuple[str, str] | None: + artifacts = _artifacts(manifest) + for role in roles: + artifact = artifacts.get(role) + path = _artifact_path(final_dir, artifact) + if path is not None: + try: + relative = path.relative_to(final_dir.parent) + except ValueError: + relative = Path("final") / path.name + return relative.as_posix(), f"artifact:{role}" + if isinstance(artifact, Mapping): + raw = artifact.get("path") + if raw and not Path(str(raw)).is_absolute() and ".." not in Path(str(raw)).parts: + return Path(str(raw)).as_posix(), f"artifact:{role}" + return None + + +def _tabular_rows(path: Path) -> list[dict[str, str]]: + try: + lines = [ + line + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + if not lines: + return [] + return [dict(row) for row in csv.DictReader(lines, delimiter="\t")] + except (OSError, UnicodeError, csv.Error) as exc: + raise ValueError(f"无法解析命名表格 {path}: {exc}") from exc + + +def _element_counts(path: Path) -> list[tuple[str, int]]: + result: list[tuple[str, int]] = [] + try: + for line in path.read_text(encoding="utf-8").splitlines(): + parts = line.split("\t", 1) + if len(parts) != 2: + continue + try: + count = int(parts[0]) + except ValueError: + continue + result.append((parts[1].strip(), count)) + except (OSError, UnicodeError) as exc: + raise ValueError(f"无法解析启动子元件计数 {path}: {exc}") from exc + return sorted(result, key=lambda item: (-item[1], item[0])) + + +def _key_artifact_sentence( + final_dir: Path, + manifest: Mapping[str, Any], + section_id: str, + label: str, + *roles: str, +) -> Sentence | None: + artifact = _relative_artifact(final_dir, manifest, *roles) + if artifact is None: + return None + path, evidence = artifact + return _sentence(f"{section_id}.artifact.{roles[0]}", f"关键产物：{label} `{path}`。", "fact", evidence) + + +def _stage_section( + final_dir: Path, + prefix: str, + stage: str, + status: str, + manifest: Mapping[str, Any], +) -> ReportSection: + section_id = STAGE_SECTION_IDS[stage] + facts: list[Sentence] = [] + observations: list[Sentence] = [] + interpretations: list[Sentence] = [] + limitations: list[Sentence] = [] + recommendations: list[Sentence] = [] + + if status != "success" or str(manifest.get("status") or "") != "success": + facts.append(_sentence(f"{section_id}.status", f"阶段状态：`{status}`。", "fact", _manifest_source(manifest, "status"))) + limitations.append( + _sentence( + f"{section_id}.not_completed", + "该阶段未成功完成，因此本报告不生成结果推断。", + "limitation", + ) + ) + return ReportSection(section_id, stage, tuple(facts), limitations=tuple(limitations)) + + metrics = _metrics(manifest) + if stage == "identify": + member_count, sources = _metric(final_dir, stage, manifest, "member_count", "最终成员数") + family = prefix.split(".", 1)[1] if "." in prefix else prefix + facts.append(_sentence(f"{section_id}.object", "分析对象：目标物种完整蛋白集中的候选成员。", "fact")) + if isinstance(member_count, MissingMetric): + facts.append(_sentence(f"{section_id}.missing.member_count", member_count.render(), "fact", *sources)) + else: + hmm = metrics.get("hmm_members") + blast = metrics.get("blast_members") + search = f"；HMM 与 BLAST 各支持 {hmm} 和 {blast} 个候选" if hmm is not None and blast is not None else "" + observations.append(_sentence( + f"{section_id}.observation.members", + f"目标物种最终成员数：{member_count}。本阶段形成 {member_count} 个 {family}家族成员{search}。", + "observation", *sources, + )) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：最终名单是后续阶段的目标物种成员集合；搜索方法的一致数量支持集合稳定，但不等同于实验功能确认。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：成员边界仍受蛋白注释、搜索阈值和转录本选择规则影响。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "目标成员 FASTA", "identified_members", "target_Ft_identified_members_pep") + elif stage == "phylogeny": + sequences = metrics.get("sequences_count") + msa_length = metrics.get("msa_length") + species = metrics.get("scope_species_counts") if isinstance(metrics.get("scope_species_counts"), Mapping) else {} + object_text = ( + f"分析对象：目标物种 {species.get('Ft')} 条与参考物种 {species.get('At')} 条蛋白合并的 {sequences} 条序列。" + if species + else f"分析对象：目标与参考成员合并集合中的 {sequences} 条蛋白序列。" + ) + facts.append(_sentence(f"{section_id}.object", object_text, "fact", _manifest_source(manifest, "scope_species_counts"))) + observations.append(_sentence(f"{section_id}.observation", f"序列数：{sequences}；多序列比对长度：{msa_length}。目标与参考成员共 {sequences} 条序列进入比对，本阶段使用 {sequences} 条序列，比对长度为 {msa_length}。", "observation", _manifest_source(manifest, "sequences_count"), _manifest_source(manifest, "msa_length"))) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：该树用于比较序列关系并辅助分组；它分析的是目标加参考集合，不应与 24 个目标成员的鉴定集合混称。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：源 Manifest 未汇总各主要分支的支持值，不能据此报告小组稳定性等级。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "关系树", "tree") + elif stage == "reference_classification": + target_count = int(metrics.get("target_gene_count") or 0) + all_rows = int(metrics.get("subfamily_rows") or 0) + assignment_path, assignment_role = _artifact_for_roles(final_dir, manifest, "assignment_table") + rows = _tabular_rows(assignment_path) if assignment_path else [] + counts: dict[str, int] = {} + subfamilies: set[str] = set() + for row in rows: + confidence = str(row.get("confidence") or "").lower() + if confidence: + counts[confidence] = counts.get(confidence, 0) + 1 + label = str(row.get("assigned_subfamily") or "").strip() + if label: + subfamilies.add(label) + facts.append(_sentence(f"{section_id}.object", f"分析对象：{target_count} 个目标成员与 {max(0, all_rows - target_count)} 个参考成员的联合分类。", "fact")) + if rows: + observations.append(_sentence(f"{section_id}.observation", f"分类结果：{len(rows)} 个目标成员分入 {len(subfamilies)} 个小组；高信心 {counts.get('high', 0)} 个、中信心 {counts.get('medium', 0)} 个、低信心 {counts.get('low', 0)} 个。", "observation", f"artifact:{assignment_role}")) + else: + missing = MissingMetric("分类信心分布", ("artifact:assignment_table",)) + facts.append(_sentence(f"{section_id}.missing", missing.render(), "fact", *missing.checked_sources)) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：参考成员为目标成员提供小组标签，其中低信心标签只能作为后续分组线索。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：分类依赖参考成员覆盖和树拓扑；低信心成员不宜用于强分组结论。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "分类明细", "assignment_table") + elif stage == "motif": + motifs = metrics.get("motifs") if isinstance(metrics.get("motifs"), Sequence) else [] + widths = [int(item["width"]) for item in motifs if isinstance(item, Mapping) and item.get("width") is not None] + count = metrics.get("motif_count") + total_sites = metrics.get("total_sites") + sequence_count = metrics.get("sequence_count") or metrics.get("protein_record_count") + if sequence_count is None: + facts.append(_sentence(f"{section_id}.object", "分析对象：当前 Motif Manifest 声明的蛋白集合。", "fact")) + missing = MissingMetric("Motif 输入序列数", (_manifest_source(manifest, "sequence_count"),)) + facts.append(_sentence(f"{section_id}.missing.sequence_count", missing.render(), "fact", *missing.checked_sources)) + else: + facts.append(_sentence(f"{section_id}.object", f"分析对象：目标与参考成员合并集合中的 {sequence_count} 条蛋白序列。", "fact")) + if widths: + observations.append(_sentence(f"{section_id}.observation", f"共识别 {count} 个 motif，总位点数为 {total_sites}，宽度范围为 {min(widths)}–{max(widths)}。", "observation", _manifest_source(manifest, "motifs"))) + else: + missing = MissingMetric("Motif 宽度范围", (_manifest_source(manifest, "motifs[].width"),)) + facts.append(_sentence(f"{section_id}.missing", missing.render(), "fact", *missing.checked_sources)) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：这些统计描述短片段的类别、总命中与长度范围；总位点数不能替代逐成员覆盖率。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：源结果未汇总同组排列一致性或异常成员，不能从总数推断特定小组功能分化。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "Motif 排列图", "motif_plot_png") + elif stage == "domain": + total = metrics.get("proteins_total") + hit = metrics.get("proteins_with_domains") + top_domains = metrics.get("top_domains") if isinstance(metrics.get("top_domains"), Sequence) else [] + top_text = "、".join(f"{item[0]} {item[1]} 次" for item in top_domains if isinstance(item, Sequence) and len(item) >= 2) + facts.append(_sentence(f"{section_id}.object", f"分析对象：目标与参考成员合并集合中的 {total} 条蛋白。", "fact")) + observations.append(_sentence(f"{section_id}.observation", f"Domain 覆盖为 {hit}/{total} 条蛋白；主要命中为 {top_text}。", "observation", _manifest_source(manifest, "proteins_with_domains"), _manifest_source(manifest, "top_domains"))) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：全部蛋白均有 Domain 命中，SBP 的普遍命中支持该合并集合保留家族典型区域；Ig_SPL7 只出现在部分命中记录中。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：命中次数不等于独立蛋白数，Ig_SPL7 的 12 次命中不能直接解释为 12 个功能不同成员。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "Domain 图", "domain_plot_png") + elif stage == "structure": + total = metrics.get("genes_total") + covered = metrics.get("genes_with_structure") + outliers = metrics.get("long_gene_outliers") if isinstance(metrics.get("long_gene_outliers"), Sequence) else [] + outlier = next((item for item in outliers if isinstance(item, Mapping)), None) + facts.append(_sentence(f"{section_id}.object", f"分析对象：目标与参考合并集合的 {total} 个成员；结构注释实际匹配 {covered} 个目标成员。", "fact")) + detail = f"；长基因异常 1 个，为 {outlier.get('gene_id')}（{outlier.get('length_bp')} bp，{outlier.get('cds_count')} 个 CDS）" if outlier else "" + observations.append(_sentence(f"{section_id}.observation", f"结构覆盖为 {covered}/{total}{detail}。", "observation", _manifest_source(manifest, "genes_with_structure"), _manifest_source(manifest, "long_gene_outliers"))) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：24/40 的覆盖差异主要反映参考成员未纳入当前目标物种 GFF3；超长基因需要回到原始注释核对。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：未匹配的参考成员不能作为目标物种结构缺失；单个长基因也不能单独证明功能变化。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "结构明细", "structure_tsv") + elif stage == "combo": + parameters = manifest.get("parameters") if isinstance(manifest.get("parameters"), Mapping) else {} + panels = [str(value).capitalize() for value in parameters.get("panels", [])] + panel_text = "、".join(panels) if panels else "Motif、Domain、Structure" + facts.append(_sentence(f"{section_id}.object", "分析对象：目标与参考成员合并集合的关系树及其多类结构证据。", "fact")) + observations.append(_sentence(f"{section_id}.observation", f"组合图已将关系树与 {panel_text} 三类面板汇总到同一图中。", "observation", _manifest_source(manifest, "parameters.panels"))) + facts.append(_sentence(f"{section_id}.missing", "组合证据一致性数值：源结果未提供此指标（已检查：Combo Manifest、combo_plot_png、combo_plot_pdf）。", "fact")) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：该图适合逐行核对多类证据，但当前命名源没有可机器追溯的一致性评分。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：没有逐成员一致性表时，不指定稳定小组、异常成员或候选基因。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "组合图", "combo_plot_png") + elif stage == "promoter": + genes = metrics.get("genes_with_elements") + total = metrics.get("total_elements") + unique = metrics.get("unique_elements") + counts_path, counts_role = _artifact_for_roles(final_dir, manifest, "element_counts") + counts = _element_counts(counts_path) if counts_path else [] + category_path, category_role = _artifact_for_roles(final_dir, manifest, "position_distribution_summary", "promoter_category_summary") + categories = _tabular_rows(category_path) if category_path else [] + enrichment_path, enrichment_role = _artifact_for_roles(final_dir, manifest, "enrichment_analysis") + enrichment_rows = _tabular_rows(enrichment_path) if enrichment_path else [] + co_path, co_role = _artifact_for_roles(final_dir, manifest, "cooccurrence_analysis") + co_rows = _tabular_rows(co_path) if co_path else [] + complexity_path, complexity_role = _artifact_for_roles(final_dir, manifest, "complexity_analysis") + complexity_rows = _tabular_rows(complexity_path) if complexity_path else [] + facts.append(_sentence(f"{section_id}.object", f"分析对象：检测到顺式元件的 {genes} 个目标基因启动子。", "fact")) + top_elements = "、".join(f"{name} {count} 个" for name, count in counts[:3]) or "源结果未提供主要元件排行" + top_category = categories[0] if categories else {} + category_name = top_category.get("big7") or top_category.get("Category") + category_count = top_category.get("element_instances") or top_category.get("Count") + category_text = "" + if category_name and category_count and total: + category_text = f"；主要类别为 {category_name} {category_count} 个（{int(category_count) / int(total) * 100:.1f}%）" + enrichment_top = enrichment_rows[0] if enrichment_rows else {} + enrichment_text = "" + if enrichment_top: + enrichment_text = f"；最高列示富集比为小组 {enrichment_top.get('Subfamily')} 的 {enrichment_top.get('Element')}（{enrichment_top.get('EnrichmentRatio')}）" + co_top = co_rows[0] if co_rows else {} + co_text = "" + if co_top: + co_text = f"；最常见共现为 {co_top.get('Element1')}—{co_top.get('Element2')}（{co_top.get('Co-occurrenceCount')} 个基因）" + complex_top = complexity_rows[0] if complexity_rows else {} + complex_text = "" + if complex_top: + complex_text = f"；复杂度最高的候选为 {complex_top.get('Gene')}（得分 {complex_top.get('ComplexityScore')}）" + observations.append(_sentence(f"{section_id}.observation", f"{genes} 个目标基因共记录 {total} 个元件、{unique} 种类型；主要元件为 {top_elements}{category_text}{enrichment_text}{co_text}{complex_text}。", "observation", *(f"artifact:{role}" for role in (counts_role, category_role, enrichment_role, co_role, complexity_role) if role))) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：类别比例、共现和复杂度描述的是上游序列预测模式；富集比可用于定位值得复核的小组—元件组合。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：仅 23 个目标基因有元件记录，且序列片段出现、共现或高复杂度都不能证明真实调控。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "启动子汇总图", "viz_Ft.SPL_promoter_v6_combined", "promoter_v6_combined") + elif stage == "tfbs": + promoters = metrics.get("family_promoter_count") + sites = metrics.get("family_site_count") + motif_count = metrics.get("family_motif_count") + background = metrics.get("background_promoter_count") + summary_path, summary_role = _artifact_for_roles(final_dir, manifest, "motif_summary") + rows = _tabular_rows(summary_path) if summary_path else [] + rows.sort(key=lambda row: int(float(row.get("family_sites") or 0)), reverse=True) + top = rows[0] if rows else {} + facts.append(_sentence(f"{section_id}.object", f"分析对象：{promoters} 条目标成员启动子；Motif 模型来自 {metrics.get('motif_database', '声明数据库')}。", "fact")) + top_text = f"；最高频 Motif {top.get('motif_id')} 有 {top.get('family_sites')} 个位点，覆盖 {top.get('family_genes_with_site')} 个基因" if top else "" + observations.append(_sentence(f"{section_id}.observation", f"{promoters} 条目标启动子共得到 {sites} 个预测位点、{motif_count} 个有命中的 Motif{top_text}。", "observation", _manifest_source(manifest, "family_site_count"), *(tuple([f"artifact:{summary_role}"]) if summary_role else ()))) + if int(background or 0) == 0: + facts.append(_sentence(f"{section_id}.no_background", "背景启动子为 0，不能据此进行家族相对背景的富集判断。", "fact", _manifest_source(manifest, "background_promoter_count"))) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：高频位点表示序列扫描中出现较多，只能作为潜在结合线索，不能写成转录因子已调控目标基因。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：Motif 使用最近分类物种投射且缺少背景集合，统计富集和物种特异调控均无法确认。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "Motif 汇总表", "motif_summary") + elif stage == "synteny": + facts.append(_sentence(f"{section_id}.object", f"分析对象：目标物种全基因组 {metrics.get('gene_count')} 个基因，而非仅家族成员。", "fact")) + observations.append(_sentence(f"{section_id}.observation", f"全基因组结果记录 {metrics.get('synteny_blocks')} 个共线性区块、{metrics.get('gene_pairs')} 个基因对，并绘制 {metrics.get('circos_chromosomes_plotted')} 条染色体。", "observation", _manifest_source(manifest, "synteny_blocks"), _manifest_source(manifest, "gene_pairs"))) + facts.append(_sentence(f"{section_id}.missing", "源结果未提供家族专属复制基因对（已检查：Synteny Manifest、collinearity artifact、dup_class_log artifact）。", "fact")) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：这些数字刻画全基因组共线性背景，不能直接归因于 SPL 家族扩张。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：缺少家族成员专属 links 或复制类型表，本节不判断大片段复制或串联复制孰为家族主要来源。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "Circos 图", "circos_png") + elif stage == "karyotype": + species = metrics.get("species") if isinstance(metrics.get("species"), Mapping) else {} + item = next(iter(species.values()), {}) if species else metrics + genes = item.get("gene_count") if isinstance(item, Mapping) else None + chromosomes = item.get("chromosome_count") if isinstance(item, Mapping) else None + facts.append(_sentence(f"{section_id}.object", f"分析对象：目标物种的 {genes} 个家族成员。", "fact")) + observations.append(_sentence(f"{section_id}.observation", f"核型分析将{genes}个家族成员定位到{chromosomes}条染色体。", "observation", _manifest_source(manifest, "species.*.gene_count"), _manifest_source(manifest, "species.*.chromosome_count"))) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：染色体位置图确认了目标成员的基因组分布范围，可与家族专属复制证据联合阅读。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：Manifest 未汇总每条染色体的成员数，不能仅凭 24/8 判断分布均匀性或成簇关系。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "核型图", "Ft_karyotype_png") + elif stage == "interspecies_synteny": + parsed = _parse_interspecies_artifacts(final_dir, manifest) + target = metrics.get("target_family_count") + reference = metrics.get("ref_family_count") + facts.append(_sentence(f"{section_id}.object", f"分析对象：目标物种 {target} 个与参考物种 {reference} 个家族成员，并结合两物种全基因组共线性。", "fact")) + if parsed is None: + missing = MissingMetric( + "种间共线性区块与家族成员配对", + ("artifact:gene_to_species", "artifact:*_collinearity"), + ) + facts.append( + _sentence( + f"{section_id}.missing.collinearity_summary", + missing.render(), + "fact", + *missing.checked_sources, + ) + ) + else: + alignment_blocks, target_ref_pairs, sources = parsed + observations.append( + _sentence( + f"{section_id}.observation.summary", + f"共线性区块数：{alignment_blocks}（全基因组 block {alignment_blocks} 个）；" + f"家族成员目标—参考配对数：{target_ref_pairs}（目标-参考家族 {target_ref_pairs} 对）。", + "observation", + *sources, + ) + ) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：全基因组区块提供跨物种背景，2 个目标—参考家族配对是当前命名产物中可直接追溯的家族关系。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：共线性配对只能支持保留关系线索，不能直接移植参考物种功能或断言一一对应。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "标记版种间共线性图", "synteny_marked_png") + elif stage == "pep": + protein_count = metrics.get("protein_count") + success = metrics.get("tasks_success") + total_tasks = metrics.get("tasks_total") + counts = metrics.get("subcellular_counts") if isinstance(metrics.get("subcellular_counts"), Mapping) else {} + facts.append(_sentence(f"{section_id}.object", f"分析对象：目标与参考成员合并集合中的 {protein_count} 条蛋白。", "fact")) + localization = "" + if counts: + localization = f"；亚细胞定位预测为 Nucleus {counts.get('Nucleus', 0)}、Cytoplasm {counts.get('Cytoplasm', 0)}、Chloroplast {counts.get('Chloroplast', 0)}" + else: + missing = MissingMetric("亚细胞定位分类计数", (_manifest_source(manifest, "subcellular_counts"),)) + facts.append(_sentence(f"{section_id}.missing.localization", missing.render(), "fact", *missing.checked_sources)) + observations.append(_sentence(f"{section_id}.observation", f"蛋白数：{protein_count}；子任务成功数：{success}/{total_tasks}。本阶段覆盖 {protein_count} 个蛋白，子任务成功数为 {success}/{total_tasks}{localization}。", "observation", _manifest_source(manifest, "protein_count"), _manifest_source(manifest, "subcellular_counts"))) + interpretations.append(_sentence(f"{section_id}.interpretation", "解释：核定位预测占多数，与转录因子研究方向相容，但这里只能作为定位线索。", "interpretation")) + limitations.append(_sentence(f"{section_id}.limitation", "限制：理化性质、跨膜与亚细胞定位均为计算预测；当前汇总不能替代逐蛋白实验验证。", "limitation")) + artifact = _key_artifact_sentence(final_dir, manifest, section_id, "亚细胞定位表", "subcellular_data") + else: + artifact = None + + if artifact is not None: + facts.append(artifact) + return ReportSection( + section_id, + stage, + tuple(facts), + tuple(observations), + tuple(interpretations), + tuple(limitations), + tuple(recommendations), + ) + + +def build_report_data( + final_dir: Path | str, + prefix: str, + planned: Sequence[str], + statuses: Mapping[str, str], + manifests: Mapping[str, Mapping[str, Any]], +) -> ReportData: + """Build the only deterministic data source used by both render modes.""" + + final_path = Path(final_dir) + unsupported_stages = tuple(stage for stage in planned if stage not in STAGE_SECTION_IDS) + included = tuple( + stage + for stage in planned + if stage in STAGE_SECTION_IDS + and statuses.get(stage) == "success" + and str((manifests.get(stage) or {}).get("status") or "") == "success" + ) + report_status = "partial" if len(included) != len(planned) else "final" + success_count = len(included) + failed_count = len(planned) - success_count + stage_sections = [ + _stage_section(final_path, prefix, stage, str(statuses.get(stage) or "missing"), manifests.get(stage) or {}) + for stage in planned + if stage in STAGE_SECTION_IDS + ] + stage_sections_by_name = {section.stage: section for section in stage_sections} + summary_texts = [ + f"本次按用户计划检查 {len(planned)} 个阶段，其中 {success_count} 个成功结果纳入正文。", + f"报告状态为{'最终报告' if report_status == 'final' else '部分报告'}，未纳入阶段数为 {failed_count}。", + ] + for stage in ( + "identify", "phylogeny", "reference_classification", "domain", "structure", + "promoter", "karyotype", "pep", + ): + section = stage_sections_by_name.get(stage) + if section and section.observations and len(summary_texts) < 9: + summary_texts.append(section.observations[0].text.replace("。", "；").rstrip("；") + "。") + for stage in ("identify", "phylogeny"): + section = stage_sections_by_name.get(stage) + if section: + for sentence in section.facts: + if sentence.text.startswith("分析对象：") and len(summary_texts) < 5: + summary_texts.append(sentence.text) + while len(summary_texts) < 5: + summary_texts.append("正文只纳入计划内成功阶段，并在各节分别声明实际分析对象。") + summary_texts.append("本摘要只组合下文可追溯的阶段事实，不据此指定候选基因或推断功能因果。") + summary_facts = tuple( + _sentence(f"common.front.summary.{index}", text, "fact") + for index, text in enumerate(summary_texts[:10], start=1) + ) + completion_facts = [ + _sentence( + "common.completeness.counts", + f"本次计划 {len(planned)} 个阶段，纳入解读 {success_count} 个，未纳入 {failed_count} 个。", + "fact", + ) + ] + for stage in planned: + supported = stage in STAGE_SECTION_IDS + completion_facts.append( + _sentence( + f"common.completeness.stage.{stage}", + f"计划阶段 `{stage}`：源状态 `{statuses.get(stage, 'missing')}`；" + f"模板支持：{'是' if supported else '不支持'}；" + f"纳入解读：{'是' if stage in included else '否'}。", + "fact", + ) + ) + sections: list[ReportSection] = [ + ReportSection( + "common.front", + None, + facts=summary_facts, + ), + ReportSection( + "common.route", + None, + facts=( + _sentence("common.route.stages", f"本报告按计划阶段顺序解读：{'、'.join(planned)}。", "fact"), + ), + ), + ReportSection( + "common.completeness", + None, + facts=tuple(completion_facts), + ), + ] + sections.extend(stage_sections) + integrated_observations = [ + section.observations[0] + for section in stage_sections + if section.observations + ][:3] + sections.extend( + [ + ReportSection( + "common.integrated", + None, + facts=tuple( + _sentence( + f"common.integrated.fact.{index}", + sentence.text, + "fact", + *sentence.evidence_ids, + ) + for index, sentence in enumerate(integrated_observations, start=1) + ), + interpretations=( + _sentence( + "common.integrated.boundary", + "综合结论只并列上述已提取事实；当前源结果不足以无偏地指定候选基因，也不据此建立功能因果。", + "interpretation", + ), + ), + ), + ReportSection( + "common.limitations", + None, + limitations=( + _sentence( + "common.limitations.general", + "基因组与注释质量、分析阈值和计算预测边界都会影响结果，缺少实验信息时不能确定具体生物学功能。", + "limitation", + ), + ), + ), + ReportSection( + "common.recommendations", + None, + recommendations=( + _sentence( + "common.recommendations.general", + "优先核对异常成员的序列与注释，并用表达、定位或功能实验验证由至少两类独立证据支持的候选。", + "recommendation", + ), + ), + ), + ReportSection( + "common.checklist", + None, + recommendations=( + _sentence( + "common.checklist.traceability", + "交付前应确认每项结论均能追溯到当前身份的 Manifest 或命名结果文件。", + "recommendation", + ), + ), + ), + ] + ) + return ReportData( + prefix, + tuple(planned), + dict(statuses), + included, + unsupported_stages, + report_status, + tuple(sections), + ) diff --git a/gfa_core/mas/agents/interpretation_evidence.py b/gfa_core/mas/agents/interpretation_evidence.py index fee2edf27..d85442f36 100755 --- a/gfa_core/mas/agents/interpretation_evidence.py +++ b/gfa_core/mas/agents/interpretation_evidence.py @@ -193,14 +193,24 @@ def _read_json(path: Path) -> Dict[str, Any]: def _artifact_path(manifest: Dict[str, Any], artifact_key: str) -> Optional[Path]: artifact = (manifest.get("artifacts") or {}).get(artifact_key) or {} - path_text = artifact.get("full_path") or artifact.get("path") - if path_text: + manifest_path = Path(str(manifest.get("_manifest_path") or "")) + job_dir = manifest_path.parent.parent if manifest_path else None + for path_text in (artifact.get("full_path"), artifact.get("path")): + if not path_text: + continue path = Path(str(path_text)) - if not path.is_absolute() and manifest.get("_manifest_path"): - job_dir = Path(str(manifest["_manifest_path"])).parent.parent - path = job_dir / path - if path.exists(): - return path + candidates = [path] + if job_dir: + if not path.is_absolute(): + candidates.append(job_dir / path) + else: + job_index = next((index for index, part in enumerate(path.parts) if part.startswith("job_")), None) + if job_index is not None: + candidates.append(job_dir.joinpath(*path.parts[job_index + 1 :])) + candidates.append(manifest_path.parent / path.name) + for candidate in candidates: + if candidate.exists(): + return candidate return None diff --git a/gfa_core/mas/core/fast_intent.py b/gfa_core/mas/core/fast_intent.py index fe0600f23..abeea717b 100755 --- a/gfa_core/mas/core/fast_intent.py +++ b/gfa_core/mas/core/fast_intent.py @@ -4,7 +4,7 @@ import re from typing import Optional, Dict, Any, List from gfa_core.mas.stage_catalog import complete_dependencies as catalog_complete_dependencies -from gfa_core.mas.utils.intent_parser import is_gwas_fmt_intent, parse_gene_family_scope_selection, resolve_gwas_intent, resolve_population_intent +from gfa_core.mas.utils.intent_parser import is_gwas_fmt_intent, parse_gene_family_scope_selection, resolve_gene_family_report_intent, resolve_gwas_intent, resolve_population_intent # 分析类型关键词映射（注意：key必须与orchestrator.py中的analysis_type匹配） ANALYSIS_KEYWORDS = { @@ -96,6 +96,12 @@ def fast_intent_recognition(query: str) -> Optional[Dict[str, Any]]: None: 无法识别 """ query_lower = query.lower() + report_intent = resolve_gene_family_report_intent(query) + if report_intent: + return { + "stage": "gene_family_report", + **report_intent, + } population = resolve_population_intent(query) if population: return { diff --git a/gfa_core/mas/stage_capabilities.py b/gfa_core/mas/stage_capabilities.py index e2035a0cd..8349971d5 100755 --- a/gfa_core/mas/stage_capabilities.py +++ b/gfa_core/mas/stage_capabilities.py @@ -812,6 +812,23 @@ CAPABILITIES: Dict[str, StageCapability] = { repair_candidates=["manifest_recovery", "artifact_reresolve"], rerun_policy="auto_after_manifest_recovery", ), + "gene_family_report": StageCapability( + stage="gene_family_report", + cli="apps/cli/run_gene_family_report.py", + env="test4py311", + category="report", + consumes=[], + provides=[ + ArtifactProduct("gene_family_report", "markdown", "{prefix}.gene_family_report.md"), + ArtifactProduct("gene_family_report_evidence", "json", "{prefix}.gene_family_report.evidence.json"), + ArtifactProduct("gene_family_report_model", "json", "{prefix}.gene_family_report.model.json"), + ArtifactProduct("gene_family_report_manifest", "manifest", "{prefix}.gene_family_report.manifest.json"), + ], + scope_modes=["target_only", "target_plus_reference", "all_selected"], + preflight_checks=["at_least_one_successful_planned_manifest"], + postflight_checks=["formal_report_four_files_exist"], + rerun_policy="open_reuses_fresh_generate_rebuilds", + ), } diff --git a/gfa_core/mas/stage_catalog.py b/gfa_core/mas/stage_catalog.py index 9ce115883..487bac47b 100755 --- a/gfa_core/mas/stage_catalog.py +++ b/gfa_core/mas/stage_catalog.py @@ -78,6 +78,7 @@ PIPELINE_STAGES = { "ppi", "kaks", "combo", + "gene_family_report", "rnaseq_align", "rnaseq_quant", "rnaseq_sample_qc", @@ -391,6 +392,7 @@ STAGE_ORDER = [ "rnaseq_fgsea", "rnaseq_candidate_family", "rnaseq_enrich", + "gene_family_report", ] STAGE_INFO = { @@ -441,6 +443,7 @@ STAGE_INFO = { "msa": {"name": "多序列比对", "estimated_time": 10, "cpu": 4}, "enrichment": {"name": "富集分析", "estimated_time": 15, "cpu": 4}, "combo": {"name": "组合图", "estimated_time": 10, "cpu": 4}, + "gene_family_report": {"name": "基因家族结果报告", "estimated_time": 2, "cpu": 1}, "genome_prepare": {"name": "基因组准备", "estimated_time": 10, "cpu": 4}, "ncbi_fetch_sequences": {"name": "NCBI序列下载", "estimated_time": 5, "cpu": 4}, "sequence_extract_by_id": {"name": "按ID提取序列", "estimated_time": 5, "cpu": 4}, @@ -529,6 +532,7 @@ EXECUTABLE_STAGES = { "ppi", "kaks", "combo", + "gene_family_report", "rnaseq_align", "rnaseq_quant", "rnaseq_sample_qc", diff --git a/gfa_core/mas/tools/stage_tools.py b/gfa_core/mas/tools/stage_tools.py index cc76e2f87..b2579bdab 100755 --- a/gfa_core/mas/tools/stage_tools.py +++ b/gfa_core/mas/tools/stage_tools.py @@ -558,9 +558,15 @@ class StageToolExecutor: tfbs_top_motifs=(extra_params or {}).get("tfbsTopMotifs", 30), tfbs_scan_both_strands=bool((extra_params or {}).get("tfbsScanBothStrands", True)), scope_file=(extra_params or {}).get("scopeFile", "") or (extra_params or {}).get("scope_file", ""), + scope_id=param_value("scopeId", "scope_id", default=""), + rnaseq_scope_id=param_value("rnaseqScopeId", "rnaseq_scope_id", default=""), analysis_route=(extra_params or {}).get("analysisRoute", "") or (extra_params or {}).get("analysis_route", ""), member_set_id=(extra_params or {}).get("memberSetId", "") or (extra_params or {}).get("member_set_id", ""), analysis_mode=(extra_params or {}).get("analysisMode", "") or (extra_params or {}).get("analysis_mode", ""), + expected_stages=param_value("expectedStages", "expected_stages", default=[]), + stage_statuses=param_value("stageStatuses", "stage_statuses", default={}), + report_action=param_value("reportAction", "report_action", default="generate"), + report_use_model=bool_param("reportUseModel", "report_use_model", default=True), species_queue=(extra_params or {}).get("speciesQueue", []) or (extra_params or {}).get("species_queue", []), target_species=target_species, interspecies_ref_species=interspecies_ref_species, @@ -1047,6 +1053,22 @@ class StageToolRegistry: "parameters": {"scope_aware": True}, "timeout": 1800 }, + { + "id": "gene_family_report", + "name": "Gene Family Result Report", + "script": "run_gene_family_report.py", + "category": "report", + "description": "Generate or open the formal report for exactly the confirmed gene-family stages", + "required_inputs": [], + "outputs": [ + "gene_family_report", + "gene_family_report_evidence", + "gene_family_report_model", + "gene_family_report_manifest", + ], + "parameters": {"action": "generate", "useModel": True}, + "timeout": 1800 + }, # 15. Genome Prepare { "id": "genome_prepare", diff --git a/gfa_core/mas/utils/intent_parser.py b/gfa_core/mas/utils/intent_parser.py index 48cc0f25a..7524543e8 100755 --- a/gfa_core/mas/utils/intent_parser.py +++ b/gfa_core/mas/utils/intent_parser.py @@ -260,6 +260,35 @@ RNASEQ_STAGES = { "rnaseq_report", } + +def resolve_gene_family_report_intent(query: str) -> Optional[Dict]: + """Resolve only explicit gene-family report generate/open requests.""" + text = str(query or "").lower() + compact = re.sub(r"\s+", "", text) + if any(marker.replace(" ", "") in compact for marker in RNASEQ_MARKERS + ["转录组"]): + return None + action = "" + if any(phrase in compact for phrase in ("打开基因家族结果报告", "打开基因家族报告", "openresultreport")): + action = "open" + elif any(phrase in compact for phrase in ("生成基因家族结果报告", "生成基因家族报告", "生成结果报告", "generateresultreport")): + action = "generate" + if not action: + return None + return { + "gene_family": "", + "species": "", + "stages": ["gene_family_report"], + "primary_stage": "gene_family_report", + "task_type": "report", + "task_kind": "report", + "params": {"geneFamilyReportAction": action, "reportAction": action}, + "confidence": 1.0, + "need_clarify": False, + "clarify_type": None, + "message": None, + "method": "gene_family_report_deterministic", + } + # 物种关键词映射 SPECIES_KEYWORDS = { "Ft": ["ft", "tartary", "苦荞", "荞麦"], @@ -648,6 +677,10 @@ def parse_intent_from_query(query: str) -> Dict: query_lower = query.lower() query_without_space = re.sub(r"\s+", "", query_lower) + report_intent = resolve_gene_family_report_intent(query) + if report_intent: + return report_intent + if any(token.replace(" ", "") in query_without_space for token in ( "wgs全流程", "vcf全流程", "wgs full pipeline", "vcf full pipeline" )): diff --git a/gfa_core/schemas/stages/gene_family_report_contract_v1.md b/gfa_core/schemas/stages/gene_family_report_contract_v1.md new file mode 100644 index 000000000..806bbde11 --- /dev/null +++ b/gfa_core/schemas/stages/gene_family_report_contract_v1.md @@ -0,0 +1,82 @@ +# Gene Family Report Stage Contract v1.0 + +## 阶段信息 + +- **阶段名称**: `gene_family_report` +- **脚本**: `apps/cli/run_gene_family_report.py` +- **职责**: 按用户确认的计划阶段，生成或打开可持久化的基因家族分析结果报告。 + +## 输入契约 + +| 字段 | 类型 | 必需 | 描述 | +|---|---|---|---| +| workspace | directory | 是 | 当前 job 工作目录 | +| prefix | string | 是 | 结果前缀 | +| expected_stages | non-empty stage list | 生成时是 | 用户确认的计划阶段，保持原顺序；不得从目录中已有 Manifest 反推计划 | +| stage_statuses | JSON object | 是 | 每个计划阶段的终态或当前状态 | +| job_id | string | 否 | job 身份 | +| scope_id / analysis_route / member_set_id | string | 否 | 严格证据身份 | +| rnaseq_scope_id | string | 否 | 可选 RNA-seq 证据身份 | +| action | enum | 否 | `generate` 生成；`open` 有现成报告时复用，否则生成 | + +阶段不启动任何上游生物分析。至少一个同身份、同计划范围的成功 Manifest 才能生成报告。 + +`action=generate`，以及 `action=open` 但不存在可复用报告时，`expected_stages` 必须为非空；否则明确拒绝生成。`action=open` 且 Markdown 与报告 Manifest 已存在时可以直接复用，不要求再次提供计划。 + +## 输出契约 + +| artifact key | 类型 | 文件 | +|---|---|---| +| gene_family_report | markdown | `*.gene_family_report.md` | +| gene_family_report_manifest | manifest | `*.gene_family_report.manifest.json` | + +Manifest 记录 `final|partial` 报告状态、`api|fallback_script` 生成模式和当前分析身份。`included_stages` 只包含模板支持且成功的计划阶段；成功但不在模板 13 个阶段中的计划项写入 `unsupported_stages`，在公共完成表标记“不支持”，不生成推断章节，并使报告状态保持 `partial`。API 不可用或编排越界时，`model.reason` 固定为 `api_unavailable_or_boundary_violation`。 + +## 确定性数据接口 + +`build_report_data(final_dir, prefix, planned, statuses, manifests) -> ReportData` 是模板渲染和 API 编排共用的唯一结果模型。每节把脚本确定的内容分为 `facts`、`observations`、`interpretations`、`limitations`、`recommendations`，并汇总为带稳定 `sentence_id`、原文、类型和证据编号的不可变句子。 + +指标优先读取同身份 Manifest 的嵌套 `metrics`。仅当模板所需指标缺失时，集中提取器才读取该指标声明的命名 artifact；不得从文件名、其他阶段或相似字段猜值。真缺失使用 `MissingMetric(label, checked_sources)`，渲染为“源结果未提供此指标（已检查：…）”。声明的数据源存在但解析失败时必须报错，不得静默写成“未解析”、`N/A` 或 `None`。 + +当前真实 schema 映射包括：Identify 的 `member_count` / `identified_members_count` / `final_members`（按此前顺序取首个已提供值），Phylogeny 的 `sequences_count` 与 `msa_length`，PEP 的 `protein_count`、`tasks_success` 与 `tasks_total`。Interspecies Synteny 在没有直接语义指标时只读取 Manifest 声明的 `gene_to_species` 与 `*_collinearity` 命名 artifact：以 `## Alignment` 计共线性区块，并按 TSV 的 `group=target|ref` 名单统计目标—参考家族直接配对。 + +## API 语言编排接口 + +API 输入字段仅限： + +```json +{ + "sections": [ + { + "section_id": "stage.identify", + "sentences": [{"sentence_id": "stage.identify.status", "text": "不可改写原句"}] + } + ], + "allowed_connectors": ["also", "however", "therefore"] +} +``` + +API 输出字段仅限： + +```json +{ + "sections": [ + { + "section_id": "stage.identify", + "paragraphs": [ + {"sentence_ids": ["stage.identify.status"], "connector_ids": []} + ] + } + ] +} +``` + +输出必须保持完整章节集合；每个不可变句子必须在原章节中恰好出现一次；连接语必须来自白名单，数量必须与段内句间位置一致。未知、遗漏或重复 `sentence_id`，未知连接语，额外字段，自由文本字段，章节遗漏、重复或改变，均使整份 API 编排作废并回退 `fallback_script`，不能局部采用。 + +## 报告生成原则 + +1. Manifest、结果表格和结果文件是报告事实的唯一来源。成员数、统计值、比较结果、证据等级、限制、可下结论范围和后续建议均由确定性脚本提取或生成。 +2. 模型 API 默认启用，但只负责排列脚本已生成的 `sentence_id`、组织段落并选择受控 `connector_id`；API 不直接返回报告文本。不得补充、推断或修改任何数值、结果判断、生物学解释、因果关系、候选基因、证据等级、限制或建议。 +3. API 输出必须通过上述严格结构与集合校验；任一边界错误都使整份 API 编排作废，并自动使用同一 `ReportData` 的默认顺序。 +4. API 不可用、超时或校验失败时，不影响报告事实和章节完整性；报告明确标注 `fallback_script`，但仍使用同一模板和同一份确定性结果数据。 +5. 只纳入用户计划中的阶段。源结果确实没有提供某项指标时，明确写“源结果未提供此指标”并指出所检查的来源；不得用“未解析”、`N/A` 或模型猜测代替可提取的数据。 diff --git a/gfa_core/templates/gene_family_report_v1.md b/gfa_core/templates/gene_family_report_v1.md new file mode 100644 index 000000000..effa63973 --- /dev/null +++ b/gfa_core/templates/gene_family_report_v1.md @@ -0,0 +1,170 @@ + +# {{PREFIX}} 基因家族分析结果解读报告 + +- 任务：{{JOB_ID}} +- 分析前缀：{{PREFIX}} +- 生成模式：{{GENERATION_MODE}} +- 生成时间：{{GENERATED_AT}} + +{{FALLBACK_NOTICE}} + +## 一、报告首页 + +### 1.1 项目信息 + +本报告只覆盖用户计划中的成功阶段；不同阶段的目标集合在各节单独标明。 + +### 1.2 研究目的 + +本报告用于确认家族成员，比较目标成员及必要参考成员的关系、结构、调控序列与蛋白预测特征，并界定结果能够支持的结论范围。 + +### 1.3 一页式结论摘要 + +{{SECTION_CONTENT}} + +### 1.4 结论可信度说明 + +成员数和命名表中的统计可直接复核；树、启动子、TFBS、共线性和蛋白定位只提供计算线索，不等同于功能或因果证据。 + + +## 二、分析路线与阅读顺序 + +{{SECTION_CONTENT}} + +### 2.1 统一解读顺序 + +各阶段依次写明分析对象、源结果中的直接观察、谨慎解释、限制，并仅在有助复核时列出少量相对路径。 + +### 2.2 语言边界 + +预测关系统一表述为线索；没有逐成员表或独立证据时，不指定候选基因，不声称功能改变或调控关系已经证实。 + + +## 三、结果总览与完整性检查 + +### 3.1 阶段完成情况 + +{{SECTION_CONTENT}} + +### 3.2 结果完整性概述 + +报告按计划逐阶段选择成功 Manifest；当各阶段使用的成员集合不同，正文明确区分目标成员、目标加参考成员和全基因组背景。 + + +## 四、基因家族鉴定（Identify） (`identify`) + +### 4.3 最终成员数量与命名 + +{{SECTION_CONTENT}} + + +## 五、亲缘关系分析（Phylogeny） (`phylogeny`) + +### 5.5 亲缘关系阶段小结 + +{{SECTION_CONTENT}} + + +## 六、参考物种辅助的小组分类（Reference Classification） (`reference_classification`) + +### 6.2 分类可信度表述 + +{{SECTION_CONTENT}} + + +## 七、短保守片段分析（Motif） (`motif`) + +### 7.2 Motif 数量与出现频率 + +{{SECTION_CONTENT}} + + +## 八、主要功能区域分析（Domain） (`domain`) + +### 8.1 Domain 整体统计 + +{{SECTION_CONTENT}} + + +## 九、基因内部结构分析（Structure） (`structure`) + +### 9.1 结构覆盖与异常结构 + +{{SECTION_CONTENT}} + + +## 十、综合组合图（Combo） (`combo`) + +### 10.1 树 + Motif + Domain + Structure 组合图 + +{{SECTION_CONTENT}} + + +## 十一、启动子元件分析（Promoter） (`promoter`) + +### 11.1 元件总量、类别与高级统计 + +{{SECTION_CONTENT}} + + +## 十二、转录因子结合位点预测（TFBS） (`tfbs`) + +### 12.1 高频结合片段与统计边界 + +{{SECTION_CONTENT}} + + +## 十三、种内共线性分析（Synteny） (`synteny`) + +### 13.1 全基因组共线性背景与家族证据缺口 + +{{SECTION_CONTENT}} + + +## 十四、染色体分布（Karyotype） (`karyotype`) + +### 14.1 染色体位置概述 + +{{SECTION_CONTENT}} + + +## 十五、种间共线性与跨物种关系（Interspecies Synteny） (`interspecies_synteny`) + +### 15.6 Interspecies Synteny 阶段小结 + +{{SECTION_CONTENT}} + + +## 十六、蛋白基本特征分析（PEP） (`pep`) + +### 16.1 蛋白理化性质分布图 + +逐蛋白理化性质保留在命名结果表中；源 Manifest 没有汇总长度、分子量或等电点范围时，正文不从图片反推数值。 + +### 16.4 亚细胞定位与阶段小结 + +{{SECTION_CONTENT}} + + +## 十七、多证据综合分析 + +### 17.5 综合结论 + +{{SECTION_CONTENT}} + + +## 十八、主要限制 + +{{SECTION_CONTENT}} + + +## 十九、后续验证建议 + +{{SECTION_CONTENT}} + + +## 二十、报告交付前检查清单 + +### 20.1 可追溯性与语言检查 + +{{SECTION_CONTENT}} diff --git a/tasks.py b/tasks.py index 6afc4df46..511c1e5c4 100755 --- a/tasks.py +++ b/tasks.py @@ -411,7 +411,7 @@ def validate_single_stage_artifacts(job_id: str, prefix: str, stage: str, result stage_status = get_stage_status_for_identity( job_id=job_id, identity=identity, - strict_identity=bool( + strict_identity=actual_stage != "gene_family_report" and bool( identity.scope_id or identity.analysis_route or identity.member_set_id diff --git a/tests/e2e/verify_gene_family_report_journey.py b/tests/e2e/verify_gene_family_report_journey.py new file mode 100644 index 000000000..a8028fd6d --- /dev/null +++ b/tests/e2e/verify_gene_family_report_journey.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Small real-data journey for the gene-family report module.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +import shutil +import sys + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from apps.cli.run_gene_family_report import main as report_cli_main +from gfa_core.mas.utils.intent_parser import parse_intent_from_query + + +PREFIX = "Real.SPL" +SCOPE_ID = "scope-real-spl" +ANALYSIS_ROUTE = "post_subfamily_confirm" +MEMBER_SET_ID = "members-real-spl" + + +def _sequence_count(path: Path) -> int: + count = sum(line.startswith(">") for line in path.read_text(encoding="utf-8").splitlines()) + if not count: + raise AssertionError("representative FASTA is empty") + return count + + +def _write_identify_manifest(workspace: Path, fasta: Path, count: int) -> None: + final_dir = workspace / "final" + final_dir.mkdir(parents=True, exist_ok=True) + manifest = { + "manifest_version": "1.0", + "stage": "identify", + "status": "success", + "prefix": PREFIX, + "scope_id": SCOPE_ID, + "analysis_route": ANALYSIS_ROUTE, + "member_set_id": MEMBER_SET_ID, + "parameters": { + "scope_id": SCOPE_ID, + "analysis_route": ANALYSIS_ROUTE, + "member_set_id": MEMBER_SET_ID, + }, + "metrics": {"member_count": count, "final_members": count, "total_proteins": count}, + "artifacts": { + "final_members": {"path": str(fasta), "full_path": str(fasta), "type": "protein_fasta"} + }, + } + (final_dir / f"{PREFIX}.identify.manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + +def _run(workspace: Path, action: str): + return report_cli_main( + [ + "--workspace", str(workspace), + "--prefix", PREFIX, + "--job-id", "representative-gene-family-report", + "--expected-stages", "identify", + "--stage-statuses-json", '{"identify":"success"}', + "--scope-id", SCOPE_ID, + "--analysis-route", ANALYSIS_ROUTE, + "--member-set-id", MEMBER_SET_ID, + "--action", action, + "--no-model", + ] + ) + + +def _stage_section(report: str, stage: str) -> str: + heading = re.search(rf"(?m)^## (?!#).*?\(`{re.escape(stage)}`\)", report) + if heading is None: + raise AssertionError(f"missing report chapter for {stage}") + next_heading = re.search(r"(?m)^## (?!#)", report[heading.end():]) + end = heading.end() + next_heading.start() if next_heading else len(report) + return report[heading.start():end] + + +def _subsection(report: str, number: str) -> str: + heading = re.search(rf"(?m)^### {re.escape(number)}(?:\s|$)", report) + if heading is None: + raise AssertionError(f"missing report subsection {number}") + next_heading = re.search(r"(?m)^#{2,3} ", report[heading.end():]) + end = heading.end() + next_heading.start() if next_heading else len(report) + return report[heading.start():end] + + +def _verify_real_job(workspace: Path, prefix: str) -> None: + final_dir = workspace / "final" + report_path = final_dir / f"{prefix}.gene_family_report.md" + manifest_path = final_dir / f"{prefix}.gene_family_report.manifest.json" + report = report_path.read_text(encoding="utf-8") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + expected_stages = [ + "identify", + "phylogeny", + "reference_classification", + "motif", + "domain", + "structure", + "combo", + "promoter", + "tfbs", + "synteny", + "karyotype", + "interspecies_synteny", + "pep", + ] + + assert report.startswith(f"# {prefix} 基因家族分析结果解读报告\n") + for placeholder in ("【填写】", "未解析", "N/A", "None"): + assert placeholder not in report + assert "/home/xzq/" not in report + assert not re.search(r"(?m)^- `[a-zA-Z0-9_.-]+`:\s*", report) + assert manifest["parameters"]["expected_stages"] == expected_stages + assert manifest["metrics"]["included_stages"] == expected_stages + assert manifest["metrics"]["generation_mode"] == "fallback_script" + assert sorted(path.name for path in final_dir.glob(f"{prefix}.gene_family_report*")) == [ + f"{prefix}.gene_family_report.manifest.json", + f"{prefix}.gene_family_report.md", + ] + for stage in expected_stages: + section = _stage_section(report, stage) + for chunk in re.split(r"(?m)(?=^### )", section)[1:]: + assert "\n".join(chunk.splitlines()[1:]).strip(), chunk.splitlines()[0] + assert "分析对象：" in section + assert "限制：" in section + assert report.count("本节解释仅限上述可追溯结果，不据此断言真实功能或因果关系。") <= 1 + summary = _subsection(report, "1.3") + assert 6 <= summary.count("。") <= 10 + assert re.search(r"目标物种.*24.*家族成员", summary) + assert re.search(r"形成\s*24\s*个\s*SPL家族", _subsection(report, "4.3")) + assert re.search( + r"使用\s*40\s*条序列.*比对长度为\s*1449", _stage_section(report, "phylogeny") + ) + motif = _stage_section(report, "motif") + assert re.search(r"识别\s*10\s*个\s*motif", motif) + assert re.search(r"总位点数为\s*105", motif) + assert re.search(r"宽度范围为\s*27–138", motif) + assert re.search(r"将\s*24\s*个家族成员定位到\s*8\s*条染色体", _stage_section(report, "karyotype")) + assert re.search( + r"全基因组\s*block\s*521\s*个.*目标-参考家族.*?2\s*对", + _stage_section(report, "interspecies_synteny"), + ) + assert re.search(r"覆盖\s*40\s*个蛋白，子任务成功数为\s*3/3", _stage_section(report, "pep")) + assert re.search(r"24\s*个目标成员.*8\s*个小组.*高信心\s*14.*中信心\s*7.*低信心\s*3", _subsection(report, "6.2")) + assert re.search(r"40/40.*SBP.*40.*Ig_SPL7.*12", _subsection(report, "8.1")) + assert re.search(r"24/40.*FtPinG0003446200\.01\.T01.*12454", _subsection(report, "9.1")) + assert "Motif、Domain、Structure" in _subsection(report, "10.1") + promoter = _subsection(report, "11.1") + assert re.search(r"23\s*个目标基因.*3699.*TATA-box.*1460", promoter) + assert "CAAT-box—MYB" in promoter and "FtPinG0006337300.01.T01" in promoter + tfbs = _subsection(report, "12.1") + assert re.search(r"24\s*条目标启动子.*3734.*AHYPO_015621.*153", tfbs) + assert re.search(r"背景启动子为\s*0.*不能.*富集", tfbs) + synteny = _subsection(report, "13.1") + assert re.search(r"全基因组.*316.*3267.*8\s*条染色体", synteny) + assert "源结果未提供家族专属复制基因对" in synteny + assert re.search(r"Nucleus\s*37.*Cytoplasm\s*2.*Chloroplast\s*1", _subsection(report, "16.4")) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--mode", choices=("data", "e2e")) + parser.add_argument("--workspace", type=Path) + parser.add_argument("--prefix") + args = parser.parse_args() + + if args.workspace is not None: + if not args.prefix: + parser.error("--prefix is required with --workspace") + _verify_real_job(args.workspace, args.prefix) + print(json.dumps({"status": "passed", "workspace": str(args.workspace)}, ensure_ascii=False)) + return 0 + if args.mode is None: + parser.error("--mode is required when --workspace is not provided") + + output_dir = PROJECT_ROOT / "test-results/gene-family-report" / args.mode + if output_dir.exists(): + shutil.rmtree(output_dir) + workspace = output_dir / "job" + fasta = PROJECT_ROOT / "tests/ft_data/SPL_Ath.fasta" + count = _sequence_count(fasta) + _write_identify_manifest(workspace, fasta, count) + + generated = _run(workspace, "generate") + report = generated.report_path.read_text(encoding="utf-8") + manifest = json.loads(generated.manifest_path.read_text(encoding="utf-8")) + assert generated.report_path.name == f"{PREFIX}.gene_family_report.md" + assert generated.manifest_path.name == f"{PREFIX}.gene_family_report.manifest.json" + assert "生成模式：候补脚本" in report + assert f"最终成员数：{count}" in report + assert "## 基因家族鉴定 (`identify`)" in report + assert "(`motif`)" not in report + assert manifest["scope_id"] == SCOPE_ID + assert manifest["analysis_route"] == ANALYSIS_ROUTE + assert manifest["member_set_id"] == MEMBER_SET_ID + assert manifest["metrics"]["included_stages"] == ["identify"] + assert set(manifest["artifacts"]) == {"gene_family_report", "gene_family_report_manifest"} + + reopened = _run(workspace, "open") + assert reopened.reused is True + intent = parse_intent_from_query("打开基因家族结果报告") + assert intent["stages"] == ["gene_family_report"] + assert intent["params"]["reportAction"] == "open" + + print(json.dumps({ + "status": "passed", + "mode": args.mode, + "sequence_count": count, + "report": str(generated.report_path), + }, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/test_gene_family_report.py b/tests/unit/test_gene_family_report.py new file mode 100644 index 000000000..45a996914 --- /dev/null +++ b/tests/unit/test_gene_family_report.py @@ -0,0 +1,844 @@ +from __future__ import annotations + +import json +from pathlib import Path +import re + +import pytest + +import gfa_core.gene_family_report as report_module +from gfa_core.gene_family_report import ( + NoSuccessfulGeneFamilyEvidence, + generate_gene_family_report, +) + + +def _write_manifest( + workspace: Path, + *, + prefix: str, + stage: str, + status: str = "success", + scope_id: str = "scope-a", + analysis_route: str = "post_subfamily_confirm", + member_set_id: str = "members-a", + metric_value: int = 1, +) -> Path: + final_dir = workspace / "final" + final_dir.mkdir(parents=True, exist_ok=True) + token = f"{scope_id}.{analysis_route}.{member_set_id}" + table_path = final_dir / f"{prefix}.{token}.{stage}.tsv" + table_path.write_text(f"gene\tvalue\n{scope_id}_gene\t{metric_value}\n", encoding="utf-8") + manifest_path = final_dir / f"{prefix}.{token}.{stage}.manifest.json" + manifest = { + "manifest_version": "1.0", + "stage": stage, + "status": status, + "prefix": prefix, + "scope_id": scope_id, + "analysis_route": analysis_route, + "member_set_id": member_set_id, + "parameters": { + "scope_id": scope_id, + "analysis_route": analysis_route, + "member_set_id": member_set_id, + }, + "metrics": {"member_count": metric_value}, + "artifacts": { + "result_table": { + "path": f"final/{table_path.name}", + "full_path": str(table_path), + "type": "table", + } + }, + } + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + return manifest_path + + +def _generate( + workspace: Path, + *, + expected_stages: list[str], + stage_statuses: dict[str, str], + use_model: bool = False, + action: str = "generate", +): + return generate_gene_family_report( + workspace, + "Ft.SPL", + expected_stages=expected_stages, + stage_statuses=stage_statuses, + scope_id="scope-a", + analysis_route="post_subfamily_confirm", + member_set_id="members-a", + use_model=use_model, + action=action, + job_id="job-report", + ) + + +def _stage_section(markdown: str, stage: str) -> str: + heading = re.search(rf"(?m)^## (?!#).*?\(`{re.escape(stage)}`\)", markdown) + if heading is None: + raise AssertionError(f"missing report chapter for {stage}") + next_heading = re.search(r"(?m)^## (?!#)", markdown[heading.end():]) + end = heading.end() + next_heading.start() if next_heading else len(markdown) + return markdown[heading.start():end] + + +def _subsection(markdown: str, number: str) -> str: + heading = re.search(rf"(?m)^### {re.escape(number)}(?:\s|$)", markdown) + if heading is None: + raise AssertionError(f"missing report subsection {number}") + next_heading = re.search(r"(?m)^#{2,3} ", markdown[heading.end():]) + end = heading.end() + next_heading.start() if next_heading else len(markdown) + return markdown[heading.start():end] + + +def _valid_api_arrangement(payload: dict, *, reverse: bool = False) -> dict: + sections = [] + for section in payload["sections"]: + sentence_ids = [sentence["sentence_id"] for sentence in section["sentences"]] + if reverse: + sentence_ids.reverse() + sections.append( + { + "section_id": section["section_id"], + "paragraphs": [ + { + "sentence_ids": sentence_ids, + "connector_ids": ["also"] * max(0, len(sentence_ids) - 1), + } + ], + } + ) + return {"sections": sections} + + +def test_report_uses_only_planned_stages_and_current_identity(tmp_path: Path) -> None: + workspace = tmp_path / "job" + _write_manifest(workspace, prefix="Ft.SPL", stage="identify", metric_value=3) + other = _write_manifest( + workspace, + prefix="Ft.SPL", + stage="identify", + scope_id="scope-b", + analysis_route="target_only", + member_set_id="members-b", + metric_value=99, + ) + + result = _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "success"}, + ) + + report_text = result.report_path.read_text(encoding="utf-8") + assert "最终成员数：3" in report_text + assert other.name not in report_text + generated = {path.name for path in (workspace / "final").glob("Ft.SPL.gene_family_report.*")} + assert generated == { + "Ft.SPL.gene_family_report.md", + "Ft.SPL.gene_family_report.manifest.json", + } + + +def test_report_renders_template_subsections_for_only_planned_stages(tmp_path: Path) -> None: + workspace = tmp_path / "job" + for stage in ("identify", "motif", "pep"): + _write_manifest(workspace, prefix="Ft.SPL", stage=stage) + + result = _generate( + workspace, + expected_stages=["identify", "motif", "pep"], + stage_statuses={"identify": "success", "motif": "success", "pep": "success"}, + ) + + markdown = result.report_path.read_text(encoding="utf-8") + expected_subsections = { + "identify": "### 4.3 最终成员数量与命名", + "motif": "### 7.2 Motif 数量与出现频率", + "pep": "### 16.1 蛋白理化性质分布图", + } + for stage, subsection in expected_subsections.items(): + assert subsection in _stage_section(markdown, stage) + assert "(`synteny`)" not in markdown + assert "【填写】" not in markdown + + +def test_report_renders_nested_motif_metrics_without_placeholder_values(tmp_path: Path) -> None: + workspace = tmp_path / "job" + manifest_path = _write_manifest(workspace, prefix="Ft.SPL", stage="motif") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["metrics"] = { + "motif_count": 10, + "total_sites": 105, + "motifs": [ + {"width": width} + for width in (78, 76, 80, 113, 57, 29, 52, 138, 27, 29) + ], + } + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + + result = _generate( + workspace, + expected_stages=["motif"], + stage_statuses={"motif": "success"}, + ) + + motif_section = _stage_section(result.report_path.read_text(encoding="utf-8"), "motif") + assert re.search(r"识别\s*10\s*个\s*motif", motif_section) + assert re.search(r"总位点数为\s*105", motif_section) + assert re.search(r"宽度范围为\s*27–138", motif_section) + assert "未解析" not in motif_section + assert "N/A" not in motif_section + assert "None" not in motif_section + + +def test_identify_uses_real_manifest_member_count_aliases(tmp_path: Path) -> None: + workspace = tmp_path / "job" + manifest_path = _write_manifest(workspace, prefix="Ft.SPL", stage="identify") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["metrics"] = {"identified_members_count": 24, "final_members": 24} + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + + result = _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "success"}, + ) + + section = _stage_section(result.report_path.read_text(encoding="utf-8"), "identify") + assert "最终成员数：24" in section + assert re.search(r"形成\s*24\s*个\s*SPL家族", section) + assert "源结果未提供此指标" not in section + + +def test_phylogeny_renders_real_manifest_schema_with_explicit_labels(tmp_path: Path) -> None: + workspace = tmp_path / "job" + manifest_path = _write_manifest(workspace, prefix="Ft.SPL", stage="phylogeny") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["metrics"] = {"sequences_count": 40, "msa_length": 1449} + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + + result = _generate( + workspace, + expected_stages=["phylogeny"], + stage_statuses={"phylogeny": "success"}, + ) + + section = _stage_section(result.report_path.read_text(encoding="utf-8"), "phylogeny") + assert "序列数：40" in section + assert "多序列比对长度：1449" in section + assert re.search(r"使用\s*40\s*条序列.*比对长度为\s*1449", section) + + +def test_interspecies_synteny_reads_only_named_real_schema_artifacts(tmp_path: Path) -> None: + workspace = tmp_path / "job" + manifest_path = _write_manifest(workspace, prefix="Ft.SPL", stage="interspecies_synteny") + final_dir = workspace / "final" + gene_to_species = final_dir / "Ft.SPL.gene_to_species.tsv" + gene_to_species.write_text( + "gene_id\tspecies\tgroup\n" + "FtGene1.01.T01\tFt\ttarget\n" + "FtGene2.01.T01\tFt\ttarget\n" + "AT1G00010\tAT\tref\n" + "AT1G00020\tAT\tref\n", + encoding="utf-8", + ) + collinearity = final_dir / "Ft.At.collinearity" + lines = [] + for index in range(521): + lines.append(f"## Alignment {index}: score=1 e_value=1e-10 N=1 Ft&At plus") + if index == 0: + lines.append(" 0- 0:\tAT1G00010.1\tFtGene1.01.T01\t1e-20") + elif index == 1: + lines.append(" 1- 0:\tAT1G00020.1\tFtGene2.01.T01\t1e-20") + collinearity.write_text("\n".join(lines) + "\n", encoding="utf-8") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["metrics"] = {"pair_success_count": 1, "pair_total_count": 1} + manifest["artifacts"] = { + "gene_to_species": {"path": f"final/{gene_to_species.name}", "type": "tsv"}, + "Ft.At_collinearity": {"path": f"final/{collinearity.name}", "type": "data"}, + } + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + + result = _generate( + workspace, + expected_stages=["interspecies_synteny"], + stage_statuses={"interspecies_synteny": "success"}, + ) + + section = _stage_section(result.report_path.read_text(encoding="utf-8"), "interspecies_synteny") + assert "共线性区块数：521" in section + assert "家族成员目标—参考配对数：2" in section + assert re.search(r"全基因组\s*block\s*521\s*个.*目标-参考家族.*?2\s*对", section) + + +def test_pep_renders_real_manifest_task_summary_with_explicit_labels(tmp_path: Path) -> None: + workspace = tmp_path / "job" + manifest_path = _write_manifest(workspace, prefix="Ft.SPL", stage="pep") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["metrics"] = {"protein_count": 40, "tasks_success": 3, "tasks_total": 3} + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + + result = _generate( + workspace, + expected_stages=["pep"], + stage_statuses={"pep": "success"}, + ) + + section = _stage_section(result.report_path.read_text(encoding="utf-8"), "pep") + assert "蛋白数：40" in section + assert "子任务成功数：3/3" in section + assert re.search(r"覆盖\s*40\s*个蛋白，子任务成功数为\s*3/3", section) + + +def test_report_has_no_empty_third_level_subsections_or_repeated_generic_interpretation( + tmp_path: Path, +) -> None: + workspace = tmp_path / "job" + for stage in ("identify", "phylogeny", "motif"): + manifest_path = _write_manifest(workspace, prefix="Ft.SPL", stage=stage) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if stage == "phylogeny": + manifest["metrics"] = {"sequences_count": 40, "msa_length": 1449} + elif stage == "motif": + manifest["metrics"] = { + "sequence_count": 40, + "motif_count": 10, + "total_sites": 105, + "motifs": [{"width": 27}, {"width": 138}], + } + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + + result = _generate( + workspace, + expected_stages=["identify", "phylogeny", "motif"], + stage_statuses={stage: "success" for stage in ("identify", "phylogeny", "motif")}, + ) + + markdown = result.report_path.read_text(encoding="utf-8") + for stage in ("identify", "phylogeny", "motif"): + section = _stage_section(markdown, stage) + chunks = re.split(r"(?m)(?=^### )", section)[1:] + assert chunks + for chunk in chunks: + body = "\n".join(chunk.splitlines()[1:]).strip() + assert body, f"empty subsection: {chunk.splitlines()[0]}" + assert markdown.count("本节解释仅限上述可追溯结果，不据此断言真实功能或因果关系。") <= 1 + assert markdown.count("结果范围受当前输入、参数和源文件完整性限制。") <= 1 + + +def test_front_summary_contains_project_numbers_in_six_to_ten_sentences(tmp_path: Path) -> None: + workspace = tmp_path / "job" + identify_path = _write_manifest(workspace, prefix="Ft.SPL", stage="identify") + identify = json.loads(identify_path.read_text(encoding="utf-8")) + identify["metrics"] = {"identified_members_count": 24, "final_members": 24} + identify_path.write_text(json.dumps(identify, ensure_ascii=False), encoding="utf-8") + phylogeny_path = _write_manifest(workspace, prefix="Ft.SPL", stage="phylogeny") + phylogeny = json.loads(phylogeny_path.read_text(encoding="utf-8")) + phylogeny["metrics"] = {"sequences_count": 40, "msa_length": 1449} + phylogeny_path.write_text(json.dumps(phylogeny, ensure_ascii=False), encoding="utf-8") + + result = _generate( + workspace, + expected_stages=["identify", "phylogeny"], + stage_statuses={"identify": "success", "phylogeny": "success"}, + ) + + summary = _subsection(result.report_path.read_text(encoding="utf-8"), "1.3") + assert re.search(r"目标物种.*24.*家族成员", summary) + assert re.search(r"目标.*参考.*40.*1449", summary) + assert 6 <= summary.count("。") <= 10 + + +def test_stage_specific_key_facts_land_in_their_semantic_subsections(tmp_path: Path) -> None: + workspace = tmp_path / "job" + final_dir = workspace / "final" + stages = [ + "reference_classification", "domain", "structure", "combo", + "promoter", "tfbs", "synteny", + ] + manifests = {stage: _write_manifest(workspace, prefix="Ft.SPL", stage=stage) for stage in stages} + + assignment = final_dir / "Ft.SPL.reference_classification.assignment.tsv" + assignment.write_text( + "target_gene\tassigned_subfamily\tconfidence\n" + + "\n".join( + f"FtGene{i}\t{'I' if i < 2 else 'II'}\t{confidence}" + for i, confidence in enumerate(("high", "high", "medium", "low"), start=1) + ) + + "\n", + encoding="utf-8", + ) + reference = json.loads(manifests["reference_classification"].read_text(encoding="utf-8")) + reference["metrics"] = {"target_gene_count": 4, "subfamily_rows": 6} + reference["artifacts"] = {"assignment_table": {"path": f"final/{assignment.name}", "type": "tsv"}} + manifests["reference_classification"].write_text(json.dumps(reference), encoding="utf-8") + + domain = json.loads(manifests["domain"].read_text(encoding="utf-8")) + domain["metrics"] = { + "proteins_total": 40, "proteins_with_domains": 40, "domain_coverage_rate": 1.0, + "top_domains": [["SBP", 40], ["Ig_SPL7", 12]], + } + manifests["domain"].write_text(json.dumps(domain), encoding="utf-8") + + structure = json.loads(manifests["structure"].read_text(encoding="utf-8")) + structure["metrics"] = { + "genes_total": 40, "genes_with_structure": 24, "structure_coverage_rate": 0.6, + "long_gene_outlier_count": 1, + "long_gene_outliers": [{"gene_id": "FtLong1", "length_bp": 12454, "cds_count": 10}], + } + manifests["structure"].write_text(json.dumps(structure), encoding="utf-8") + + combo = json.loads(manifests["combo"].read_text(encoding="utf-8")) + combo["metrics"] = None + combo["parameters"] = {"panels": ["motif", "domain", "structure"]} + combo["artifacts"] = { + "combo_plot_png": {"path": "final/Ft.SPL.combo.png", "type": "image"}, + "combo_plot_pdf": {"path": "final/Ft.SPL.combo.pdf", "type": "image"}, + } + manifests["combo"].write_text(json.dumps(combo), encoding="utf-8") + + element_counts = final_dir / "Ft.SPL.element_counts.txt" + element_counts.write_text("1460\tTATA-box\n852\tCAAT-box\n72\tMYB\n", encoding="utf-8") + category_summary = final_dir / "Ft.SPL.promoter_category_summary.tsv" + category_summary.write_text("big7\telement_instances\nCore promoter\t2600\nLight response\t188\n", encoding="utf-8") + enrichment = final_dir / "Ft.SPL.enrichment_analysis.txt" + enrichment.write_text( + "Subfamily\tElement\tCount\tSubfamilyTotal\tGlobalCount\tEnrichmentRatio\nI\tAACA_motif\t1\t222\t1\t16.66\n", + encoding="utf-8", + ) + cooccurrence = final_dir / "Ft.SPL.cooccurrence_analysis.txt" + cooccurrence.write_text( + "Element1\tElement2\tCo-occurrenceCount\tGenes\nCAAT-box\tMYB\t23\tFtGene1,FtGene2\n", + encoding="utf-8", + ) + complexity = final_dir / "Ft.SPL.complexity_analysis.txt" + complexity.write_text( + "Gene\tTotalElements\tUniqueElements\tComplexityScore\tComplexityClass\nFtComplex1\t278\t33\t361.44\tVery High\n", + encoding="utf-8", + ) + promoter = json.loads(manifests["promoter"].read_text(encoding="utf-8")) + promoter["metrics"] = {"total_elements": 3699, "unique_elements": 87, "genes_with_elements": 23} + promoter["artifacts"] = { + "element_counts": {"path": f"final/{element_counts.name}"}, + "viz_promoter_position_distribution_summary": {"path": f"final/{category_summary.name}"}, + "enrichment_analysis": {"path": f"final/{enrichment.name}"}, + "cooccurrence_analysis": {"path": f"final/{cooccurrence.name}"}, + "complexity_analysis": {"path": f"final/{complexity.name}"}, + } + manifests["promoter"].write_text(json.dumps(promoter), encoding="utf-8") + + motif_summary = final_dir / "Ft.SPL.tfbs.motif_summary.tsv" + motif_summary.write_text( + "motif_id\tmotif_alt_id\tfamily_sites\tfamily_genes_with_site\tbackground_sites\tbackground_genes_with_site\n" + "MOTIF_A\t\t153\t18\t0\t0\n", + encoding="utf-8", + ) + tfbs = json.loads(manifests["tfbs"].read_text(encoding="utf-8")) + tfbs["metrics"] = { + "family_promoter_count": 24, "background_promoter_count": 0, + "family_site_count": 3734, "family_motif_count": 230, "motif_database": "PlantTFDB", + } + tfbs["artifacts"] = {"motif_summary": {"path": f"final/{motif_summary.name}"}} + manifests["tfbs"].write_text(json.dumps(tfbs), encoding="utf-8") + + synteny = json.loads(manifests["synteny"].read_text(encoding="utf-8")) + synteny["metrics"] = {"gene_count": 31679, "synteny_blocks": 316, "gene_pairs": 3267, "circos_chromosomes_plotted": 8} + manifests["synteny"].write_text(json.dumps(synteny), encoding="utf-8") + + result = _generate( + workspace, + expected_stages=stages, + stage_statuses={stage: "success" for stage in stages}, + ) + markdown = result.report_path.read_text(encoding="utf-8") + assert re.search(r"4\s*个目标成员.*2\s*个小组.*高信心\s*2", _subsection(markdown, "6.2")) + assert re.search(r"40/40.*SBP.*40.*Ig_SPL7.*12", _subsection(markdown, "8.1")) + assert re.search(r"24/40.*FtLong1.*12454", _subsection(markdown, "9.1")) + assert "Motif、Domain、Structure" in _subsection(markdown, "10.1") + promoter_text = _subsection(markdown, "11.1") + assert re.search(r"23\s*个目标基因.*3699.*TATA-box.*1460", promoter_text) + assert "FtComplex1" in promoter_text and "CAAT-box—MYB" in promoter_text + assert re.search(r"24\s*条目标启动子.*3734.*MOTIF_A.*153", _subsection(markdown, "12.1")) + assert re.search(r"背景启动子为\s*0.*不能.*富集", _subsection(markdown, "12.1")) + assert re.search(r"全基因组.*316.*3267.*8\s*条染色体", _subsection(markdown, "13.1")) + assert "源结果未提供家族专属复制基因对" in _subsection(markdown, "13.1") + + +def test_all_planned_success_is_final(tmp_path: Path) -> None: + workspace = tmp_path / "job" + _write_manifest(workspace, prefix="Ft.SPL", stage="identify") + _write_manifest(workspace, prefix="Ft.SPL", stage="motif") + + result = _generate( + workspace, + expected_stages=["identify", "motif"], + stage_statuses={"identify": "success", "motif": "success"}, + ) + + manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) + assert result.report_status == "final" + assert manifest["metrics"]["report_status"] == "final" + assert manifest["metrics"]["included_stages"] == ["identify", "motif"] + + +def test_unsupported_successful_stage_is_visible_but_not_included_or_final(tmp_path: Path) -> None: + workspace = tmp_path / "job" + _write_manifest(workspace, prefix="Ft.SPL", stage="identify") + _write_manifest(workspace, prefix="Ft.SPL", stage="ppi") + + result = _generate( + workspace, + expected_stages=["identify", "ppi"], + stage_statuses={"identify": "success", "ppi": "success"}, + ) + + markdown = result.report_path.read_text(encoding="utf-8") + manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) + assert result.report_status == "partial" + assert result.included_stages == ("identify",) + assert "ppi" in markdown + assert "不支持" in markdown + assert "(`ppi`)" not in markdown + assert manifest["metrics"]["included_stages"] == ["identify"] + assert manifest["metrics"]["unsupported_stages"] == ["ppi"] + + +@pytest.mark.parametrize("expected_stages", [None, []]) +def test_generate_rejects_missing_plan_instead_of_inferring_historical_manifests( + tmp_path: Path, expected_stages: list[str] | None +) -> None: + workspace = tmp_path / "job" + _write_manifest(workspace, prefix="Ft.SPL", stage="identify") + + with pytest.raises(ValueError, match="expected_stages"): + generate_gene_family_report( + workspace, + "Ft.SPL", + expected_stages=expected_stages, + stage_statuses={"identify": "success"}, + scope_id="scope-a", + analysis_route="post_subfamily_confirm", + member_set_id="members-a", + use_model=False, + action="generate", + ) + + assert not list((workspace / "final").glob("*.gene_family_report.*")) + + +def test_report_selects_key_facts_without_dumping_manifest_or_stale_paths(tmp_path: Path) -> None: + workspace = tmp_path / "job" + manifest_path = _write_manifest(workspace, prefix="Ft.SPL", stage="identify", metric_value=3) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["metrics"].update({"total_proteins": 31679, "blast_enabled": True}) + manifest["artifacts"]["result_table"]["path"] = "/home/xzq/old/job/final/Ft.SPL.identify.tsv" + manifest["artifacts"]["result_table"]["full_path"] = "/home/xzq/old/job/final/Ft.SPL.identify.tsv" + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + + result = _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "success"}, + ) + + markdown = result.report_path.read_text(encoding="utf-8") + assert "最终成员数：3" in _subsection(markdown, "4.3") + assert not re.search(r"(?m)^- `(?:member_count|total_proteins|blast_enabled)\s*`:", markdown) + assert "/home/xzq/" not in markdown + + +def test_fallback_uses_nested_karyotype_metrics(tmp_path: Path) -> None: + workspace = tmp_path / "job" + manifest_path = _write_manifest(workspace, prefix="Ft.SPL", stage="karyotype") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["metrics"] = { + "species": { + "Ft": { + "gene_count": 24, + "chromosome_count": 8, + "chromosomes": [f"Ft{i}" for i in range(1, 9)], + "tandem_count": 0, + } + } + } + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + + result = _generate( + workspace, + expected_stages=["karyotype"], + stage_statuses={"karyotype": "success"}, + ) + + markdown = result.report_path.read_text(encoding="utf-8") + assert "24个家族成员定位到8条染色体" in markdown + assert "核型分析将N/A" not in markdown + + +def test_artifact_path_remaps_remote_job_to_current_mount(tmp_path: Path) -> None: + from gfa_core.mas.agents.interpretation_evidence import _artifact_path + + final_dir = tmp_path / "job" / "final" + final_dir.mkdir(parents=True) + manifest_path = final_dir / "Ft.SPL.stage.manifest.json" + local_artifact = final_dir / "Ft.SPL.result.tsv" + local_artifact.write_text("gene\tvalue\nFtSPL1\t1\n", encoding="utf-8") + manifest = { + "_manifest_path": str(manifest_path), + "artifacts": { + "result": { + "full_path": "/home/xzq/old/PlantAI-dev/workspace/job-old/final/Ft.SPL.result.tsv", + "path": "/home/xzq/old/PlantAI-dev/workspace/job-old/final/Ft.SPL.result.tsv", + } + }, + } + + assert _artifact_path(manifest, "result") == local_artifact + + +def test_failed_or_missing_planned_stage_is_partial(tmp_path: Path) -> None: + workspace = tmp_path / "job" + _write_manifest(workspace, prefix="Ft.SPL", stage="identify") + _write_manifest(workspace, prefix="Ft.SPL", stage="motif", status="failed") + + result = _generate( + workspace, + expected_stages=["identify", "motif"], + stage_statuses={"identify": "success", "motif": "failed"}, + ) + + markdown = result.report_path.read_text(encoding="utf-8") + manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) + assert result.report_status == "partial" + assert "motif" in markdown.lower() + assert "failed" in markdown.lower() + assert manifest["metrics"]["failed_or_missing_stages"] == ["motif"] + + +def test_model_failure_creates_labeled_fallback_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workspace = tmp_path / "job" + _write_manifest(workspace, prefix="Ft.SPL", stage="identify") + + def fail_model(_evidence: dict, _stages: list[str]): + raise TimeoutError("provider timeout") + + monkeypatch.setattr(report_module, "_generate_api_narrative", fail_model) + result = _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "success"}, + use_model=True, + ) + + markdown = result.report_path.read_text(encoding="utf-8") + manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) + assert result.generation_mode == "fallback_script" + assert "生成模式：候补脚本" in markdown + assert manifest["metrics"]["generation_mode"] == "fallback_script" + assert manifest["model"]["reason"] == "api_unavailable_or_boundary_violation" + assert "{'action':" not in markdown + + +def test_valid_model_sections_create_api_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workspace = tmp_path / "job" + _write_manifest(workspace, prefix="Ft.SPL", stage="identify") + + seen_payload: dict = {} + + def valid_model(payload: dict, _stages: list[str]): + seen_payload.update(payload) + return ( + _valid_api_arrangement(payload, reverse=True), + {"status": "generated", "provider": "test", "model": "test-model"}, + ) + + monkeypatch.setattr(report_module, "_generate_api_narrative", valid_model) + result = _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "success"}, + use_model=True, + ) + + markdown = result.report_path.read_text(encoding="utf-8") + manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) + assert result.generation_mode == "api" + assert set(seen_payload) == {"sections", "allowed_connectors"} + assert all(set(section) == {"section_id", "sentences"} for section in seen_payload["sections"]) + assert all( + set(sentence) == {"sentence_id", "text"} + for section in seen_payload["sections"] + for sentence in section["sentences"] + ) + assert "最终成员数：1" in markdown + assert manifest["metrics"]["generation_mode"] == "api" + assert "【填写】" not in markdown + + +def test_unknown_model_sentence_id_falls_back_for_the_whole_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workspace = tmp_path / "job" + _write_manifest(workspace, prefix="Ft.SPL", stage="identify", metric_value=3) + + def invalid_model(payload: dict, _stages: list[str]): + response = _valid_api_arrangement(payload) + response["sections"][0]["paragraphs"][0]["sentence_ids"][0] = "unknown.sentence" + return response, {"status": "generated"} + + monkeypatch.setattr(report_module, "_generate_api_narrative", invalid_model) + result = _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "success"}, + use_model=True, + ) + + markdown = result.report_path.read_text(encoding="utf-8") + manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) + assert result.generation_mode == "fallback_script" + assert "最终成员数：3" in markdown + assert manifest["model"]["reason"] == "api_unavailable_or_boundary_violation" + + +def test_missing_and_duplicate_model_sentence_ids_fall_back_for_the_whole_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workspace = tmp_path / "job" + _write_manifest(workspace, prefix="Ft.SPL", stage="identify", metric_value=3) + + def invalid_model(payload: dict, _stages: list[str]): + response = _valid_api_arrangement(payload) + paragraph = next( + item["paragraphs"][0] + for item in response["sections"] + if len(item["paragraphs"][0]["sentence_ids"]) >= 2 + ) + paragraph["sentence_ids"][-1] = paragraph["sentence_ids"][0] + return response, {"status": "generated"} + + monkeypatch.setattr(report_module, "_generate_api_narrative", invalid_model) + result = _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "success"}, + use_model=True, + ) + + markdown = result.report_path.read_text(encoding="utf-8") + manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) + assert result.generation_mode == "fallback_script" + assert "最终成员数：3" in markdown + assert manifest["model"]["reason"] == "api_unavailable_or_boundary_violation" + + +@pytest.mark.parametrize( + "violation", + ["unknown_connector", "extra_field", "free_text_field", "changed_section"], +) +def test_model_schema_or_connector_boundary_violation_falls_back_for_the_whole_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, violation: str +) -> None: + workspace = tmp_path / "job" + _write_manifest(workspace, prefix="Ft.SPL", stage="identify", metric_value=3) + + def invalid_model(payload: dict, _stages: list[str]): + response = _valid_api_arrangement(payload) + if violation == "unknown_connector": + paragraph = next( + item["paragraphs"][0] + for item in response["sections"] + if item["paragraphs"][0]["connector_ids"] + ) + paragraph["connector_ids"][0] = "invented" + elif violation == "extra_field": + response["unexpected"] = True + elif violation == "free_text_field": + response["sections"][0]["paragraphs"][0]["text"] = "API 自由文本" + else: + response["sections"][0]["section_id"] = "changed.section" + return response, {"status": "generated"} + + monkeypatch.setattr(report_module, "_generate_api_narrative", invalid_model) + result = _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "success"}, + use_model=True, + ) + + markdown = result.report_path.read_text(encoding="utf-8") + manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) + assert result.generation_mode == "fallback_script" + assert "最终成员数：3" in markdown + assert "API 自由文本" not in markdown + assert manifest["model"]["reason"] == "api_unavailable_or_boundary_violation" + + +def test_open_reuses_existing_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workspace = tmp_path / "job" + manifest_path = _write_manifest(workspace, prefix="Ft.SPL", stage="identify") + calls = {"count": 0} + + def valid_model(payload: dict, _stages: list[str]): + calls["count"] += 1 + return ( + _valid_api_arrangement(payload), + {"status": "generated", "provider": "test", "model": "test-model"}, + ) + + monkeypatch.setattr(report_module, "_generate_api_narrative", valid_model) + first = _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "success"}, + use_model=True, + ) + reopened = _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "success"}, + use_model=True, + action="open", + ) + assert first.reused is False + assert reopened.reused is True + assert calls["count"] == 1 + + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["metrics"]["member_count"] = 2 + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False), encoding="utf-8") + reopened_after_source_change = _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "success"}, + use_model=True, + action="open", + ) + assert reopened_after_source_change.reused is True + assert calls["count"] == 1 + + +def test_no_successful_evidence_creates_no_empty_report(tmp_path: Path) -> None: + workspace = tmp_path / "job" + _write_manifest(workspace, prefix="Ft.SPL", stage="identify", status="failed") + + with pytest.raises(NoSuccessfulGeneFamilyEvidence, match="暂无可生成"): + _generate( + workspace, + expected_stages=["identify"], + stage_statuses={"identify": "failed"}, + ) + + assert not list((workspace / "final").glob("*.gene_family_report.*")) diff --git a/tests/unit/test_gene_family_report_intent.py b/tests/unit/test_gene_family_report_intent.py new file mode 100644 index 000000000..f5dcad547 --- /dev/null +++ b/tests/unit/test_gene_family_report_intent.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import asyncio +import json + +import pytest + +from gfa_core.mas.core.fast_intent import fast_intent_recognition +from gfa_core.mas.utils.intent_parser import parse_intent_from_query + + +@pytest.mark.parametrize( + ("query", "action"), + [ + ("生成结果报告", "generate"), + ("生成基因家族结果报告", "generate"), + ("打开基因家族结果报告", "open"), + ], +) +def test_explicit_gene_family_report_intent_is_provider_independent(query: str, action: str) -> None: + for parsed in (parse_intent_from_query(query), fast_intent_recognition(query)): + assert parsed is not None + assert parsed["stages"] == ["gene_family_report"] + assert parsed["params"]["geneFamilyReportAction"] == action + assert parsed["params"]["reportAction"] == action + assert parsed["method"] == "gene_family_report_deterministic" + + +def test_rnaseq_report_intent_is_not_stolen() -> None: + for parsed in ( + parse_intent_from_query("打开 RNA-seq 结果报告"), + fast_intent_recognition("打开 RNA-seq 结果报告"), + ): + assert not parsed or parsed.get("stages") != ["gene_family_report"] + + +@pytest.mark.parametrize( + ("query", "action"), + [ + ("生成结果报告", "generate"), + ("生成基因家族结果报告", "generate"), + ("打开基因家族结果报告", "open"), + ], +) +def test_gene_family_report_chat_bypasses_router( + monkeypatch, + tmp_path, + query: str, + action: str, +) -> None: + from apps.web.vue3 import main as web_main + from gfa_core.mas.agents import router as router_module + + final_dir = tmp_path / "workspace" / "job-report" / "final" + final_dir.mkdir(parents=True) + (final_dir / "Ft.SPL.gene_family_report.manifest.json").write_text( + json.dumps( + { + "stage": "gene_family_report", + "status": "success", + "prefix": "Ft.SPL", + "parameters": { + "expected_stages": ["identify", "motif"], + "stage_statuses": {"identify": "success", "motif": "success"}, + }, + } + ), + encoding="utf-8", + ) + + async def forbidden_router(*_args, **_kwargs): + raise AssertionError("report intent must return before Router/LLM") + + class ForbiddenOrchestrator: + async def plan(self, *_args, **_kwargs): + raise AssertionError("report plan must not call the LLM orchestrator") + + monkeypatch.setattr(web_main, "PROJECT_ROOT", tmp_path) + monkeypatch.setattr(web_main, "WORKSPACE_ROOT", tmp_path / "workspace") + monkeypatch.setattr(web_main, "agent_conversation_store", None, raising=False) + monkeypatch.setattr(web_main, "llm_manager", object()) + monkeypatch.setattr(web_main, "orchestrator", ForbiddenOrchestrator()) + monkeypatch.setattr(web_main, "MAS_AVAILABLE", True) + monkeypatch.setattr(router_module.RouterAgent, "execute", forbidden_router) + + result = asyncio.run( + web_main._run_agent_chat( + web_main.AgentChatRequest(message=query, job_id="job-report") + ) + ) + + assert result["action"] == "propose_analysis" + assert result["model_used"] == "rule" + assert result["intent"]["stages"] == ["gene_family_report"] + assert result["intent"]["prefix"] == "Ft.SPL" + assert result["intent"]["params"]["reportAction"] == action + assert result["intent"]["params"]["expectedStages"] == ["identify", "motif"] + + plan = asyncio.run( + web_main.agent_plan( + web_main.AgentPlanRequest( + message=query, + job_id="job-report", + prefix="Ft.SPL", + intent=result["intent"], + ) + ) + ) + assert plan["success"] is True + assert plan["mode"] == "gene_family_report_deterministic" + assert plan["plan"]["stages"] == ["gene_family_report"] + assert plan["plan"]["params"]["expectedStages"] == ["identify", "motif"] diff --git a/tests/unit/test_gene_family_report_triggers.py b/tests/unit/test_gene_family_report_triggers.py new file mode 100644 index 000000000..ae3b53a67 --- /dev/null +++ b/tests/unit/test_gene_family_report_triggers.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +def _request_and_stages(): + from apps.web.vue3.main import ConfirmedPlanExecutionRequest, ConfirmedPlanStage + + stages = [ + ConfirmedPlanStage( + id="identify", + params={ + "scopeId": "scope-a", + "analysisRoute": "post_subfamily_confirm", + "memberSetId": "members-a", + }, + ), + ConfirmedPlanStage(id="motif", dependencies=["identify"]), + ConfirmedPlanStage(id="pep", dependencies=["motif"]), + ] + request = ConfirmedPlanExecutionRequest( + session_id="session-a", + job_id="job-a", + prefix="Ft.SPL", + plan_id="plan-a", + stages=stages, + ) + return request, stages + + +def test_confirmed_plan_terminal_success_requests_final_report() -> None: + from apps.web.vue3.main import _confirmed_plan_report_request + + request, stages = _request_and_stages() + report = _confirmed_plan_report_request( + request, + stages, + {"status": "success", "completed_stages": ["identify", "motif", "pep"]}, + ) + + assert report is not None + assert report.stage == "gene_family_report" + assert report.expectedStages == ["identify", "motif", "pep"] + assert report.stageStatuses == {"identify": "success", "motif": "success", "pep": "success"} + assert report.reportAction == "generate" + + +def test_confirmed_plan_failure_requests_partial_report() -> None: + from apps.web.vue3.main import _confirmed_plan_report_request + + request, stages = _request_and_stages() + report = _confirmed_plan_report_request( + request, + stages, + { + "status": "failed", + "completed_stages": ["identify"], + "failure": {"stage": "motif"}, + }, + ) + + assert report is not None + assert report.stageStatuses == {"identify": "success", "motif": "failed", "pep": "not_run"} + assert report.scopeId == "scope-a" + assert report.analysisRoute == "post_subfamily_confirm" + assert report.memberSetId == "members-a" + + +def test_paused_plan_does_not_auto_finalize_report() -> None: + from apps.web.vue3.main import _confirmed_plan_report_request + + request, stages = _request_and_stages() + report = _confirmed_plan_report_request( + request, + stages, + {"status": "paused", "completed_stages": ["identify"]}, + ) + assert report is None + + +def test_frontend_batch_terminal_path_calls_report_stage_with_planned_statuses() -> None: + source = (Path(__file__).resolve().parents[2] / "apps/web/vue3/index.html").read_text(encoding="utf-8") + + assert "submitGeneFamilyReportForBatch" in source + assert "executeSingleStageWithParams('gene_family_report'" in source + assert "expectedStages: plannedIds" in source + assert "stageStatuses" in source + assert "if (pausedCount > 0) return" in source + assert "continuationOpened" in source + assert "name: 'gene_family_report'" in source + assert "openGeneratedGeneFamilyReport" in source + + +def test_report_context_prefers_terminal_confirmed_plan(monkeypatch, tmp_path: Path) -> None: + from apps.web.vue3 import main as web_main + + final_dir = tmp_path / "job-a" / "final" + confirmed_dir = final_dir / ".confirmed_plans" + confirmed_dir.mkdir(parents=True) + (final_dir / "Ft.SPL.gene_family_report.manifest.json").write_text( + json.dumps( + { + "stage": "gene_family_report", + "status": "success", + "prefix": "Ft.SPL", + "parameters": { + "expected_stages": ["identify"], + "stage_statuses": {"identify": "success"}, + }, + } + ), + encoding="utf-8", + ) + (confirmed_dir / "0123456789abcdefabcd.json").write_text( + json.dumps( + { + "status": "failed", + "job_id": "job-a", + "prefix": "Ft.SPL", + "stages": ["identify", "motif", "pep", "gene_family_report"], + "completed_stages": ["identify", "motif"], + "failure": {"stage": "pep"}, + "scopeId": "scope-a", + "memberSetId": "members-a", + } + ), + encoding="utf-8", + ) + monkeypatch.setattr(web_main, "WORKSPACE_ROOT", tmp_path) + + context = web_main._gene_family_report_plan_context("job-a") + + assert context == { + "prefix": "Ft.SPL", + "expectedStages": ["identify", "motif", "pep"], + "stageStatuses": {"identify": "success", "motif": "success", "pep": "failed"}, + "scopeId": "scope-a", + "analysisRoute": "", + "memberSetId": "members-a", + "rnaseqScopeId": "", + } + + +def test_report_context_reuses_formal_report_plan_only(monkeypatch, tmp_path: Path) -> None: + from apps.web.vue3 import main as web_main + + final_dir = tmp_path / "job-a" / "final" + final_dir.mkdir(parents=True) + (final_dir / "Ft.SPL.motif.manifest.json").write_text( + json.dumps({"stage": "motif", "status": "success", "prefix": "Ft.SPL"}), + encoding="utf-8", + ) + monkeypatch.setattr(web_main, "WORKSPACE_ROOT", tmp_path) + assert web_main._gene_family_report_plan_context("job-a") is None + + (final_dir / "Ft.SPL.gene_family_report.manifest.json").write_text( + json.dumps( + { + "stage": "gene_family_report", + "status": "success", + "prefix": "Ft.SPL", + "scope_id": "scope-old", + "parameters": { + "expected_stages": ["identify", "motif"], + "stage_statuses": {"identify": "success", "motif": "success"}, + }, + } + ), + encoding="utf-8", + ) + + context = web_main._gene_family_report_plan_context("job-a", "Ft.SPL") + + assert context is not None + assert context["expectedStages"] == ["identify", "motif"] + assert context["stageStatuses"] == {"identify": "success", "motif": "success"} + assert context["scopeId"] == "" + assert context["analysisRoute"] == "" + assert context["memberSetId"] == "" + assert context["rnaseqScopeId"] == "" diff --git a/tests/unit/test_gene_family_report_wiring.py b/tests/unit/test_gene_family_report_wiring.py new file mode 100644 index 000000000..c8555aee0 --- /dev/null +++ b/tests/unit/test_gene_family_report_wiring.py @@ -0,0 +1,347 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys + + +def _write_identify_manifest(workspace: Path, prefix: str = "Ft.SPL") -> Path: + final_dir = workspace / "final" + final_dir.mkdir(parents=True, exist_ok=True) + table = final_dir / f"{prefix}.identify.tsv" + table.write_text("gene\tvalue\nFtSPL1\t1\n", encoding="utf-8") + manifest = final_dir / f"{prefix}.identify.manifest.json" + manifest.write_text( + json.dumps( + { + "manifest_version": "1.0", + "stage": "identify", + "status": "success", + "prefix": prefix, + "metrics": {"member_count": 1}, + "artifacts": { + "members": { + "path": f"final/{table.name}", + "full_path": str(table), + "type": "table", + } + }, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + return manifest + + +def _flag_value(command: list[str], flag: str) -> str: + return command[command.index(flag) + 1] + + +def test_report_stage_is_registered_without_analysis_dependencies() -> None: + from gfa_core.mas.stage_capabilities import CAPABILITIES + from gfa_core.mas.stage_catalog import STAGE_ORDER, is_stage_executable + from gfa_core.mas.tools.stage_tools import create_stage_registry + + capability = CAPABILITIES["gene_family_report"] + stage_tool = create_stage_registry().get("gene_family_report") + + assert "gene_family_report" in STAGE_ORDER + assert is_stage_executable("gene_family_report") + assert capability.consumes == [] + assert stage_tool is not None + assert stage_tool.required_inputs == [] + assert stage_tool.script == "run_gene_family_report.py" + + +def test_report_cli_can_start_as_a_script() -> None: + root = Path(__file__).resolve().parents[2] + completed = subprocess.run( + [sys.executable, "apps/cli/run_gene_family_report.py", "--help"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr + + +def test_report_cli_writes_report_and_manifest(tmp_path: Path) -> None: + from apps.cli.run_gene_family_report import main + + workspace = tmp_path / "job" + _write_identify_manifest(workspace) + result = main( + [ + "--workspace", + str(workspace), + "--prefix", + "Ft.SPL", + "--job-id", + "job-wiring", + "--expected-stages", + "identify", + "--stage-statuses-json", + '{"identify":"success"}', + "--no-model", + ] + ) + + manifest = json.loads(result.manifest_path.read_text(encoding="utf-8")) + assert result.report_path.is_file() + assert result.manifest_path.is_file() + assert "gene_family_report" in manifest["artifacts"] + + +def test_pipeline_backend_passes_plan_status_and_identity_to_report_cli( + tmp_path: Path, monkeypatch +) -> None: + from apps.web import pipeline_backend + + workspace = tmp_path / "job" + input_dir = workspace / "input" + logs_dir = workspace / "logs" + input_dir.mkdir(parents=True) + logs_dir.mkdir(parents=True) + _write_identify_manifest(workspace) + captured: dict[str, list[str]] = {} + + def capture(command, _log_file, cwd=None): + captured["command"] = [str(value) for value in command] + return 0 + + monkeypatch.setattr(pipeline_backend, "can_start_new_task", lambda: (True, "")) + monkeypatch.setattr(pipeline_backend, "run_cmd_logged", capture) + job = pipeline_backend.JobInfo( + job_id="job-backend", + input_dir=str(input_dir), + workspace_dir=str(workspace), + logs_dir=str(logs_dir), + created_at="2026-08-08 00:00:00", + ) + + pipeline_backend.run_stage( + "gene_family_report", + job, + "Ft.SPL", + expected_stages=["identify", "motif"], + stage_statuses={"identify": "success", "motif": "failed"}, + report_action="open", + report_use_model=False, + scope_id="scope-a", + analysis_route="post_subfamily_confirm", + member_set_id="members-a", + rnaseq_scope_id="rna-a", + ) + + command = captured["command"] + assert command[1:3] == ["apps/cli/run_gene_family_report.py", "--workspace"] + assert _flag_value(command, "--expected-stages") == "identify,motif" + assert json.loads(_flag_value(command, "--stage-statuses-json")) == { + "identify": "success", + "motif": "failed", + } + assert _flag_value(command, "--action") == "open" + assert _flag_value(command, "--job-id") == "job-backend" + assert _flag_value(command, "--scope-id") == "scope-a" + assert _flag_value(command, "--analysis-route") == "post_subfamily_confirm" + assert _flag_value(command, "--member-set-id") == "members-a" + assert _flag_value(command, "--rnaseq-scope-id") == "rna-a" + assert "--no-model" in command + assert not any("run_identify.py" in value or "run_motif.py" in value for value in command) + + +def test_pipeline_backend_rejects_empty_plan_before_starting_report_cli( + tmp_path: Path, monkeypatch +) -> None: + from apps.web import pipeline_backend + + workspace = tmp_path / "job" + input_dir = workspace / "input" + logs_dir = workspace / "logs" + input_dir.mkdir(parents=True) + logs_dir.mkdir(parents=True) + invoked = False + + def capture(*_args, **_kwargs): + nonlocal invoked + invoked = True + return 0 + + monkeypatch.setattr(pipeline_backend, "can_start_new_task", lambda: (True, "")) + monkeypatch.setattr(pipeline_backend, "run_cmd_logged", capture) + job = pipeline_backend.JobInfo( + job_id="job-empty-plan", + input_dir=str(input_dir), + workspace_dir=str(workspace), + logs_dir=str(logs_dir), + created_at="2026-08-08 00:00:00", + ) + + message, _manifest = pipeline_backend.run_stage( + "gene_family_report", + job, + "Ft.SPL", + expected_stages=[], + report_action="generate", + report_use_model=False, + ) + + assert "expected_stages" in message + assert invoked is False + + +def test_stage_tool_executor_forwards_gene_family_report_context( + tmp_path: Path, monkeypatch +) -> None: + import asyncio + import types + from dataclasses import dataclass + + from gfa_core.mas.tools.stage_tools import StageTool, StageToolExecutor + + captured = {} + + @dataclass + class FakeJobInfo: + job_id: str + input_dir: str + workspace_dir: str + logs_dir: str + created_at: str + + def fake_run_stage(**kwargs): + captured.update(kwargs) + manifest = Path(kwargs["job"].workspace_dir) / "final" / "Ft.SPL.gene_family_report.manifest.json" + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text( + json.dumps({"stage": "gene_family_report", "status": "success"}), + encoding="utf-8", + ) + return "报告已生成", manifest + + fake_backend = types.ModuleType("apps.web.pipeline_backend") + fake_backend.JobInfo = FakeJobInfo + fake_backend.run_stage = fake_run_stage + monkeypatch.setitem(sys.modules, "apps.web.pipeline_backend", fake_backend) + + result = asyncio.run( + StageToolExecutor(project_root=tmp_path).execute( + StageTool( + "gene_family_report", + "Gene Family Report", + "run_gene_family_report.py", + "report", + "test", + timeout=5, + ), + "job-report", + "Ft.SPL", + { + "expectedStages": ["identify", "motif"], + "stageStatuses": {"identify": "success", "motif": "failed"}, + "reportAction": "open", + "reportUseModel": False, + "scopeId": "scope-a", + "rnaseqScopeId": "rna-a", + }, + ) + ) + + assert result["status"] == "success" + assert captured["expected_stages"] == ["identify", "motif"] + assert captured["stage_statuses"] == {"identify": "success", "motif": "failed"} + assert captured["report_action"] == "open" + assert captured["report_use_model"] is False + assert captured["scope_id"] == "scope-a" + assert captured["rnaseq_scope_id"] == "rna-a" + + +def test_execute_report_does_not_invent_scope_identity(monkeypatch) -> None: + import asyncio + from unittest.mock import Mock + + from apps.web.vue3 import main as web_main + from gfa_core.mas import resource_governor + from tasks import execute_single_stage_task + + submitted = Mock() + submitted.return_value.id = "report-task" + monkeypatch.setattr(execute_single_stage_task, "apply_async", submitted) + monkeypatch.setattr( + web_main, + "_ensure_celery_queue_health", + lambda *_args, **_kwargs: ({"available": True}, None), + ) + monkeypatch.setattr( + web_main, + "_ensure_gene_family_scope_file", + lambda **_kwargs: ("", ""), + ) + monkeypatch.setattr(resource_governor, "record_task", lambda **_kwargs: {}) + + request = web_main.ExecuteRequest( + stage="gene_family_report", + job_id="job-report-no-invented-identity", + prefix="Ft.SPL", + expectedStages=["identify", "motif"], + stageStatuses={"identify": "success", "motif": "success"}, + ) + result = asyncio.run(web_main.execute_stage(request)) + + assert result["success"] is True + params = submitted.call_args.kwargs["kwargs"]["params"] + assert params["scopeId"] == "" + assert params["analysisRoute"] == "" + assert params["memberSetId"] == "" + assert params["rnaseqScopeId"] == "" + + +def test_report_artifact_validation_does_not_require_invented_identity(monkeypatch) -> None: + from types import SimpleNamespace + + import tasks + from gfa_core.mas.tools import stage_identity + + captured = {} + identity = SimpleNamespace( + scope_id="", + analysis_route="custom", + member_set_id="Ft.SPL:custom:all_members", + wgs_sample_set_id="", + wgs_reference_id="", + wgs_analysis_id="", + genotype_scope_id="", + phenotype_scope_id="", + covariate_scope_id="", + model_scope_id="", + source_model_scope_id="", + analysis_scope_id="", + ) + monkeypatch.setattr(stage_identity, "resolve_stage_identity", lambda **_kwargs: identity) + + def capture_status(**kwargs): + captured.update(kwargs) + return SimpleNamespace(completed=True) + + monkeypatch.setattr(stage_identity, "get_stage_status_for_identity", capture_status) + + failure = tasks.validate_single_stage_artifacts( + "job-report", + "Ft.SPL", + "gene_family_report", + { + "manifest": { + "stage": "gene_family_report", + "status": "success", + "prefix": "Ft.SPL", + "analysis_route": "", + "member_set_id": "", + } + }, + ) + + assert failure is None + assert captured["strict_identity"] is False