from __future__ import annotations import gzip import hashlib import importlib import json import asyncio from pathlib import Path from gfa_core import gwas_fmt_context as gwas def _write_vcf(path: Path, sample: str = "S1", pos: int = 10) -> Path: path.parent.mkdir(parents=True, exist_ok=True) content = ( "##fileformat=VCFv4.2\n" f"#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{sample}\n" f"1\t{pos}\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\n" ) if path.name.lower().endswith(".gz"): with gzip.open(path, "wt", encoding="utf-8") as handle: handle.write(content) else: path.write_text(content, encoding="utf-8") return path def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _resolve(project_root: Path, job_id: str = "job_test", selected: str = "") -> dict: resolver = getattr(gwas, "resolve_gwas_fmt_input", None) assert callable(resolver), "resolve_gwas_fmt_input must be implemented" return resolver( job_id=job_id, prefix="cohort", params={"gwasVcf": selected} if selected else {}, project_root=project_root, ) def _write_manifest(final_dir: Path, vcf: Path, *, status: str = "success", digest: str | None = None) -> Path: final_dir.mkdir(parents=True, exist_ok=True) manifest = { "schema_version": "1.0", "stage": "wgs_impute", "prefix": "cohort", "status": status, "artifacts": [{ "role": "imputed_snp_vcf", "path": str(vcf), "sha256": digest if digest is not None else (_sha256(vcf) if vcf.exists() else ""), }], } path = final_dir / "cohort.wgs_impute.manifest.json" path.write_text(json.dumps(manifest), encoding="utf-8") return path def _write_empty_vcf(path: Path, sample: str = "S1") -> Path: path.parent.mkdir(parents=True, exist_ok=True) content = ( "##fileformat=VCFv4.2\n" f"#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{sample}\n" ) if path.name.lower().endswith(".gz"): with gzip.open(path, "wt", encoding="utf-8") as handle: handle.write(content) else: path.write_text(content, encoding="utf-8") return path def _write_wgs_manifest( final_dir: Path, stage: str, artifacts: dict[str, Path], *, status: str = "success", digests: dict[str, str] | None = None, ) -> Path: final_dir.mkdir(parents=True, exist_ok=True) manifest = { "manifest_version": "1.0", "stage": stage, "prefix": "cohort", "status": status, "artifacts": { role: { "artifact": role, "role": role, "full_path": str(path), "sha256": (digests or {}).get(role, _sha256(path) if path.exists() else ""), } for role, path in artifacts.items() }, } path = final_dir / f"cohort.{stage}.manifest.json" path.write_text(json.dumps(manifest), encoding="utf-8") return path def test_no_manifest_selects_the_only_input_vcf(tmp_path: Path) -> None: vcf = _write_vcf(tmp_path / "input/job_test/upload.vcf.gz") result = _resolve(tmp_path) assert result["valid"] is True assert result["needs_confirmation"] is False assert result["decisions"]["external_vcf"]["path"] == str(vcf.resolve()) assert result["decisions"]["external_vcf"]["source"] == "input" def test_no_manifest_also_scans_current_job_final(tmp_path: Path) -> None: vcf = _write_vcf(tmp_path / "workspace/job_test/final/upstream.vcf") result = _resolve(tmp_path) assert result["valid"] is True assert result["decisions"]["external_vcf"]["path"] == str(vcf.resolve()) assert result["decisions"]["external_vcf"]["source"] == "final" def test_no_manifest_requires_confirmation_for_distinct_input_and_final_vcfs(tmp_path: Path) -> None: first = _write_vcf(tmp_path / "input/job_test/a.vcf", sample="A", pos=10) second = _write_vcf(tmp_path / "workspace/job_test/final/b.vcf.gz", sample="B", pos=20) result = _resolve(tmp_path) assert result["valid"] is False assert result["needs_confirmation"] is True assert result["issues"][0]["code"] == "ambiguous_external_vcf" assert {item["path"] for item in result["candidates"]} == {str(first.resolve()), str(second.resolve())} def test_no_manifest_deduplicates_identical_vcf_bytes_across_roots(tmp_path: Path) -> None: first = _write_vcf(tmp_path / "input/job_test/a.vcf") second = tmp_path / "workspace/job_test/final/copy.vcf" second.parent.mkdir(parents=True, exist_ok=True) second.write_bytes(first.read_bytes()) result = _resolve(tmp_path) assert result["valid"] is True assert len(result["candidates"]) == 1 assert result["decisions"]["external_vcf"]["sha256"] == _sha256(first) def test_valid_manifest_wins_over_unmanifested_final_candidates(tmp_path: Path) -> None: final_dir = tmp_path / "workspace/job_test/final" authoritative = _write_vcf(final_dir / "upstream.imputed.vcf.gz", sample="M") _write_manifest(final_dir, authoritative) _write_vcf(final_dir / "unmanifested.vcf", sample="F", pos=30) result = _resolve(tmp_path) assert result["valid"] is True assert result["decisions"]["external_vcf"]["path"] == str(authoritative.resolve()) assert result["decisions"]["external_vcf"]["source"] == "manifest" assert {item["path"] for item in result["candidates"]} == {str(authoritative.resolve())} def test_failed_wgs_manifest_allows_uploaded_vcf(tmp_path: Path) -> None: final_dir = tmp_path / "workspace/job_test/final" declared = _write_vcf(final_dir / "declared.vcf") _write_manifest(final_dir, declared, status="failed") _write_vcf(tmp_path / "input/job_test/upload.vcf", sample="I") _write_vcf(final_dir / "other.vcf", sample="F") upload = tmp_path / "input/job_test/upload.vcf" result = _resolve(tmp_path) assert result["valid"] is True assert result["needs_confirmation"] is False assert result["decisions"]["external_vcf"]["path"] == str(upload.resolve()) def test_manifest_hash_mismatch_never_falls_back(tmp_path: Path) -> None: final_dir = tmp_path / "workspace/job_test/final" declared = _write_vcf(final_dir / "declared.vcf") _write_manifest(final_dir, declared, digest="0" * 64) _write_vcf(tmp_path / "input/job_test/upload.vcf", sample="I") result = _resolve(tmp_path) assert result["valid"] is False assert result["issues"][0]["code"] == "invalid_manifest_input" def test_prefers_maf_snp_manifest_and_ignores_empty_indel_artifacts(tmp_path: Path) -> None: final_dir = tmp_path / "workspace/job_test/final" pass_snp = _write_vcf(final_dir / "pass.snp.vcf.gz", sample="P", pos=10) empty_indel = _write_empty_vcf(final_dir / "pass.indel.vcf.gz", sample="P") final_snp = _write_vcf(final_dir / "final.snp.vcf.gz", sample="M", pos=20) _write_wgs_manifest( final_dir, "wgs_variant_filter", {"pass_snp_vcf": pass_snp, "pass_indel_vcf": empty_indel}, ) maf_manifest = _write_wgs_manifest( final_dir, "wgs_maf_missing_filter", {"final_snp_vcf": final_snp}, ) result = _resolve(tmp_path) assert result["valid"] is True assert result["needs_confirmation"] is False decision = result["decisions"]["external_vcf"] assert decision["path"] == str(final_snp.resolve()) assert decision["source_manifest"] == str(maf_manifest) assert {item["path"] for item in result["candidates"]} == {str(final_snp.resolve())} def test_falls_back_through_successful_wgs_snp_stages(tmp_path: Path) -> None: final_dir = tmp_path / "workspace/job_test/final" pass_snp = _write_vcf(final_dir / "pass.snp.vcf.gz", sample="P", pos=10) imputed_snp = _write_vcf(final_dir / "imputed.snp.vcf.gz", sample="I", pos=20) _write_wgs_manifest(final_dir, "wgs_variant_filter", {"pass_snp_vcf": pass_snp}) _write_wgs_manifest(final_dir, "wgs_impute", {"imputed_snp_vcf": imputed_snp}) result = _resolve(tmp_path) assert result["valid"] is True assert result["decisions"]["external_vcf"]["path"] == str(imputed_snp.resolve()) def test_distinct_uploaded_and_authoritative_wgs_vcfs_require_confirmation(tmp_path: Path) -> None: final_dir = tmp_path / "workspace/job_test/final" upload = _write_vcf(tmp_path / "input/job_test/upload.vcf", sample="U", pos=10) final_snp = _write_vcf(final_dir / "final.snp.vcf.gz", sample="M", pos=20) _write_wgs_manifest(final_dir, "wgs_maf_missing_filter", {"final_snp_vcf": final_snp}) result = _resolve(tmp_path) assert result["valid"] is False assert result["needs_confirmation"] is True assert result["issues"][0]["code"] == "ambiguous_external_vcf" assert {item["path"] for item in result["candidates"]} == { str(upload.resolve()), str(final_snp.resolve()), } def test_invalid_authoritative_wgs_snp_never_falls_back(tmp_path: Path) -> None: final_dir = tmp_path / "workspace/job_test/final" final_snp = _write_vcf(final_dir / "final.snp.vcf.gz", sample="M", pos=20) _write_wgs_manifest( final_dir, "wgs_maf_missing_filter", {"final_snp_vcf": final_snp}, digests={"final_snp_vcf": "0" * 64}, ) _write_vcf(tmp_path / "input/job_test/upload.vcf", sample="U", pos=10) result = _resolve(tmp_path) assert result["valid"] is False assert result["needs_confirmation"] is False assert result["issues"][0]["code"] == "invalid_manifest_input" def test_scanner_excludes_gvcf_gwas_outputs_and_symlink_escape(tmp_path: Path) -> None: input_dir = tmp_path / "input/job_test" final_dir = tmp_path / "workspace/job_test/final" _write_vcf(input_dir / "sample.g.vcf.gz") _write_vcf(final_dir / "cohort.gwas_fmt.sorted.vcf.gz") outside = _write_vcf(tmp_path / "outside.vcf") input_dir.mkdir(parents=True, exist_ok=True) (input_dir / "escape.vcf").symlink_to(outside) result = _resolve(tmp_path) assert result["valid"] is False assert result["needs_confirmation"] is False assert result["issues"][0]["code"] == "missing_external_vcf" assert result["candidates"] == [] def test_selected_vcf_must_be_one_of_the_current_candidates(tmp_path: Path) -> None: allowed = _write_vcf(tmp_path / "input/job_test/allowed.vcf") outside = _write_vcf(tmp_path / "outside.vcf", sample="X") rejected = _resolve(tmp_path, selected=str(outside)) accepted = _resolve(tmp_path, selected=str(allowed)) assert rejected["valid"] is False assert rejected["issues"][0]["code"] == "invalid_external_vcf_selection" assert accepted["valid"] is True assert accepted["decisions"]["external_vcf"]["path"] == str(allowed.resolve()) def test_planning_input_check_resolves_single_job_vcf_without_path_prompt(tmp_path: Path) -> None: from gfa_core.mas.agents.planning_v2 import PlanningAgentV2 vcf = _write_vcf(tmp_path / "input/job_test/cohort.vcf.gz") workspace = tmp_path / "workspace/job_test" workspace.mkdir(parents=True) agent = PlanningAgentV2.__new__(PlanningAgentV2) agent.project_root = tmp_path issues = agent._check_input_files( "gwas_fmt", tmp_path / "input/job_test", workspace, "cohort", context_index=None, planned_stages=["gwas_fmt"], ) assert not any(issue.get("fix_action") == "confirm_external_vcf" for issue in issues) resolved = next(issue for issue in issues if issue.get("category") == "input_resolved") assert resolved["resolver"]["decisions"]["external_vcf"]["path"] == str(vcf.resolve()) def test_planning_stage_params_reuse_resolved_gwas_fields() -> None: from gfa_core.mas.agents.planning_v2 import PlanningAgentV2 agent = PlanningAgentV2.__new__(PlanningAgentV2) stages = agent._generate_default_stages( ["gwas_fmt"], "cohort", intent_params={ "gwasVcf": "/srv/job/input/cohort.vcf.gz", "gwasVcfCandidates": [{"path": "/srv/job/input/cohort.vcf.gz", "name": "cohort.vcf.gz"}], }, ) assert stages[0]["params"]["gwasVcf"] == "/srv/job/input/cohort.vcf.gz" assert stages[0]["params"]["gwasVcfCandidates"][0]["name"] == "cohort.vcf.gz" def test_planning_preflight_carries_resolved_vcf_into_intent_params(tmp_path: Path, monkeypatch) -> None: from gfa_core.mas.agents import planning_v2 vcf = _write_vcf(tmp_path / "input/job_test/cohort.vcf") agent = planning_v2.PlanningAgentV2.__new__(planning_v2.PlanningAgentV2) agent.project_root = tmp_path agent.workspace_root = tmp_path / "workspace" agent.input_root = tmp_path / "input" monkeypatch.setattr(planning_v2, "build_project_context", lambda **_kwargs: object()) monkeypatch.setattr( agent, "_check_stage_status_preflight", lambda *_args, **_kwargs: planning_v2.StageStatus(stage="gwas_fmt", status="not_found"), ) monkeypatch.setattr(agent, "_check_dependencies", lambda *_args, **_kwargs: []) monkeypatch.setattr(agent, "_check_resources", lambda: []) intent = {"stages": ["gwas_fmt"], "params": {}} result = asyncio.run(agent._run_preflight_validation(intent, "job_test", "cohort", "")) assert result["valid"] is True assert intent["params"]["gwasVcf"] == str(vcf.resolve()) assert intent["params"]["gwasVcfCandidates"][0]["path"] == str(vcf.resolve()) def test_api_semantic_preflight_binds_single_job_vcf_server_side(tmp_path: Path, monkeypatch) -> None: api = importlib.import_module("apps.web.vue3.main") vcf = _write_vcf(tmp_path / "input/job_test/cohort.vcf.gz") tassel = tmp_path / "run_pipeline.pl" tassel.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") tassel.chmod(0o755) monkeypatch.setattr(api, "PROJECT_ROOT", tmp_path) params = {"tasselPipeline": str(tassel)} result = api._api_semantic_preflight("job_test", "gwas_fmt", "cohort", params) assert result["valid"] is True assert result["decisions"]["external_vcf"]["path"] == str(vcf.resolve()) assert params["gwasVcf"] == str(vcf.resolve()) def test_semantic_preflight_uses_deployment_tassel_for_default_request(tmp_path: Path, monkeypatch) -> None: vcf = _write_vcf(tmp_path / "cohort.vcf") tassel = tmp_path / "tassel-5-standalone/run_pipeline.pl" tassel.parent.mkdir(parents=True) tassel.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") tassel.chmod(0o755) monkeypatch.setenv("TASSEL_PIPELINE", str(tassel)) result = gwas.semantic_external_vcf_preflight({ "gwasVcf": str(vcf), "tasselPipeline": "run_pipeline.pl", }) assert result["valid"] is True assert result["decisions"]["tassel"] == { "requested": "run_pipeline.pl", "path": str(tassel.resolve()), "executable": True, } def test_api_preflight_passes_resolved_deployment_tassel_to_execution_params(tmp_path: Path, monkeypatch) -> None: api = importlib.import_module("apps.web.vue3.main") _write_vcf(tmp_path / "input/job_test/cohort.vcf") tassel = tmp_path / "tassel-5-standalone/run_pipeline.pl" tassel.parent.mkdir(parents=True) tassel.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") tassel.chmod(0o755) monkeypatch.setenv("TASSEL_PIPELINE", str(tassel)) monkeypatch.setattr(api, "PROJECT_ROOT", tmp_path) params = {"tasselPipeline": "run_pipeline.pl"} result = api._api_semantic_preflight("job_test", "gwas_fmt", "cohort", params) assert result["valid"] is True assert params["tasselPipeline"] == str(tassel.resolve()) def test_api_semantic_preflight_returns_confirmation_for_multiple_job_vcfs(tmp_path: Path, monkeypatch) -> None: api = importlib.import_module("apps.web.vue3.main") _write_vcf(tmp_path / "input/job_test/a.vcf", sample="A") _write_vcf(tmp_path / "workspace/job_test/final/b.vcf", sample="B") monkeypatch.setattr(api, "PROJECT_ROOT", tmp_path) result = api._api_semantic_preflight("job_test", "gwas_fmt", "cohort", {}) assert result["valid"] is False assert result["needs_confirmation"] is True assert result["issues"][0]["code"] == "ambiguous_external_vcf" assert len(result["candidates"]) == 2 def test_execute_api_surfaces_the_first_gwas_preflight_root_cause(tmp_path: Path, monkeypatch) -> None: api = importlib.import_module("apps.web.vue3.main") monkeypatch.setattr(api, "PROJECT_ROOT", tmp_path) monkeypatch.setattr(api, "WORKSPACE_ROOT", tmp_path / "workspace") api.tasks.clear() request = api.ExecuteRequest(stage="gwas_fmt", job_id="job_missing", prefix="cohort") result = asyncio.run(api.execute_stage(request)) issue_message = result["semantic_preflight"]["issues"][0]["message"] assert result["status"] == "preflight_failed" assert result["message"] == issue_message