318 lines
8.8 KiB
Python
318 lines
8.8 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from gfa_core import wgs_upstream_manifest
|
||
|
|
from gfa_core.wgs_upstream_manifest import (
|
||
|
|
WgsUpstreamManifestError,
|
||
|
|
materialize_wgs_identity_params,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
SAMPLES = "wgs-samples:" + "a" * 64
|
||
|
|
REFERENCE = "wgs-reference:" + "b" * 64
|
||
|
|
ANALYSIS = "wgs-analysis:" + "c" * 64
|
||
|
|
|
||
|
|
|
||
|
|
def _artifact(
|
||
|
|
workspace: Path,
|
||
|
|
bundle: Path,
|
||
|
|
name: str,
|
||
|
|
role: str,
|
||
|
|
*,
|
||
|
|
sample_id: str = "",
|
||
|
|
) -> dict:
|
||
|
|
path = bundle / name
|
||
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
path.write_bytes(f"{role}:{sample_id or 'cohort'}".encode())
|
||
|
|
record = {
|
||
|
|
"artifact": role,
|
||
|
|
"role": role,
|
||
|
|
"path": str(path.relative_to(workspace)),
|
||
|
|
"full_path": str(path.resolve()),
|
||
|
|
"size": path.stat().st_size,
|
||
|
|
"size_bytes": path.stat().st_size,
|
||
|
|
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||
|
|
}
|
||
|
|
if sample_id:
|
||
|
|
record["sample_id"] = sample_id
|
||
|
|
return record
|
||
|
|
|
||
|
|
|
||
|
|
def _write_manifest(
|
||
|
|
workspace: Path,
|
||
|
|
*,
|
||
|
|
stage: str,
|
||
|
|
roles: tuple[str, ...],
|
||
|
|
method: str = "",
|
||
|
|
sample_set_id: str = SAMPLES,
|
||
|
|
reference_id: str = REFERENCE,
|
||
|
|
) -> Path:
|
||
|
|
run_id = "1" * 32
|
||
|
|
final = workspace / "final"
|
||
|
|
method_part = f".{method}" if method else ""
|
||
|
|
bundle = final / f".{stage}_runs" / "Cohort"
|
||
|
|
if method:
|
||
|
|
bundle = bundle / method
|
||
|
|
bundle = bundle / run_id
|
||
|
|
artifacts = {}
|
||
|
|
for role in roles:
|
||
|
|
sample_id = "S1" if role in {
|
||
|
|
"clean_reads_r1",
|
||
|
|
"clean_reads_r2",
|
||
|
|
"sorted_bam",
|
||
|
|
"sorted_bam_index",
|
||
|
|
"align_log",
|
||
|
|
} else ""
|
||
|
|
artifacts[role] = _artifact(
|
||
|
|
workspace,
|
||
|
|
bundle,
|
||
|
|
f"{role}.dat",
|
||
|
|
role,
|
||
|
|
sample_id=sample_id,
|
||
|
|
)
|
||
|
|
manifest = {
|
||
|
|
"manifest_version": "1.0",
|
||
|
|
"stage": stage,
|
||
|
|
"status": "success",
|
||
|
|
"prefix": "Cohort",
|
||
|
|
"job_id": "job-1",
|
||
|
|
"run_id": run_id,
|
||
|
|
"workspace": str(workspace.resolve()),
|
||
|
|
"wgs_analysis_id": ANALYSIS,
|
||
|
|
"artifacts": artifacts,
|
||
|
|
}
|
||
|
|
if sample_set_id:
|
||
|
|
manifest["wgs_sample_set_id"] = sample_set_id
|
||
|
|
if reference_id:
|
||
|
|
manifest["wgs_reference_id"] = reference_id
|
||
|
|
if method:
|
||
|
|
manifest["method"] = method
|
||
|
|
manifest["parameters"] = {"method": method}
|
||
|
|
path = final / f"Cohort.{stage}{method_part}.manifest.json"
|
||
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
path.write_text(json.dumps(manifest), encoding="utf-8")
|
||
|
|
return path
|
||
|
|
|
||
|
|
|
||
|
|
def test_completed_qc_authority_is_seeded_without_planned_index(
|
||
|
|
tmp_path: Path,
|
||
|
|
) -> None:
|
||
|
|
workspace = tmp_path / "workspace"
|
||
|
|
qc_authority = _write_manifest(
|
||
|
|
workspace,
|
||
|
|
stage="wgs_qc",
|
||
|
|
roles=("clean_reads_r1", "clean_reads_r2"),
|
||
|
|
reference_id="",
|
||
|
|
)
|
||
|
|
|
||
|
|
materialized = wgs_upstream_manifest.materialize_completed_wgs_upstream_params(
|
||
|
|
stage="wgs_align",
|
||
|
|
job_id="job-1",
|
||
|
|
prefix="Cohort",
|
||
|
|
params={},
|
||
|
|
completed_stages={"wgs_qc"},
|
||
|
|
workspace=workspace,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert materialized == {
|
||
|
|
"wgsQcManifest": str(qc_authority.resolve()),
|
||
|
|
"wgs_qc_manifest": str(qc_authority.resolve()),
|
||
|
|
"wgsSampleSetId": SAMPLES,
|
||
|
|
"wgs_sample_set_id": SAMPLES,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def test_empty_stage04_identity_is_materialized_from_exact_align_authority(
|
||
|
|
tmp_path: Path,
|
||
|
|
) -> None:
|
||
|
|
workspace = tmp_path / "workspace"
|
||
|
|
authority = _write_manifest(
|
||
|
|
workspace,
|
||
|
|
stage="wgs_align",
|
||
|
|
roles=("sorted_bam", "sorted_bam_index", "align_log"),
|
||
|
|
)
|
||
|
|
|
||
|
|
materialized = materialize_wgs_identity_params(
|
||
|
|
stage="wgs_markdup",
|
||
|
|
job_id="job-1",
|
||
|
|
prefix="Cohort",
|
||
|
|
params={},
|
||
|
|
workspace=workspace,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert materialized == {
|
||
|
|
"wgsAlignManifest": str(authority.resolve()),
|
||
|
|
"wgs_align_manifest": str(authority.resolve()),
|
||
|
|
"wgsSampleSetId": SAMPLES,
|
||
|
|
"wgs_sample_set_id": SAMPLES,
|
||
|
|
"wgsReferenceId": REFERENCE,
|
||
|
|
"wgs_reference_id": REFERENCE,
|
||
|
|
"wgsAnalysisId": ANALYSIS,
|
||
|
|
"wgs_analysis_id": ANALYSIS,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def test_partial_stage04_identity_fails_without_mixing_sources(
|
||
|
|
tmp_path: Path,
|
||
|
|
) -> None:
|
||
|
|
workspace = tmp_path / "workspace"
|
||
|
|
authority = _write_manifest(
|
||
|
|
workspace,
|
||
|
|
stage="wgs_align",
|
||
|
|
roles=("sorted_bam", "sorted_bam_index", "align_log"),
|
||
|
|
)
|
||
|
|
|
||
|
|
with pytest.raises(WgsUpstreamManifestError, match="partial"):
|
||
|
|
materialize_wgs_identity_params(
|
||
|
|
stage="wgs_markdup",
|
||
|
|
job_id="job-1",
|
||
|
|
prefix="Cohort",
|
||
|
|
params={"wgsAlignManifest": str(authority)},
|
||
|
|
workspace=workspace,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_complete_stage04_identity_is_strictly_validated(
|
||
|
|
tmp_path: Path,
|
||
|
|
) -> None:
|
||
|
|
workspace = tmp_path / "workspace"
|
||
|
|
authority = _write_manifest(
|
||
|
|
workspace,
|
||
|
|
stage="wgs_align",
|
||
|
|
roles=("sorted_bam", "sorted_bam_index", "align_log"),
|
||
|
|
)
|
||
|
|
payload = json.loads(authority.read_text(encoding="utf-8"))
|
||
|
|
artifact_path = Path(payload["artifacts"]["sorted_bam"]["full_path"])
|
||
|
|
artifact_path.write_bytes(b"tampered")
|
||
|
|
|
||
|
|
with pytest.raises(WgsUpstreamManifestError, match="sha256|size"):
|
||
|
|
materialize_wgs_identity_params(
|
||
|
|
stage="wgs_markdup",
|
||
|
|
job_id="job-1",
|
||
|
|
prefix="Cohort",
|
||
|
|
params={
|
||
|
|
"wgsAlignManifest": str(authority),
|
||
|
|
"wgsSampleSetId": SAMPLES,
|
||
|
|
"wgsReferenceId": REFERENCE,
|
||
|
|
"wgsAnalysisId": ANALYSIS,
|
||
|
|
},
|
||
|
|
workspace=workspace,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_stage10_uses_requested_method_and_never_switches_authority(
|
||
|
|
tmp_path: Path,
|
||
|
|
) -> None:
|
||
|
|
workspace = tmp_path / "workspace"
|
||
|
|
bcftools = _write_manifest(
|
||
|
|
workspace,
|
||
|
|
stage="wgs_variant_select",
|
||
|
|
method="bcftools",
|
||
|
|
roles=(
|
||
|
|
"selected_snp_vcf",
|
||
|
|
"selected_snp_vcf_index",
|
||
|
|
"selected_indel_vcf",
|
||
|
|
"selected_indel_vcf_index",
|
||
|
|
),
|
||
|
|
)
|
||
|
|
|
||
|
|
with pytest.raises(WgsUpstreamManifestError, match="gatk"):
|
||
|
|
materialize_wgs_identity_params(
|
||
|
|
stage="wgs_variant_filter",
|
||
|
|
job_id="job-1",
|
||
|
|
prefix="Cohort",
|
||
|
|
params={"wgsMethod": "gatk"},
|
||
|
|
workspace=workspace,
|
||
|
|
)
|
||
|
|
|
||
|
|
materialized = materialize_wgs_identity_params(
|
||
|
|
stage="wgs_variant_filter",
|
||
|
|
job_id="job-1",
|
||
|
|
prefix="Cohort",
|
||
|
|
params={"wgsMethod": "bcftools"},
|
||
|
|
workspace=workspace,
|
||
|
|
)
|
||
|
|
assert materialized["wgsSelectManifest"] == str(bcftools.resolve())
|
||
|
|
assert materialized["wgsMethod"] == "bcftools"
|
||
|
|
assert materialized["wgsSelectAnalysisId"] == ANALYSIS
|
||
|
|
|
||
|
|
|
||
|
|
def test_current_stage_identity_uses_its_own_exact_authority(
|
||
|
|
tmp_path: Path,
|
||
|
|
) -> None:
|
||
|
|
workspace = tmp_path / "workspace"
|
||
|
|
_write_manifest(
|
||
|
|
workspace,
|
||
|
|
stage="wgs_align",
|
||
|
|
roles=("sorted_bam", "sorted_bam_index", "align_log"),
|
||
|
|
)
|
||
|
|
|
||
|
|
identity = materialize_wgs_identity_params(
|
||
|
|
stage="wgs_align",
|
||
|
|
job_id="job-1",
|
||
|
|
prefix="Cohort",
|
||
|
|
params={},
|
||
|
|
workspace=workspace,
|
||
|
|
current_stage=True,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert identity["wgsSampleSetId"] == SAMPLES
|
||
|
|
assert identity["wgsReferenceId"] == REFERENCE
|
||
|
|
assert identity["wgsAnalysisId"] == ANALYSIS
|
||
|
|
|
||
|
|
|
||
|
|
def test_api_preflight_materializes_wgs_consumer_params(
|
||
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
|
|
) -> None:
|
||
|
|
from apps.web.vue3 import main as web_main
|
||
|
|
|
||
|
|
workspace_root = tmp_path / "workspace"
|
||
|
|
workspace = workspace_root / "job-1"
|
||
|
|
authority = _write_manifest(
|
||
|
|
workspace,
|
||
|
|
stage="wgs_align",
|
||
|
|
roles=("sorted_bam", "sorted_bam_index", "align_log"),
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(web_main, "WORKSPACE_ROOT", workspace_root)
|
||
|
|
|
||
|
|
result = web_main._api_semantic_preflight(
|
||
|
|
"job-1", "wgs_markdup", "Cohort", {}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["valid"]
|
||
|
|
assert result["materialized_params"]["wgsAlignManifest"] == str(
|
||
|
|
authority.resolve()
|
||
|
|
)
|
||
|
|
assert result["materialized_params"]["wgsAnalysisId"] == ANALYSIS
|
||
|
|
|
||
|
|
|
||
|
|
def test_api_preflight_rejects_partial_wgs_consumer_params(
|
||
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||
|
|
) -> None:
|
||
|
|
from apps.web.vue3 import main as web_main
|
||
|
|
|
||
|
|
workspace_root = tmp_path / "workspace"
|
||
|
|
workspace = workspace_root / "job-1"
|
||
|
|
authority = _write_manifest(
|
||
|
|
workspace,
|
||
|
|
stage="wgs_align",
|
||
|
|
roles=("sorted_bam", "sorted_bam_index", "align_log"),
|
||
|
|
)
|
||
|
|
monkeypatch.setattr(web_main, "WORKSPACE_ROOT", workspace_root)
|
||
|
|
|
||
|
|
result = web_main._api_semantic_preflight(
|
||
|
|
"job-1",
|
||
|
|
"wgs_markdup",
|
||
|
|
"Cohort",
|
||
|
|
{"wgsAlignManifest": str(authority)},
|
||
|
|
)
|
||
|
|
|
||
|
|
assert not result["valid"]
|
||
|
|
assert result["issues"][0]["code"] == "invalid_wgs_manifest_identity"
|
||
|
|
assert "partial" in result["issues"][0]["message"]
|