"""Deterministic, job-local authority resolution for root GWAS inputs.""" from __future__ import annotations import hashlib import json import os import re from pathlib import Path from typing import Any, Iterable, Mapping _ROLE_SUFFIXES = { "phenotype_table": (".tsv", ".txt"), "admixture_q": (".q",), "sample_fam": (".fam",), } def _sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _allowed_roots(project_root: Path, extra: Iterable[str | Path] = ()) -> list[Path]: configured = [item for item in os.environ.get("GWAS_EXTERNAL_INPUT_ROOTS", "").split(os.pathsep) if item] roots = [project_root.resolve(), project_root.parent.resolve()] for raw in [*configured, *extra]: try: roots.append(Path(raw).expanduser().resolve(strict=True)) except OSError: continue return list(dict.fromkeys(roots)) def _inside(path: Path, roots: Iterable[Path]) -> bool: for root in roots: try: path.relative_to(root) return True except ValueError: continue return False def _validate_record( record: Mapping[str, Any], *, role: str, source: str, roots: Iterable[Path], workspace: Path | None = None, ) -> tuple[dict[str, Any] | None, str]: raw_path = Path(str(record.get("path") or "")).expanduser() if not raw_path.is_absolute() and workspace is not None: raw_path = workspace / raw_path try: path = raw_path.resolve(strict=True) except OSError: return None, "input path does not exist" if not path.is_file() or not _inside(path, roots): return None, "input path is outside the allowed roots" if role == "phenotype_table" and path.suffix.lower() == ".csv": return None, "CSV phenotype tables are not supported; provide TSV" if role == "phenotype_table": try: lines = path.read_text(encoding="utf-8-sig").splitlines() except (OSError, UnicodeError): return None, "phenotype table has no readable header" if not lines or not lines[0].strip(): return None, "phenotype table first line must contain the header" header = lines[0] if "\t" not in header: return None, "phenotype table must be tab-delimited" declared_size = record.get("size", record.get("size_bytes")) declared_sha = str(record.get("sha256") or "").lower() if ( isinstance(declared_size, bool) or not isinstance(declared_size, int) or declared_size != path.stat().st_size or not re.fullmatch(r"[0-9a-f]{64}", declared_sha) or _sha256(path) != declared_sha ): return None, "input size or sha256 does not match its authority record" return { **dict(record), "path": str(path), "size": path.stat().st_size, "sha256": declared_sha, "source": source, }, "" def _binding_records(workspace: Path, role: str) -> tuple[list[Mapping[str, Any]], str]: path = workspace / "external_inputs.manifest.json" if not path.is_file(): return [], "" try: manifest = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return [], "external input binding is unreadable" if manifest.get("manifest_version") != "1.0": return [], "external input binding version is invalid" inputs = manifest.get("inputs") if isinstance(inputs, dict): if role not in inputs: return [], "" value = inputs.get(role) if isinstance(value, dict): return [value], "" if isinstance(value, list): if value and all(isinstance(item, dict) for item in value): return value, "" return [], f"external input binding role {role} is malformed" if isinstance(inputs, list): if any(not isinstance(item, dict) for item in inputs): return [], "external input binding list contains a malformed entry" return [item for item in inputs if isinstance(item, dict) and item.get("role") == role], "" return [], "external input binding inputs must be an object or list" def _manifest_records(workspace: Path, role: str) -> tuple[list[Mapping[str, Any]], str]: final_dir = workspace / "final" applicable: list[Mapping[str, Any]] = [] for path in sorted(final_dir.glob("*.manifest.json")) if final_dir.is_dir() else []: try: manifest = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): continue if manifest.get("stage") in {"gwas_trait", "gwas_cov"}: continue artifacts = manifest.get("artifacts") if not isinstance(artifacts, dict) or role not in artifacts: continue if manifest.get("status") != "success": return [], f"authoritative Manifest for {role} is not successful" record = artifacts.get(role) if not isinstance(record, dict) or record.get("role") != role: return [], f"authoritative Manifest artifact {role} is malformed" applicable.append({ **record, "source_genotype_scope_id": str( manifest.get("genotype_scope_id") or manifest.get("source_genotype_scope_id") or "" ), "pair_id": str(manifest.get("run_id") or path.name), }) return applicable, "" def _looks_like_role(path: Path, role: str) -> bool: name = path.name.lower() if ".gwas_" in name or not name.endswith(_ROLE_SUFFIXES.get(role, ())): return False try: raw_lines = path.read_text(encoding="utf-8-sig").splitlines() except (OSError, UnicodeError): return False if role == "phenotype_table" and (not raw_lines or not raw_lines[0].strip()): return False lines = [line.strip() for line in raw_lines if line.strip()][:3] if not lines: return False if role == "phenotype_table": return bool("\t" in lines[0] and len(lines[0].split("\t")) >= 2 and not lines[0].startswith("<")) if role == "admixture_q": try: widths = {len(line.split()) for line in lines} return len(widths) == 1 and next(iter(widths)) >= 2 and all( all(0.0 <= float(value) <= 1.0 for value in line.split()) for line in lines ) except (ValueError, StopIteration): return False if role == "sample_fam": return all(len(line.split()) >= 6 for line in lines) return False def _result(candidates: list[dict[str, Any]], selected: str = "") -> dict[str, Any]: deduplicated: dict[str, dict[str, Any]] = {} for candidate in sorted(candidates, key=lambda item: item["path"]): deduplicated.setdefault(candidate["sha256"], candidate) unique = list(deduplicated.values()) if selected: try: selected_path = str(Path(selected).expanduser().resolve(strict=True)) except OSError: selected_path = "" match = next((item for item in unique if item["path"] == selected_path), None) if match is None: return { "valid": False, "needs_confirmation": len(unique) > 1, "candidates": unique, "issues": [{"code": "invalid_root_input_selection", "message": "selected input is not authoritative"}], "decision": {}, } unique = [match] if len(unique) == 1: return {"valid": True, "needs_confirmation": False, "issues": [], "candidates": unique, "decision": unique[0]} code = "ambiguous_root_input" if unique else "missing_root_input" return { "valid": False, "needs_confirmation": len(unique) > 1, "candidates": unique, "issues": [{"code": code, "message": "multiple inputs require confirmation" if unique else "required input is missing"}], "decision": {}, } def resolve_root_input( *, project_root: str | Path, job_id: str, role: str, selected: str = "", allowed_roots: Iterable[str | Path] = (), ) -> dict[str, Any]: project = Path(project_root).resolve() workspace = project / "workspace" / job_id roots = _allowed_roots(project, allowed_roots) manifest_records, manifest_error = _manifest_records(workspace, role) if manifest_error: return {"valid": False, "needs_confirmation": False, "candidates": [], "decision": {}, "issues": [{"code": "invalid_upstream_manifest", "message": manifest_error}]} if manifest_records: validated: list[dict[str, Any]] = [] for record in manifest_records: item, error = _validate_record( record, role=role, source="upstream_manifest", roots=roots, workspace=workspace, ) if error: return {"valid": False, "needs_confirmation": False, "candidates": [], "decision": {}, "issues": [{"code": "invalid_upstream_manifest", "message": error}]} validated.append(item or {}) return _result(validated, selected) binding_records, binding_error = _binding_records(workspace, role) if binding_error: return {"valid": False, "needs_confirmation": False, "candidates": [], "decision": {}, "issues": [{"code": "invalid_external_binding", "message": binding_error}]} if binding_records: validated = [] for record in binding_records: item, error = _validate_record(record, role=role, source="external_binding", roots=roots) if error: return {"valid": False, "needs_confirmation": False, "candidates": [], "decision": {}, "issues": [{"code": "invalid_external_binding", "message": error}]} validated.append(item or {}) return _result(validated, selected) scanned: list[dict[str, Any]] = [] for source, root in (("input", project / "input" / job_id), ("final", workspace / "final")): if not root.is_dir(): continue for path in sorted(root.rglob("*")): if not path.is_file() or not _looks_like_role(path, role): continue scanned.append({ "role": role, "path": str(path.resolve()), "size": path.stat().st_size, "sha256": _sha256(path), "source": source, }) return _result(scanned, selected) def resolve_root_pair( *, project_root: str | Path, job_id: str, first_role: str, second_role: str, first_selected: str = "", second_selected: str = "", ) -> dict[str, Any]: first = resolve_root_input( project_root=project_root, job_id=job_id, role=first_role, selected=first_selected, ) second = resolve_root_input( project_root=project_root, job_id=job_id, role=second_role, selected=second_selected, ) if not first["valid"] or not second["valid"]: return { "valid": False, "needs_confirmation": bool(first.get("needs_confirmation") or second.get("needs_confirmation")), "issues": [*first.get("issues", []), *second.get("issues", [])], "first": {}, "second": {}, } first_record, second_record = first["decision"], second["decision"] first_pair, second_pair = str(first_record.get("pair_id") or ""), str(second_record.get("pair_id") or "") if first_pair or second_pair: if not first_pair or first_pair != second_pair: return {"valid": False, "needs_confirmation": True, "issues": [{"code": "root_input_pair_mismatch", "message": "Q and FAM pair IDs do not match"}], "first": first_record, "second": second_record} first_scope = str(first_record.get("source_genotype_scope_id") or "") second_scope = str(second_record.get("source_genotype_scope_id") or "") if first_record.get("source") == second_record.get("source") == "upstream_manifest": verified_scope = first_scope if first_scope == second_scope else "" else: verified_scope = "" workspace = Path(project_root).resolve() / "workspace" / job_id lineage_paths = [ workspace / "legacy_lineage.manifest.json", workspace / "final" / "legacy_lineage.manifest.json", Path(project_root).resolve() / "input" / job_id / "legacy_lineage.manifest.json", ] existing = list(dict.fromkeys(path.resolve() for path in lineage_paths if path.is_file())) if len(existing) == 1: try: from gfa_core.gwas_legacy_lineage import validate_legacy_lineage lineage = validate_legacy_lineage( existing[0], allowed_roots=_allowed_roots(Path(project_root).resolve()), ) artifacts = lineage["artifacts"] if ( str(artifacts[first_role].get("sha256") or "") == first_record["sha256"] and str(artifacts[second_role].get("sha256") or "") == second_record["sha256"] ): verified_scope = "genotype:" + str(lineage["source_genotype_sha256"]) except (KeyError, ValueError): verified_scope = "" if not re.fullmatch(r"genotype:[0-9a-f]{64}", verified_scope): return {"valid": False, "needs_confirmation": False, "issues": [{"code": "unverified_covariate_lineage", "message": "Q/FAM genotype lineage is not verified"}], "first": first_record, "second": second_record} return { "valid": True, "needs_confirmation": False, "issues": [], "first": first_record, "second": second_record, "source_genotype_scope_id": verified_scope, }