#!/usr/bin/env bash set -euo pipefail PROJECT="PRJNA530826" CONDA_SH="${CONDA_SH:-/home/xia-lab/miniconda3/etc/profile.d/conda.sh}" CONDA_ENV="${CONDA_ENV:-rnaseq}" OUTDIR="${OUTDIR:-workspace/rnaseq_downloads/${PROJECT}}" JOBS="${JOBS:-4}" CONNECTIONS="${CONNECTIONS:-8}" SPLITS="${SPLITS:-8}" MIN_SPLIT_SIZE="${MIN_SPLIT_SIZE:-20M}" RETRY_WAIT="${RETRY_WAIT:-10}" MAX_TRIES="${MAX_TRIES:-10}" usage() { cat <&2 usage >&2 exit 2 ;; esac done mkdir -p "${OUTDIR}" if [[ -f "${CONDA_SH}" ]]; then # shellcheck source=/dev/null source "${CONDA_SH}" conda activate "${CONDA_ENV}" || { echo "ERROR: failed to activate conda env: ${CONDA_ENV}" >&2 exit 1 } else echo "WARN: conda init script not found: ${CONDA_SH}; using current PATH" >&2 fi if ! command -v aria2c >/dev/null 2>&1; then cat >&2 </dev/null 2>&1; then echo "ERROR: curl is required to fetch the ENA file list." >&2 exit 127 fi META_TSV="${OUTDIR}/${PROJECT}.ena_fastq.tsv" URLS_TXT="${OUTDIR}/${PROJECT}.aria2c.urls.txt" MD5_TXT="${OUTDIR}/${PROJECT}.fastq.md5" LOG_FILE="${OUTDIR}/${PROJECT}.aria2c.log" ENA_URL="https://www.ebi.ac.uk/ena/portal/api/filereport?accession=${PROJECT}&result=read_run&fields=run_accession,sample_accession,experiment_accession,scientific_name,fastq_ftp,fastq_aspera,fastq_bytes,fastq_md5&format=tsv&download=false&limit=0" echo "Fetching ENA FASTQ metadata: ${PROJECT}" curl -L -sS "${ENA_URL}" -o "${META_TSV}" awk -F '\t' ' NR > 1 && $5 != "" { n = split($5, urls, ";") for (i = 1; i <= n; i++) { if (urls[i] != "") { u = urls[i] sub(/^ftp:\/\//, "", u) print "https://" u } } } ' "${META_TSV}" > "${URLS_TXT}" awk -F '\t' ' NR > 1 && $8 != "" && $5 != "" { n = split($5, urls, ";") m = split($8, md5s, ";") for (i = 1; i <= n; i++) { if (urls[i] != "" && md5s[i] != "") { file = urls[i] sub(/^.*\//, "", file) print md5s[i] " " file } } } ' "${META_TSV}" > "${MD5_TXT}" URL_COUNT="$(wc -l < "${URLS_TXT}")" if [[ "${URL_COUNT}" -eq 0 ]]; then echo "ERROR: no FASTQ URLs were generated. Check ${META_TSV}" >&2 exit 1 fi echo "FASTQ URL count: ${URL_COUNT}" echo "Output directory: ${OUTDIR}" echo "aria2c settings: jobs=${JOBS}, connections=${CONNECTIONS}, splits=${SPLITS}" aria2c \ --dir="${OUTDIR}" \ --input-file="${URLS_TXT}" \ --continue=true \ --max-concurrent-downloads="${JOBS}" \ --max-connection-per-server="${CONNECTIONS}" \ --split="${SPLITS}" \ --min-split-size="${MIN_SPLIT_SIZE}" \ --retry-wait="${RETRY_WAIT}" \ --max-tries="${MAX_TRIES}" \ --file-allocation=none \ --auto-file-renaming=false \ --allow-overwrite=false \ --summary-interval=60 \ --console-log-level=notice \ --log="${LOG_FILE}" echo "Download finished." echo "URL list: ${URLS_TXT}" echo "MD5 list: ${MD5_TXT}" echo "Log file: ${LOG_FILE}" echo "To verify MD5 after download:" echo " cd '${OUTDIR}' && md5sum -c '${PROJECT}.fastq.md5'"